$ oc -n openshift-monitoring logs -c telemeter-client deploy/telemeter-client
level=info caller=main.go:102 ts=2023-08-14T07:37:24.991515121Z msg="telemeter client initialized"
level=warn caller=forwarder.go:139 ts=2023-08-14T07:37:24.991680881Z component=forwarder msg="not anonymizing any labels"
level=debug caller=forwarder.go:156 ts=2023-08-14T07:37:24.991690481Z component=forwarder msg="TLS configuration" ca_file=/etc/serving-certs-ca-bundle/service-ca.crt cert_file=/etc/tls/private/tls.crt key_file=/etc/tls/private/tls.key
level=debug caller=forwarder.go:191 ts=2023-08-14T07:37:25.005893919Z component=forwarder msg="enabling the token file round tripper for the fromClient transport"
level=debug caller=forwarder.go:199 ts=2023-08-14T07:37:25.005955039Z component=forwarder msg="enabling the token round tripper for the fromClient transport"
level=info caller=main.go:301 ts=2023-08-14T07:37:25.005997559Z msg="starting telemeter-client" from=https://prometheus-k8s.openshift-monitoring.svc:9092 to=https://infogw.api.openshift.com/ listen=localhost:8080
level=warn caller=forwarder.go:139 ts=2023-08-14T07:40:25.105405836Z component=forwarder msg="not anonymizing any labels"
level=debug caller=forwarder.go:156 ts=2023-08-14T07:40:25.105437676Z component=forwarder msg="TLS configuration" ca_file=/etc/serving-certs-ca-bundle/service-ca.crt cert_file=/etc/tls/private/tls.crt key_file=/etc/tls/private/tls.key
level=debug caller=forwarder.go:191 ts=2023-08-14T07:40:25.105731876Z component=forwarder msg="enabling the token file round tripper for the fromClient transport"
level=debug caller=forwarder.go:199 ts=2023-08-14T07:40:25.105786796Z component=forwarder msg="enabling the token round tripper for the fromClient transport"
level=error caller=forwarder.go:296 ts=2023-08-15T00:38:30.110408971Z component=forwarder/worker msg="unable to forward results" err="Prometheus server forbidden: https://prometheus-k8s.openshift-monitoring.svc:9092/federate?match%5B%5D=%7B__name__%3D~%22cluster%3Ausage%3A.%2A%22%7D&match%5B%5D=%7B__name__%3D%22count%3Aup0%22%7D&match%5B%5D=%7B__name__%3D%22count%3Aup1%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_available_updates%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_capability%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22cluster_installer%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_feature_set%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_object_counts%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%2Calertstate%3D%22firing%22%7D&match%5B%5D=%7B__name__%3D%22code%3Aapiserver_request_total%3Arate%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22workload%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22workload%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avirt_platform_nodes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_instance_type_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avmi_request_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_role_os_version_machine%3Acpu_capacity_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_role_os_version_machine%3Acpu_capacity_sockets%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22subscription_sync_total%22%7D&match%5B%5D=%7B__name__%3D%22olm_resolution_duration_seconds%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akube_persistentvolumeclaim_resource_requests_storage_bytes%3Aprovisioner%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akubelet_volume_stats_used_bytes%3Aprovisioner%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ceph_cluster_total_bytes%22%7D&match%5B%5D=%7B__name__%3D%22ceph_cluster_total_used_raw_bytes%22%7D&match%5B%5D=%7B__name__%3D%22ceph_health_status%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_raw_capacity_total_bytes%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_raw_capacity_used_bytes%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_osd_metadata%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Akube_pv%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aodf_system_pvs%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_pools_iops%3Atotal%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_pools_iops_bytes%3Atotal%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_versions_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_total_unhealthy_buckets%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_bucket_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_total_object_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_bucket_count%22%2C+system_type%3D%22OCS%22%2C+system_vendor%3D%22Red+Hat%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_objects_total%22%2C+system_type%3D%22OCS%22%2C+system_vendor%3D%22Red+Hat%22%7D&match%5B%5D=%7B__name__%3D%22noobaa_accounts_num%22%7D&match%5B%5D=%7B__name__%3D%22noobaa_total_usage%22%7D&match%5B%5D=%7B__name__%3D%22console_url%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_requests_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_successes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_failures_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_logout_requests_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_usage_users%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_plugins_info%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_customization_perspectives_info%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aovnkube_master_egress_routing_via_host%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anetwork_attachment_definition_instances%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anetwork_attachment_definition_enabled_instance_up%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aingress_controller_aws_nlb_active%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amin%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Aavg%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amedian%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aopenshift_route_info%3Atls_termination%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22insightsclient_request_send_total%22%7D&match%5B%5D=%7B__name__%3D%22cam_app_workload_migrations%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aapiserver_current_inflight_requests%3Asum%3Amax_over_time%3A2m%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aalertmanager_integrations%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Atelemetry_selected_series%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Aprometheus_tsdb_head_series%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Aprometheus_tsdb_head_samples_appended_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22monitoring%3Acontainer_memory_working_set_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_job%3Ascrape_series_added%3Atopk3_sum1h%22%7D&match%5B%5D=%7B__name__%3D%22namespace_job%3Ascrape_samples_post_metric_relabeling%3Atopk3%22%7D&match%5B%5D=%7B__name__%3D%22monitoring%3Ahaproxy_server_http_responses_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22profile%3Acluster_monitoring_operator_collection_profile%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhmi_status%22%7D&match%5B%5D=%7B__name__%3D%22status%3Aupgrading%3Aversion%3Arhoam_state%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22state%3Arhoam_critical_alerts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22state%3Arhoam_warning_alerts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhoam_7d_slo_percentile%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhoam_7d_slo_remaining_error_budget%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster_legacy_scheduler_policy%22%7D&match%5B%5D=%7B__name__%3D%22cluster_master_schedulable%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_status%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_started_total%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_failure_total%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_start_time_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_start_time_seconds_count%22%7D&match%5B%5D=%7B__name__%3D%22cco_credentials_mode%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akube_persistentvolume_plugin_type_counts%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22visual_web_terminal_sessions_total%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_vcenter_info%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_esxi_version_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_node_hw_version_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Abuild_by_strategy%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22rhods_aggregate_availability%22%7D&match%5B%5D=%7B__name__%3D%22rhods_total_users%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_disk_wal_fsync_duration_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_mvcc_db_total_size_in_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_network_peer_round_trip_time_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_mvcc_db_total_size_in_use_in_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_disk_backend_commit_duration_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_storage_types%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_strategies%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_agent_strategies%22%7D&match%5B%5D=%7B__name__%3D%22appsvcs%3Acores_by_product%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22nto_custom_profiles%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_configmap%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_secret%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_mount_failures_total%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_mount_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_storage_info%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_redundancy_policy_info%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_defined_delete_namespaces_total%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_misconfigured_memory_resources_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_data_nodes_total%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_documents_created_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_documents_deleted_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22pod%3Aeo_es_shards_total%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_cluster_management_state_info%22%7D&match%5B%5D=%7B__name__%3D%22imageregistry%3Aimagestreamtags_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22imageregistry%3Aoperations_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22log_logging_info%22%7D&match%5B%5D=%7B__name__%3D%22log_collector_error_count_total%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_pipeline_info%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_input_info%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_output_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Alog_collected_bytes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Alog_logged_bytes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akata_monitor_running_shim_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22platform%3Ahypershift_hostedclusters%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22platform%3Ahypershift_nodepools%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_bucket_claims%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_buckets_claims%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_namespace_resources%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_namespace_resources%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_namespace_buckets%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_namespace_buckets%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_accounts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_usage%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_system_health_status%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22ocs_advanced_feature_usage%22%7D&match%5B%5D=%7B__name__%3D%22os_image_url_override%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_topology_tags%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_infrastructure_failure_domains%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_csi_migration%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22apiserver_list_watch_request_success_total%3Arate%3Asum%22%2C+verb%3D~%22LIST%7CWATCH%22%7D"
level=error caller=forwarder.go:296 ts=2023-08-15T00:39:30.137306555Z component=forwarder/worker msg="unable to forward results" err="Prometheus server forbidden: https://prometheus-k8s.openshift-monitoring.svc:9092/federate?match%5B%5D=%7B__name__%3D~%22cluster%3Ausage%3A.%2A%22%7D&match%5B%5D=%7B__name__%3D%22count%3Aup0%22%7D&match%5B%5D=%7B__name__%3D%22count%3Aup1%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_available_updates%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_capability%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22cluster_installer%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_feature_set%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_object_counts%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%2Calertstate%3D%22firing%22%7D&match%5B%5D=%7B__name__%3D%22code%3Aapiserver_request_total%3Arate%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22workload%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22workload%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avirt_platform_nodes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_instance_type_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avmi_request_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_role_os_version_machine%3Acpu_capacity_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_role_os_version_machine%3Acpu_capacity_sockets%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22subscription_sync_total%22%7D&match%5B%5D=%7B__name__%3D%22olm_resolution_duration_seconds%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akube_persistentvolumeclaim_resource_requests_storage_bytes%3Aprovisioner%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akubelet_volume_stats_used_bytes%3Aprovisioner%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ceph_cluster_total_bytes%22%7D&match%5B%5D=%7B__name__%3D%22ceph_cluster_total_used_raw_bytes%22%7D&match%5B%5D=%7B__name__%3D%22ceph_health_status%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_raw_capacity_total_bytes%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_raw_capacity_used_bytes%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_osd_metadata%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Akube_pv%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aodf_system_pvs%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_pools_iops%3Atotal%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_pools_iops_bytes%3Atotal%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_versions_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_total_unhealthy_buckets%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_bucket_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_total_object_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_bucket_count%22%2C+system_type%3D%22OCS%22%2C+system_vendor%3D%22Red+Hat%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_objects_total%22%2C+system_type%3D%22OCS%22%2C+system_vendor%3D%22Red+Hat%22%7D&match%5B%5D=%7B__name__%3D%22noobaa_accounts_num%22%7D&match%5B%5D=%7B__name__%3D%22noobaa_total_usage%22%7D&match%5B%5D=%7B__name__%3D%22console_url%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_requests_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_successes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_failures_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_logout_requests_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_usage_users%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_plugins_info%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_customization_perspectives_info%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aovnkube_master_egress_routing_via_host%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anetwork_attachment_definition_instances%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anetwork_attachment_definition_enabled_instance_up%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aingress_controller_aws_nlb_active%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amin%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Aavg%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amedian%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aopenshift_route_info%3Atls_termination%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22insightsclient_request_send_total%22%7D&match%5B%5D=%7B__name__%3D%22cam_app_workload_migrations%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aapiserver_current_inflight_requests%3Asum%3Amax_over_time%3A2m%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aalertmanager_integrations%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Atelemetry_selected_series%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Aprometheus_tsdb_head_series%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Aprometheus_tsdb_head_samples_appended_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22monitoring%3Acontainer_memory_working_set_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_job%3Ascrape_series_added%3Atopk3_sum1h%22%7D&match%5B%5D=%7B__name__%3D%22namespace_job%3Ascrape_samples_post_metric_relabeling%3Atopk3%22%7D&match%5B%5D=%7B__name__%3D%22monitoring%3Ahaproxy_server_http_responses_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22profile%3Acluster_monitoring_operator_collection_profile%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhmi_status%22%7D&match%5B%5D=%7B__name__%3D%22status%3Aupgrading%3Aversion%3Arhoam_state%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22state%3Arhoam_critical_alerts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22state%3Arhoam_warning_alerts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhoam_7d_slo_percentile%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhoam_7d_slo_remaining_error_budget%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster_legacy_scheduler_policy%22%7D&match%5B%5D=%7B__name__%3D%22cluster_master_schedulable%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_status%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_started_total%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_failure_total%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_start_time_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_start_time_seconds_count%22%7D&match%5B%5D=%7B__name__%3D%22cco_credentials_mode%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akube_persistentvolume_plugin_type_counts%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22visual_web_terminal_sessions_total%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_vcenter_info%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_esxi_version_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_node_hw_version_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Abuild_by_strategy%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22rhods_aggregate_availability%22%7D&match%5B%5D=%7B__name__%3D%22rhods_total_users%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_disk_wal_fsync_duration_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_mvcc_db_total_size_in_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_network_peer_round_trip_time_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_mvcc_db_total_size_in_use_in_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_disk_backend_commit_duration_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_storage_types%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_strategies%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_agent_strategies%22%7D&match%5B%5D=%7B__name__%3D%22appsvcs%3Acores_by_product%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22nto_custom_profiles%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_configmap%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_secret%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_mount_failures_total%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_mount_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_storage_info%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_redundancy_policy_info%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_defined_delete_namespaces_total%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_misconfigured_memory_resources_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_data_nodes_total%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_documents_created_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_documents_deleted_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22pod%3Aeo_es_shards_total%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_cluster_management_state_info%22%7D&match%5B%5D=%7B__name__%3D%22imageregistry%3Aimagestreamtags_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22imageregistry%3Aoperations_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22log_logging_info%22%7D&match%5B%5D=%7B__name__%3D%22log_collector_error_count_total%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_pipeline_info%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_input_info%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_output_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Alog_collected_bytes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Alog_logged_bytes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akata_monitor_running_shim_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22platform%3Ahypershift_hostedclusters%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22platform%3Ahypershift_nodepools%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_bucket_claims%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_buckets_claims%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_namespace_resources%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_namespace_resources%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_namespace_buckets%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_namespace_buckets%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_accounts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_usage%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_system_health_status%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22ocs_advanced_feature_usage%22%7D&match%5B%5D=%7B__name__%3D%22os_image_url_override%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_topology_tags%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_infrastructure_failure_domains%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_csi_migration%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22apiserver_list_watch_request_success_total%3Arate%3Asum%22%2C+verb%3D~%22LIST%7CWATCH%22%7D"
...

$ token=`oc create token prometheus-k8s -n openshift-monitoring`
$ oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- curl -k -H "Authorization: Bearer $token" 'https://prometheus-k8s.openshift-monitoring.svc:9092/federate?match%5B%5D=%7B__name__%3D~%22cluster%3Ausage%3A.%2A%22%7D&match%5B%5D=%7B__name__%3D%22count%3Aup0%22%7D&match%5B%5D=%7B__name__%3D%22count%3Aup1%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_available_updates%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_capability%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_up%22%7D&match%5B%5D=%7B__name__%3D%22cluster_operator_conditions%22%7D&match%5B%5D=%7B__name__%3D%22cluster_version_payload%22%7D&match%5B%5D=%7B__name__%3D%22cluster_installer%22%7D&match%5B%5D=%7B__name__%3D%22cluster_infrastructure_provider%22%7D&match%5B%5D=%7B__name__%3D%22cluster_feature_set%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_object_counts%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ALERTS%22%2Calertstate%3D%22firing%22%7D&match%5B%5D=%7B__name__%3D%22code%3Aapiserver_request_total%3Arate%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acapacity_memory_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22workload%3Acpu_usage_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22workload%3Amemory_usage_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avirt_platform_nodes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anode_instance_type_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cnv%3Avmi_status_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avmi_request_cpu_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_role_os_version_machine%3Acpu_capacity_cores%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22node_role_os_version_machine%3Acpu_capacity_sockets%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22subscription_sync_total%22%7D&match%5B%5D=%7B__name__%3D%22olm_resolution_duration_seconds%22%7D&match%5B%5D=%7B__name__%3D%22csv_succeeded%22%7D&match%5B%5D=%7B__name__%3D%22csv_abnormal%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akube_persistentvolumeclaim_resource_requests_storage_bytes%3Aprovisioner%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akubelet_volume_stats_used_bytes%3Aprovisioner%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22ceph_cluster_total_bytes%22%7D&match%5B%5D=%7B__name__%3D%22ceph_cluster_total_used_raw_bytes%22%7D&match%5B%5D=%7B__name__%3D%22ceph_health_status%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_raw_capacity_total_bytes%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_raw_capacity_used_bytes%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_health_status%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_osd_metadata%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Akube_pv%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aodf_system_pvs%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_pools_iops%3Atotal%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_pools_iops_bytes%3Atotal%22%7D&match%5B%5D=%7B__name__%3D%22job%3Aceph_versions_running%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_total_unhealthy_buckets%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_bucket_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22job%3Anoobaa_total_object_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_bucket_count%22%2C+system_type%3D%22OCS%22%2C+system_vendor%3D%22Red+Hat%22%7D&match%5B%5D=%7B__name__%3D%22odf_system_objects_total%22%2C+system_type%3D%22OCS%22%2C+system_vendor%3D%22Red+Hat%22%7D&match%5B%5D=%7B__name__%3D%22noobaa_accounts_num%22%7D&match%5B%5D=%7B__name__%3D%22noobaa_total_usage%22%7D&match%5B%5D=%7B__name__%3D%22console_url%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_requests_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_successes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_login_failures_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_auth_logout_requests_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_usage_users%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_plugins_info%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aconsole_customization_perspectives_info%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aovnkube_master_egress_routing_via_host%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anetwork_attachment_definition_instances%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Anetwork_attachment_definition_enabled_instance_up%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aingress_controller_aws_nlb_active%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amin%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Aavg%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aroute_metrics_controller_routes_per_shard%3Amedian%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aopenshift_route_info%3Atls_termination%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22insightsclient_request_send_total%22%7D&match%5B%5D=%7B__name__%3D%22cam_app_workload_migrations%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aapiserver_current_inflight_requests%3Asum%3Amax_over_time%3A2m%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aalertmanager_integrations%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Atelemetry_selected_series%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Aprometheus_tsdb_head_series%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Aprometheus_tsdb_head_samples_appended_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22monitoring%3Acontainer_memory_working_set_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22namespace_job%3Ascrape_series_added%3Atopk3_sum1h%22%7D&match%5B%5D=%7B__name__%3D%22namespace_job%3Ascrape_samples_post_metric_relabeling%3Atopk3%22%7D&match%5B%5D=%7B__name__%3D%22monitoring%3Ahaproxy_server_http_responses_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22profile%3Acluster_monitoring_operator_collection_profile%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhmi_status%22%7D&match%5B%5D=%7B__name__%3D%22status%3Aupgrading%3Aversion%3Arhoam_state%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22state%3Arhoam_critical_alerts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22state%3Arhoam_warning_alerts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhoam_7d_slo_percentile%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22rhoam_7d_slo_remaining_error_budget%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster_legacy_scheduler_policy%22%7D&match%5B%5D=%7B__name__%3D%22cluster_master_schedulable%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_status%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_started_total%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_failure_total%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_start_time_seconds_sum%22%7D&match%5B%5D=%7B__name__%3D%22che_workspace_start_time_seconds_count%22%7D&match%5B%5D=%7B__name__%3D%22cco_credentials_mode%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akube_persistentvolume_plugin_type_counts%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22visual_web_terminal_sessions_total%22%7D&match%5B%5D=%7B__name__%3D%22acm_managed_cluster_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_vcenter_info%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_esxi_version_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_node_hw_version_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22openshift%3Abuild_by_strategy%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22rhods_aggregate_availability%22%7D&match%5B%5D=%7B__name__%3D%22rhods_total_users%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_disk_wal_fsync_duration_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_mvcc_db_total_size_in_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_network_peer_round_trip_time_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_mvcc_db_total_size_in_use_in_bytes%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22instance%3Aetcd_disk_backend_commit_duration_seconds%3Ahistogram_quantile%22%2Cquantile%3D%220.99%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_storage_types%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_strategies%22%7D&match%5B%5D=%7B__name__%3D%22jaeger_operator_instances_agent_strategies%22%7D&match%5B%5D=%7B__name__%3D%22appsvcs%3Acores_by_product%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22nto_custom_profiles%3Acount%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_configmap%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_secret%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_mount_failures_total%22%7D&match%5B%5D=%7B__name__%3D%22openshift_csi_share_mount_requests_total%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_storage_info%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_redundancy_policy_info%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_defined_delete_namespaces_total%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_misconfigured_memory_resources_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_data_nodes_total%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_documents_created_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Aeo_es_documents_deleted_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22pod%3Aeo_es_shards_total%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22eo_es_cluster_management_state_info%22%7D&match%5B%5D=%7B__name__%3D%22imageregistry%3Aimagestreamtags_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22imageregistry%3Aoperations_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22log_logging_info%22%7D&match%5B%5D=%7B__name__%3D%22log_collector_error_count_total%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_pipeline_info%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_input_info%22%7D&match%5B%5D=%7B__name__%3D%22log_forwarder_output_info%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Alog_collected_bytes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Alog_logged_bytes_total%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Akata_monitor_running_shim_count%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22platform%3Ahypershift_hostedclusters%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22platform%3Ahypershift_nodepools%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_bucket_claims%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_buckets_claims%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_namespace_resources%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_namespace_resources%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_unhealthy_namespace_buckets%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_namespace_buckets%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_accounts%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_usage%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22namespace%3Anoobaa_system_health_status%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22ocs_advanced_feature_usage%22%7D&match%5B%5D=%7B__name__%3D%22os_image_url_override%3Asum%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_topology_tags%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_infrastructure_failure_domains%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22cluster%3Avsphere_csi_migration%3Amax%22%7D&match%5B%5D=%7B__name__%3D%22apiserver_list_watch_request_success_total%3Arate%3Asum%22%2C+verb%3D~%22LIST%7CWATCH%22%7D'
# TYPE ALERTS untyped
ALERTS{alertname="AlertmanagerReceiversNotConfigured",alertstate="firing",namespace="openshift-monitoring",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090351788
ALERTS{alertname="CannotRetrieveUpdates",alertstate="firing",namespace="openshift-cluster-version",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090344970
ALERTS{alertname="ClusterNotUpgradeable",alertstate="firing",condition="Upgradeable",endpoint="metrics",name="version",namespace="openshift-cluster-version",severity="info",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090353568
ALERTS{alertname="InsightsRecommendationActive",alertstate="firing",container="insights-operator",description="Prometheus metrics data will be lost when the Prometheus pod is restarted or recreated",endpoint="https",info_link="https://console.redhat.com/openshift/insights/advisor/clusters/94838b8e-1467-425d-97ed-04dfca7dd2d5?first=ccx_rules_ocp.external.rules.empty_prometheus_db_volume|PROMETHEUS_DB_VOLUME_IS_EMPTY",instance="10.129.0.94:8443",job="metrics",namespace="openshift-insights",pod="insights-operator-7f74c88c85-7zjjr",service="metrics",severity="info",total_risk="Low",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090369705
ALERTS{alertname="PodSecurityViolation",alertstate="firing",namespace="openshift-kube-apiserver",ocp_namespace="kube-system",policy_level="restricted",severity="info",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090351615
ALERTS{alertname="PodSecurityViolation",alertstate="firing",namespace="openshift-kube-apiserver",policy_level="restricted",severity="info",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090351615
ALERTS{alertname="TelemeterClientFailures",alertstate="firing",namespace="openshift-monitoring",severity="warning",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090361483
ALERTS{alertname="Watchdog",alertstate="firing",namespace="openshift-monitoring",severity="none",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090374321
# TYPE apiserver_list_watch_request_success_total:rate:sum untyped
apiserver_list_watch_request_success_total:rate:sum{verb="LIST",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 2.314814814814815 1692090366768
apiserver_list_watch_request_success_total:rate:sum{verb="WATCH",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 6.788888888888893 1692090366768
# TYPE cco_credentials_mode untyped
cco_credentials_mode{container="kube-rbac-proxy",endpoint="metrics",instance="10.130.0.195:8443",job="cco-metrics",mode="credsremoved",namespace="openshift-cloud-credential-operator",pod="cloud-credential-operator-5b958cdd86-75wf4",service="cco-metrics",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090346978
# TYPE cluster:alertmanager_integrations:max untyped
cluster:alertmanager_integrations:max{instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 0 1692090351788
# TYPE cluster:apiserver_current_inflight_requests:sum:max_over_time:2m untyped
cluster:apiserver_current_inflight_requests:sum:max_over_time:2m{apiserver="kube-apiserver",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 16 1692090370310
cluster:apiserver_current_inflight_requests:sum:max_over_time:2m{apiserver="openshift-apiserver",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 4 1692090370310
# TYPE cluster:capacity_cpu_cores:sum untyped
cluster:capacity_cpu_cores:sum{label_beta_kubernetes_io_instance_type="t2a-standard-4",label_kubernetes_io_arch="arm64",label_node_openshift_io_os_id="rhcos",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 12 1692090351788
cluster:capacity_cpu_cores:sum{label_beta_kubernetes_io_instance_type="t2a-standard-4",label_kubernetes_io_arch="arm64",label_node_openshift_io_os_id="rhcos",label_node_role_kubernetes_io="master",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 12 1692090351788
# TYPE cluster:capacity_memory_bytes:sum untyped
cluster:capacity_memory_bytes:sum{label_beta_kubernetes_io_instance_type="t2a-standard-4",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 5.020551168e+10 1692090351788
cluster:capacity_memory_bytes:sum{label_beta_kubernetes_io_instance_type="t2a-standard-4",label_node_role_kubernetes_io="master",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 5.0205507584e+10 1692090351788
# TYPE cluster:console_auth_login_failures_total:sum untyped
cluster:console_auth_login_failures_total:sum{reason="unknown",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 0 1692090346898
# TYPE cluster:console_auth_login_requests_total:sum untyped
cluster:console_auth_login_requests_total:sum{instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 6 1692090346898
# TYPE cluster:console_auth_login_successes_total:sum untyped
cluster:console_auth_login_successes_total:sum{role="cluster-admin",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 0 1692090346898
cluster:console_auth_login_successes_total:sum{role="developer",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 2 1692090346898
cluster:console_auth_login_successes_total:sum{role="kubeadmin",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 3 1692090346898
# TYPE cluster:console_auth_logout_requests_total:sum untyped
cluster:console_auth_logout_requests_total:sum{reason="unknown",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090346898
# TYPE cluster:console_plugins_info:max untyped
cluster:console_plugins_info:max{name="other",state="enabled",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090346898
cluster:console_plugins_info:max{name="other",state="notfound",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090346898
# TYPE cluster:console_usage_users:max untyped
cluster:console_usage_users:max{role="kubeadmin",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 1 1692090346898
# TYPE cluster:cpu_usage_cores:sum untyped
cluster:cpu_usage_cores:sum{instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 3.5917773122178644 1692090351788
# TYPE cluster:ingress_controller_aws_nlb_active:sum untyped
cluster:ingress_controller_aws_nlb_active:sum{instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 0 1692090363712
# TYPE cluster:kube_persistentvolume_plugin_type_counts:sum untyped
cluster:kube_persistentvolume_plugin_type_counts:sum{plugin_name="kubernetes.io/csi:pd.csi.storage.gke.io",volume_mode="Filesystem",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 2 1692090351788
# TYPE cluster:kube_persistentvolumeclaim_resource_requests_storage_bytes:provisioner:sum untyped
cluster:kube_persistentvolumeclaim_resource_requests_storage_bytes:provisioner:sum{provisioner="pd.csi.storage.gke.io",instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 4.315938816e+09 1692090351788
# TYPE cluster:memory_usage_bytes:sum untyped
cluster:memory_usage_bytes:sum{instance="",prometheus="openshift-monitoring/k8s",prometheus_replica="prometheus-k8s-0"} 3.4649325568e+10 1692090351788
...

ts=2023-08-14T23:00:22.734Z caller=db.go:1617 level=info component=tsdb msg="Deleting obsolete block" block=01H7TPPNZE5KJSHR6M2G0VYD3V
ts=2023-08-14T23:00:22.751Z caller=db.go:1617 level=info component=tsdb msg="Deleting obsolete block" block=01H7T8Z7S303V0TQ635P898R6G
ts=2023-08-14T23:00:22.768Z caller=db.go:1617 level=info component=tsdb msg="Deleting obsolete block" block=01H7TFTYQEZANJRV7DTVQY45RJ
ts=2023-08-15T01:00:07.194Z caller=compact.go:523 level=info component=tsdb msg="write block" mint=1692050400105 maxt=1692057600000 ulid=01H7VB9W0BJ38756TDSB54STJA duration=6.79804897s
ts=2023-08-15T01:00:07.818Z caller=head.go:1293 level=info component=tsdb msg="Head GC completed" caller=truncateMemory duration=621.187761ms 

$ oc -n openshift-monitoring get cm cluster-monitoring-config
Error from server (NotFound): configmaps "cluster-monitoring-config" not found

$ oc get clusterversion
NAME      VERSION                                    AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.14.0-0.nightly-arm64-2023-08-11-011949   True        False         32h     Cluster version is 4.14.0-0.nightly-arm64-2023-08-11-011949

not sure

1. see the description
2.
3.

unable to push metrics to telemeter server

able to push metrics to telemeter server