Name:             metrics-collector-deployment-566bd6cfb5-7smkf
Namespace:        open-cluster-management-addon-observability
Priority:         0
Service Account:  endpoint-observability-operator-sa
Node:             ip-10-0-26-116.ec2.internal/10.0.26.116
Start Time:       Tue, 19 Mar 2024 08:14:59 +0100
Labels:           component=metrics-collector
                  pod-template-hash=1226827961
Annotations:      openshift.io/scc: restricted
                  owner: observabilityaddon
                  target.workload.openshift.io/management: {"effect":"PreferredDuringScheduling"}
Status:           Pending
IP:               
IPs:              <none>
Controlled By:    ReplicaSet/metrics-collector-deployment-566bd6cfb5
Containers:
  metrics-collector:
    Container ID:  
    Image:         quay.io:443/acm-d/metrics-collector-rhel9@sha256:39ff2cb09ea7de4f4cbb374552b51b78253fdc06d73fbdacdbd4d69ad4666f07
    Image ID:      
    Port:          8080/TCP
    Host Port:     0/TCP
    Command:
      /usr/bin/metrics-collector
      --listen=:8080
      --from=$(FROM)
      --from-query=$(FROM_QUERY)
      --to-upload=$(TO)
      --to-upload-ca=/tlscerts/ca/ca.crt
      --to-upload-cert=/tlscerts/certs/tls.crt
      --to-upload-key=/tlscerts/certs/tls.key
      --interval=30s
      --evaluate-interval=30s
      --limit-bytes=1073741824
      --label="cluster=clc311-239"
      --label="clusterID=clc311-239"
      --from-token-file=/var/run/secrets/kubernetes.io/serviceaccount/token
      --from-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
      --label="clusterType=ocp3"
      --match={__name__=":node_memory_MemAvailable_bytes:sum"}
      --match={__name__="container_cpu_cfs_periods_total"}
      --match={__name__="container_cpu_cfs_throttled_periods_total"}
      --match={__name__="etcd_debugging_mvcc_db_total_size_in_bytes"}
      --match={__name__="etcd_disk_backend_commit_duration_seconds_bucket"}
      --match={__name__="etcd_disk_wal_fsync_duration_seconds_bucket"}
      --match={__name__="etcd_network_client_grpc_received_bytes_total"}
      --match={__name__="etcd_network_client_grpc_sent_bytes_total"}
      --match={__name__="etcd_network_peer_received_bytes_total"}
      --match={__name__="etcd_network_peer_sent_bytes_total"}
      --match={__name__="etcd_server_has_leader"}
      --match={__name__="etcd_server_leader_changes_seen_total"}
      --match={__name__="etcd_server_proposals_failed_total"}
      --match={__name__="etcd_server_proposals_pending"}
      --match={__name__="etcd_server_proposals_committed_total"}
      --match={__name__="etcd_server_proposals_applied_total"}
      --match={__name__="grpc_server_started_total"}
      --match={__name__="instance:node_cpu_utilisation:rate1m"}
      --match={__name__="instance:node_load1_per_cpu:ratio"}
      --match={__name__="instance:node_memory_utilisation:ratio"}
      --match={__name__="instance:node_network_receive_bytes_excluding_lo:rate1m"}
      --match={__name__="instance:node_network_receive_drop_excluding_lo:rate1m"}
      --match={__name__="instance:node_network_transmit_bytes_excluding_lo:rate1m"}
      --match={__name__="instance:node_network_transmit_drop_excluding_lo:rate1m"}
      --match={__name__="instance:node_num_cpu:sum"}
      --match={__name__="instance:node_vmstat_pgmajfault:rate1m"}
      --match={__name__="instance_device:node_disk_io_time_seconds:rate1m"}
      --match={__name__="instance_device:node_disk_io_time_weighted_seconds:rate1m"}
      --match={__name__="kube_node_status_allocatable"}
      --match={__name__="kube_node_status_allocatable_cpu_cores"}
      --match={__name__="kube_node_status_allocatable_memory_bytes"}
      --match={__name__="kube_node_status_capacity_cpu_cores"}
      --match={__name__="kube_node_status_condition"}
      --match={__name__="kube_pod_container_resource_limits"}
      --match={__name__="kube_pod_container_resource_limits_cpu_cores"}
      --match={__name__="kube_pod_container_resource_limits_memory_bytes"}
      --match={__name__="kube_pod_container_resource_requests"}
      --match={__name__="kube_pod_container_resource_requests_cpu_cores"}
      --match={__name__="kube_pod_container_resource_requests_memory_bytes"}
      --match={__name__="kube_pod_info"}
      --match={__name__="kube_pod_owner"}
      --match={__name__="kube_resourcequota"}
      --match={__name__="machine_cpu_cores"}
      --match={__name__="machine_memory_bytes"}
      --match={__name__="mixin_pod_workload"}
      --match={__name__="namespace_workload_pod:kube_pod_owner:relabel"}
      --match={__name__="node_cpu_seconds_total"}
      --match={__name__="node_filesystem_avail_bytes"}
      --match={__name__="node_filesystem_size_bytes"}
      --match={__name__="node_memory_MemAvailable_bytes"}
      --match={__name__="node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate"}
      --match={__name__="node_netstat_Tcp_OutSegs"}
      --match={__name__="node_netstat_Tcp_RetransSegs"}
      --match={__name__="node_netstat_TcpExt_TCPSynRetrans"}
      --match={__name__="up"}
      --match={__name__="node:node_cpu_utilisation:avg1m"}
      --match={__name__="kube_node_labels"}
      --match={__name__="node_namespace_pod:kube_pod_info:"}
      --match={__name__="container_memory_usage_bytes"}
      --match={__name__="node_memory_MemTotal_bytes"}
      --match={__name__="node:node_memory_bytes_total:sum"}
      --match={__name__="node:node_net_utilisation:sum_irate"}
      --match={__name__="node_network_receive_bytes"}
      --match={__name__="node_network_transmit_bytes"}
      --match={__name__="node_disk_bytes_read"}
      --match={__name__="node_disk_bytes_written"}
      --match={__name__="node:node_disk_utilisation:avg_irate"}
      --match={__name__="kube_pod_status_ready"}
      --match={__name__="kube_pod_status_phase"}
      --match={__name__="node_filesystem_size"}
      --match={__name__="node_filesystem_avail"}
      --match={__name__="kube_pod_container_status_restarts_total"}
      --match={__name__="openshift_clusterresourcequota_usage"}
      --match={__name__="openshift_clusterresourcequota_labels"}
      --match={__name__="namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate"}
      --match={__name__="kube_namespace_labels"}
      --match={__name__="container_memory_rss"}
      --match={__name__="container_network_receive_bytes_total"}
      --match={__name__="container_network_transmit_bytes_total"}
      --match={__name__="container_network_receive_packets_total"}
      --match={__name__="container_network_transmit_packets_total"}
      --match={__name__="container_network_receive_packets_dropped_total"}
      --match={__name__="container_network_transmit_packets_dropped_total"}
      --match={__name__="container_cpu_usage_seconds_total"}
      --match={__name__="workqueue_queue_duration_seconds_bucket",job="apiserver"}
      --match={__name__="workqueue_adds_total",job="apiserver"}
      --match={__name__="workqueue_depth",job="apiserver"}
      --match={__name__="go_goroutines",job="apiserver"}
      --match={__name__="process_cpu_seconds_total",job="apiserver"}
      --match={__name__="process_resident_memory_bytes",job="apiserver"}
      --match={__name__="container_memory_cache",container!=""}
      --match={__name__="container_memory_rss",container!=""}
      --match={__name__="container_memory_swap",container!=""}
      --match={__name__="container_memory_working_set_bytes",container_name!=""}
      --rename="etcd_mvcc_db_total_size_in_bytes=etcd_debugging_mvcc_db_total_size_in_bytes"
      --rename="mixin_pod_workload=namespace_workload_pod:kube_pod_owner:relabel"
      --rename="namespace:kube_pod_container_resource_requests_cpu_cores:sum=namespace_cpu:kube_pod_container_resource_requests:sum"
      --rename="node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate=node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate"
      --recordingrule={"name":"apiserver_request_duration_seconds:histogram_quantile_99","query":"(histogram_quantile(0.99,sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\", verb!=\"WATCH\"}[5m])) by (le)))/1000000"}
      --recordingrule={"name":"apiserver_request_duration_seconds:histogram_quantile_99:instance","query":"(histogram_quantile(0.99, sum(rate(apiserver_request_latencies_bucket{job=\"apiserver\", verb!=\"WATCH\"}[5m])) by (le, verb, instance)))/1000000"}
      --recordingrule={"name":"sum:apiserver_request_total:1h","query":"sum(rate(apiserver_request_count{job=\"apiserver\"}[1h])) by(code, instance)"}
      --recordingrule={"name":"sum:apiserver_request_total:5m","query":"sum(rate(apiserver_request_count{job=\"apiserver\"}[5m])) by(code, instance)"}
      --recordingrule={"name":"rpc_rate:grpc_server_handled_total:sum_rate","query":"sum(rate(grpc_server_handled_total{job=\"etcd\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))"}
      --recordingrule={"name":"active_streams_watch:grpc_server_handled_total:sum","query":"sum(grpc_server_started_total{job=\"etcd\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"etcd\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})"}
      --recordingrule={"name":"active_streams_lease:grpc_server_handled_total:sum","query":"sum(grpc_server_started_total{job=\"etcd\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"etcd\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})"}
      --recordingrule={"name":"cluster:kube_pod_container_resource_requests:cpu:sum","query":"sum(sum(sum(kube_pod_container_resource_requests_cpu_cores) by (pod,namespace,container) * on(pod,namespace) group_left(phase) max(kube_pod_status_phase{phase=~\"Running|Pending|Unknown\"} >0) by (pod,namespace,phase)) by (pod,namespace,phase))"}
      --recordingrule={"name":"cluster:kube_pod_container_resource_requests:memory:sum","query":"sum(sum(sum(kube_pod_container_resource_requests_memory_bytes) by (pod,namespace,container) * on(pod,namespace) group_left(phase) max(kube_pod_status_phase{phase=~\"Running|Pending|Unknown\"} >0) by (pod,namespace,phase)) by (pod,namespace,phase))"}
      --recordingrule={"name":"sli:apiserver_request_duration_seconds:trend:1m","query":"sum(increase(apiserver_request_latencies_bucket{job=\"apiserver\",service=\"kubernetes\",le=\"1\",verb=~\"POST|PUT|DELETE|PATCH\"}[1m])) / sum(increase(apiserver_request_latencies_count{job=\"apiserver\",service=\"kubernetes\",verb=~\"POST|PUT|DELETE|PATCH\"}[1m]))"}
      --recordingrule={"name":":node_memory_MemAvailable_bytes:sum","query":"sum(node_memory_MemAvailable_bytes{job=\"node-exporter\"}or(node_memory_Buffers_bytes{job=\"node-exporter\"} + node_memory_Cached_bytes{job=\"node-exporter\"} + node_memory_MemFree_bytes{job=\"node-exporter\"} + node_memory_Slab_bytes{job=\"node-exporter\"}))"}
      --recordingrule={"name":"instance:node_network_receive_bytes_excluding_lo:rate1m","query":"sum(rate(node_network_receive_bytes_total{job=\"node-exporter\", device!=\"lo\"}[1m])) without(device)"}
      --recordingrule={"name":"instance:node_network_transmit_bytes_excluding_lo:rate1m","query":"sum(rate(node_network_transmit_bytes_total{job=\"node-exporter\", device!=\"lo\"}[1m])) without(device)"}
      --recordingrule={"name":"instance:node_network_receive_drop_excluding_lo:rate1m","query":"sum(rate(node_network_receive_drop_total{job=\"node-exporter\", device!=\"lo\"}[1m])) without(device)"}
      --recordingrule={"name":"instance:node_network_transmit_drop_excluding_lo:rate1m","query":"sum(rate(node_network_transmit_drop_total{job=\"node-exporter\", device!=\"lo\"}[1m])) without(device)"}
    State:          Waiting
      Reason:       ContainerCreating
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     200m
      memory:  700Mi
    Requests:
      cpu:     10m
      memory:  100Mi
    Environment:
      FROM:        https://prometheus-k8s.openshift-monitoring.svc:9091
      FROM_QUERY:  https://prometheus-k8s.openshift-monitoring.svc:9091
      TO:          https://observatorium-api-open-cluster-management-observability.apps.ci-vb-acm210fc8-fi.gcp.dev09.red-chesterfield.com/api/metrics/v1/default/api/v1/receive
    Mounts:
      /tlscerts/ca from mtlsca (rw)
      /tlscerts/certs from mtlscerts (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from endpoint-observability-operator-sa-token-d7fj8 (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  mtlscerts:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  observability-controller-open-cluster-management.io-observability-signer-client-cert
    Optional:    false
  mtlsca:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  observability-managed-cluster-certs
    Optional:    false
  secret-kube-rbac-proxy-tls:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  metrics-collector-kube-rbac-tls
    Optional:    false
  secret-kube-rbac-proxy-metric:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  metrics-collector-kube-rbac-proxy-metric
    Optional:    false
  metrics-client-ca:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      metrics-collector-clientca-metric
    Optional:  false
  endpoint-observability-operator-sa-token-d7fj8:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  endpoint-observability-operator-sa-token-d7fj8
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  node-role.kubernetes.io/compute=true
Tolerations:     node.kubernetes.io/memory-pressure:NoSchedule op=Exists
Events:
  Type     Reason       Age                    From     Message
  ----     ------       ----                   ----     -------
  Warning  FailedMount  46m (x45 over 122m)    kubelet  MountVolume.SetUp failed for volume "secret-kube-rbac-proxy-tls" : secrets "metrics-collector-kube-rbac-tls" not found
  Warning  FailedMount  6m51s (x51 over 120m)  kubelet  Unable to mount volumes for pod "metrics-collector-deployment-566bd6cfb5-7smkf_open-cluster-management-addon-observability(65bb6ab2-e5c0-11ee-a5fc-0e13e690c4a7)": timeout expired waiting for volumes to attach or mount for pod "open-cluster-management-addon-observability"/"metrics-collector-deployment-566bd6cfb5-7smkf". list of unmounted volumes=[secret-kube-rbac-proxy-tls secret-kube-rbac-proxy-metric metrics-client-ca]. list of unattached volumes=[mtlscerts mtlsca secret-kube-rbac-proxy-tls secret-kube-rbac-proxy-metric metrics-client-ca endpoint-observability-operator-sa-token-d7fj8]
  Warning  FailedMount  2m1s (x67 over 122m)   kubelet  MountVolume.SetUp failed for volume "metrics-client-ca" : configmaps "metrics-collector-clientca-metric" not found