oc describe smmr -n istio-system
Name:         default
Namespace:    istio-system
Labels:       <none>
Annotations:  <none>
API Version:  maistra.io/v1
Kind:         ServiceMeshMemberRoll
Metadata:
  Creation Timestamp:  2021-08-12T00:24:44Z
  Finalizers:
    maistra.io/istio-operator
  Generation:  1
  Managed Fields:
    API Version:  maistra.io/v1
    Fields Type:  FieldsV1
    fieldsV1:
      f:spec:
        .:
        f:members:
    Manager:      Mozilla
    Operation:    Update
    Time:         2021-08-12T00:24:44Z
    API Version:  maistra.io/v1
    Fields Type:  FieldsV1
    fieldsV1:
      f:metadata:
        f:finalizers:
          .:
          v:"maistra.io/istio-operator":
      f:status:
        .:
        f:annotations:
          .:
          f:configuredMemberCount:
        f:conditions:
        f:configuredMembers:
        f:memberStatuses:
        f:members:
        f:observedGeneration:
        f:pendingMembers:
        f:terminatingMembers:
    Manager:         istio-operator
    Operation:       Update
    Time:            2021-08-12T00:24:44Z
  Resource Version:  40074
  Self Link:         /apis/maistra.io/v1/namespaces/istio-system/servicemeshmemberrolls/default
  UID:               895c4bf4-85d1-4dae-8906-f1c3550306d9
Spec:
  Members:
    bookinfo
Status:
  Annotations:
    Configured Member Count:  0/0
  Conditions:
    Last Transition Time:  2021-08-12T00:24:44Z
    Message:               Initial service mesh installation has not completed
    Reason:                SMCPReconciling
    Status:                False
    Type:                  Ready
  Configured Members:      <nil>
  Member Statuses:         <nil>
  Members:                 <nil>
  Observed Generation:     1
  Pending Members:         <nil>
  Terminating Members:     <nil>
Events:                    <none>

~/Downloads/ossminstall
❯ oc get pods -n istio-system
NAME                            READY   STATUS    RESTARTS   AGE
istiod-basic-558ddd7d75-n7thl   1/1     Running   0          11m
prometheus-67f67d49fc-9x6xn     2/3     Running   0          11m

~/Downloads/ossminstall
❯ og describe pods -n istio-system prometheus-67f67d49fc-9x6xn
zsh: command not found: og

~/Downloads/ossminstall
❯ oc describe pods -n istio-system prometheus-67f67d49fc-9x6xn
Name:         prometheus-67f67d49fc-9x6xn
Namespace:    istio-system
Priority:     0
Node:         ci-ln-b6vfclb-f76d1-cfc4w-worker-a-6c8m2/10.0.32.2
Start Time:   Wed, 11 Aug 2021 20:42:21 -0400
Labels:       app=prometheus
              maistra-control-plane=istio-system
              pod-template-hash=67f67d49fc
              release=istio
Annotations:  k8s.v1.cni.cncf.io/network-status:
                [{
                    "name": "",
                    "interface": "eth0",
                    "ips": [
                        "10.128.2.49"
                    ],
                    "default": true,
                    "dns": {}
                }]
              k8s.v1.cni.cncf.io/networks-status:
                [{
                    "name": "",
                    "interface": "eth0",
                    "ips": [
                        "10.128.2.49"
                    ],
                    "default": true,
                    "dns": {}
                }]
              openshift.io/scc: restricted
              sidecar.istio.io/inject: false
Status:       Running
IP:           10.128.2.49
IPs:
  IP:           10.128.2.49
Controlled By:  ReplicaSet/prometheus-67f67d49fc
Containers:
  prometheus-proxy:
    Container ID:  cri-o://2b7c3f7d939c356226ac981228a39a6eaee143c4504e2600936c2cc540482b33
    Image:         quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e147c7bd3538be20d703a092257ac80c712bea52b0ddf004bbf1587aac2e5ecc
    Image ID:      quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e147c7bd3538be20d703a092257ac80c712bea52b0ddf004bbf1587aac2e5ecc
    Port:          3001/TCP
    Host Port:     0/TCP
    Args:
      -provider=openshift
      -https-address=:3001
      -http-address=
      -email-domain=*
      -upstream=http://localhost:9090
      -htpasswd-file=/etc/proxy/htpasswd/auth
      -display-htpasswd-form=false
      -openshift-sar={"namespace": "istio-system", "resource": "pods", "verb": "get"}
      -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
      -openshift-service-account=prometheus
      -cookie-secret-file=/etc/proxy/secrets/session_secret
      -tls-cert=/etc/tls/private/tls.crt
      -tls-key=/etc/tls/private/tls.key
      -openshift-ca=/etc/pki/tls/cert.pem
      -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    State:          Running
      Started:      Wed, 11 Aug 2021 20:42:24 -0400
    Ready:          True
    Restart Count:  0
    Requests:
      cpu:        10m
      memory:     128Mi
    Readiness:    http-get https://:https/oauth/healthz delay=0s timeout=1s period=10s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/pki/ca-trust/extracted/pem/ from trusted-ca-bundle (ro)
      /etc/proxy/htpasswd from secret-htpasswd (rw)
      /etc/proxy/secrets from secret-prometheus-proxy (rw)
      /etc/tls/private from secret-prometheus-tls (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from prometheus-token-gj9cb (ro)
  prometheus:
    Container ID:  cri-o://bfa2122b3ceddfaff6236a8bbd0da27930da3029f8a509c457973a2928a0f0a9
    Image:         quay.io/maistra/prometheus-rhel8@sha256:a4ffcd385dfd1177a2f8783af3fbd38e6cdc051763be156c7f639424102da28b
    Image ID:      quay.io/maistra/prometheus-rhel8@sha256:a4ffcd385dfd1177a2f8783af3fbd38e6cdc051763be156c7f639424102da28b
    Port:          9090/TCP
    Host Port:     0/TCP
    Args:
      --storage.tsdb.retention=6h
      --storage.tsdb.path=/prometheus
      --config.file=/etc/prometheus/prometheus.yml
      --discovery.member-roll-name=default
      --discovery.member-roll-namespace=istio-system
    State:          Running
      Started:      Wed, 11 Aug 2021 20:42:24 -0400
    Ready:          True
    Restart Count:  0
    Requests:
      cpu:        10m
      memory:     128Mi
    Liveness:     http-get http://:9090/-/healthy delay=0s timeout=1s period=10s #success=1 #failure=3
    Readiness:    http-get http://:9090/-/ready delay=0s timeout=1s period=10s #success=1 #failure=3
    Environment:  <none>
    Mounts:
      /etc/istio-certs from istio-certs (rw)
      /etc/prometheus from config-volume (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from prometheus-token-gj9cb (ro)
  istio-proxy:
    Container ID:  cri-o://21160c060a418ac31dca44152555cd9a4eed7f758c07b7f32acd79d81de8b415
    Image:         quay.io/maistra/proxyv2-rhel8@sha256:b60dffd792b7857633db90489da793232bb5213339bbf5ee77dd92e2c0a5cf24
    Image ID:      quay.io/maistra/proxyv2-rhel8@sha256:20a3919ef7b9ffffe4c8a1f1e1af8c1f9f1fa73eca00b641d9041357dbc1b7fd
    Port:          15090/TCP
    Host Port:     0/TCP
    Args:
      proxy
      sidecar
      --domain
      $(POD_NAMESPACE).svc.cluster.local
      istio-proxy-prometheus
      --proxyLogLevel=warning
      --proxyComponentLogLevel=misc:error
      --controlPlaneAuthPolicy
      NONE
      --trust-domain=cluster.local
    State:          Running
      Started:      Wed, 11 Aug 2021 20:42:25 -0400
    Ready:          False
    Restart Count:  0
    Readiness:      http-get http://:15020/healthz/ready delay=1s timeout=1s period=2s #success=1 #failure=30
    Environment:
      OUTPUT_CERTS:           /etc/istio-certs
      JWT_POLICY:             first-party-jwt
      PILOT_CERT_PROVIDER:    istiod
      CA_ADDR:                istiod-basic.istio-system.svc:15012
      POD_NAME:               prometheus-67f67d49fc-9x6xn (v1:metadata.name)
      POD_NAMESPACE:          istio-system (v1:metadata.namespace)
      INSTANCE_IP:             (v1:status.podIP)
      SERVICE_ACCOUNT:         (v1:spec.serviceAccountName)
      HOST_IP:                 (v1:status.hostIP)
      ISTIO_META_MESH_ID:     cluster.local
      ISTIO_META_CLUSTER_ID:  Kubernetes
    Mounts:
      /etc/istio-certs/ from istio-certs (rw)
      /etc/istio/config from istio-config-volume (rw)
      /etc/istio/proxy from istio-envoy (rw)
      /var/run/secrets/istio from istiod-ca-cert (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from prometheus-token-gj9cb (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  secret-prometheus-tls:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  prometheus-tls
    Optional:    false
  secret-htpasswd:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  htpasswd
    Optional:    false
  secret-prometheus-proxy:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  prometheus-proxy
    Optional:    false
  trusted-ca-bundle:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      trusted-ca-bundle
    Optional:  true
  istio-config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      istio-basic
    Optional:  true
  config-volume:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      prometheus
    Optional:  false
  istio-certs:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  istio-envoy:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:     Memory
    SizeLimit:  <unset>
  istiod-ca-cert:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      istio-ca-root-cert
    Optional:  false
  prometheus-token-gj9cb:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  prometheus-token-gj9cb
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  <none>
Tolerations:     node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                 node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                 node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason          Age                  From               Message
  ----     ------          ----                 ----               -------
  Normal   Scheduled       11m                  default-scheduler  Successfully assigned istio-system/prometheus-67f67d49fc-9x6xn to ci-ln-b6vfclb-f76d1-cfc4w-worker-a-6c8m2
  Normal   AddedInterface  11m                  multus             Add eth0 [10.128.2.49/23]
  Normal   Created         11m                  kubelet            Created container prometheus
  Normal   Created         11m                  kubelet            Created container prometheus-proxy
  Normal   Started         11m                  kubelet            Started container prometheus-proxy
  Normal   Pulled          11m                  kubelet            Container image "quay.io/maistra/prometheus-rhel8@sha256:a4ffcd385dfd1177a2f8783af3fbd38e6cdc051763be156c7f639424102da28b" already present on machine
  Normal   Pulled          11m                  kubelet            Container image "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e147c7bd3538be20d703a092257ac80c712bea52b0ddf004bbf1587aac2e5ecc" already present on machine
  Normal   Started         11m                  kubelet            Started container prometheus
  Normal   Pulling         11m                  kubelet            Pulling image "quay.io/maistra/proxyv2-rhel8@sha256:b60dffd792b7857633db90489da793232bb5213339bbf5ee77dd92e2c0a5cf24"
  Normal   Pulled          11m                  kubelet            Successfully pulled image "quay.io/maistra/proxyv2-rhel8@sha256:b60dffd792b7857633db90489da793232bb5213339bbf5ee77dd92e2c0a5cf24" in 620.733523ms
  Normal   Created         11m                  kubelet            Created container istio-proxy
  Normal   Started         11m                  kubelet            Started container istio-proxy
  Warning  Unhealthy       89s (x300 over 11m)  kubelet            Readiness probe failed: HTTP probe failed with statuscode: 503

level=error ts=2021-08-12T00:42:29.657Z caller=manager.go:123 component="scrape manager" msg="error creating new scrape pool" err="error creating HTTP client: unable to load specified CA cert /etc/istio-certs/root-cert.pem: open /etc/istio-certs/root-cert.pem: no such file or directory" scrape_pool=kubernetes-pods-istio-secure
level=error ts=2021-08-12T00:42:34.657Z caller=manager.go:123 component="scrape manager" msg="error creating new scrape pool" err="error creating HTTP client: unable to load specified CA cert /etc/istio-certs/root-cert.pem: open /etc/istio-certs/root-cert.pem: no such file or directory" scrape_pool=kubernetes-pods-istio-secure
level=warn ts=2021-08-12T00:48:28.666Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296: watch of *v1.Endpoints ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:48:48.677Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:298: watch of *v1.Pod ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:48:51.678Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:298: watch of *v1.Pod ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:48:59.667Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:297: watch of *v1.Service ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:49:18.667Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:297: watch of *v1.Service ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:49:44.675Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:320: watch of *v1.Pod ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:51:28.663Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296: watch of *v1.Endpoints ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:55:17.671Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:296: watch of *v1.Endpoints ended with: Underlying Result Channel closed"
level=warn ts=2021-08-12T00:55:29.672Z caller=klog.go:86 component=k8s_client_runtime func=Warningf msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:297: watch of *v1.Service ended with: Underlying Result Channel closed"