Bug Link: https://issues.redhat.com/browse/OSSM-2419

Will automate this bug into "Maistra Test Tool"

Bug description:

Validation Steps:

1. Deployed the bookinfo  to the "istio-system" namespace

2. Deployed the 50  empty namespaces to the "Istio-system" namespace

3. created the 2.2 CP

4. verified the "istio-system" pods 

5. restarted the "Istiod" pod multiple time, then you can see the issue.

error: 

oc get pods -n istio-system | grep istiod
istiod-basic-5749f77777-jxzlz          0/1     Running   0          20

6. described the  Istiod pod: 

• we can check the error in the Events, it is Probe failed

  oc describe pod istiod-basic-5749f77777-jxzlz -n istio-system 

  Name:         istiod-basic-5749f77777-jxzlz
  Namespace:    istio-system
  Priority:     0
  Node:         pbajjurinew-v7glr-worker-0-b82v7/192.168.0.183
  Start Time:   Wed, 11 Jan 2023 10:16:02 -0600
  Labels:       app=istiod
                istio=istiod
                istio.io/rev=basic
                maistra-control-plane=istio-system
                pod-template-hash=5749f77777
                sidecar.istio.io/inject=false
  Annotations:  k8s.v1.cni.cncf.io/network-status:
                  [{
                      "name": "openshift-sdn",
                      "interface": "eth0",
                      "ips": [
                          "10.129.2.166"
                      ],
                      "default": true,
                      "dns": {}
                  }]
                k8s.v1.cni.cncf.io/networks-status:
                  [{
                      "name": "openshift-sdn",
                      "interface": "eth0",
                      "ips": [
                          "10.129.2.166"
                      ],
                      "default": true,
                      "dns": {}
                  }]
                openshift.io/scc: restricted-v2
                prometheus.io/port: 15014
                prometheus.io/scrape: true
                seccomp.security.alpha.kubernetes.io/pod: runtime/default
                sidecar.istio.io/inject: false
  Status:       Running
  IP:           10.129.2.166
  IPs:
    IP:           10.129.2.166
  Controlled By:  ReplicaSet/istiod-basic-5749f77777
  Containers:
    discovery:
      Container ID:  cri-o://0a7b5a736fa2ede6c72372f7ec7d7476c45d82562eb1f9ceef55c8b18044c020
      Image:         registry.redhat.io/openshift-service-mesh/pilot-rhel8@sha256:cc9c994d867d4ab0125d3e8d61073c48562c7b3ac69429f6b69f9520b966279e
      Image ID:      registry.redhat.io/openshift-service-mesh/pilot-rhel8@sha256:826fbf09668571013351b4537dd1ee3a46da007283d99f2edb8954c27fef084b
      Ports:         8080/TCP, 15010/TCP, 15017/TCP, 8188/TCP
      Host Ports:    0/TCP, 0/TCP, 0/TCP, 0/TCP
      Args:
        discovery
        --memberRollName=default
        --cacheCluster=outbound|80||wasm-cacher-basic.istio-system.svc.cluster.local
        --enableCRDScan=false
        --enableIngressClassName=false
        --disableNodeAccess=true
        --monitoringAddr=:15014
        --log_output_level=default:warn
        --domain
        cluster.local
        --keepaliveMaxServerConnectionAge
        30m
      State:          Running
        Started:      Wed, 11 Jan 2023 10:16:05 -0600
      Ready:          False
      Restart Count:  0
      Requests:
        cpu:      10m
        memory:   128Mi
      Readiness:  http-get http://:8080/ready delay=1s timeout=5s period=3s #success=1 #failure=3
      Environment:
        ENABLE_MAISTRA_EXTENSIONS:                       true
        ENABLE_IOR:                                      true
        PRIORITIZED_LEADER_ELECTION:                     false
        VALIDATION_WEBHOOK_CONFIG_NAME:                  
        INJECTION_WEBHOOK_CONFIG_NAME:                   
        REVISION:                                        basic
        JWT_POLICY:                                      first-party-jwt
        PILOT_CERT_PROVIDER:                             istiod
        POD_NAME:                                        istiod-basic-5749f77777-jxzlz (v1:metadata.name)
        POD_NAMESPACE:                                   istio-system (v1:metadata.namespace)
        SERVICE_ACCOUNT:                                  (v1:spec.serviceAccountName)
        KUBECONFIG:                                      /var/run/secrets/remote/config
        PILOT_ENABLE_GATEWAY_API:                        false
        PILOT_ENABLE_GATEWAY_API_DEPLOYMENT_CONTROLLER:  false
        PILOT_ENABLE_GATEWAY_API_STATUS:                 false
        PILOT_ENABLE_FEDERATION:                         true
        PILOT_TRACE_SAMPLING:                            100
        PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND:     false
        PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND:      false
        ISTIOD_ADDR:                                     istiod-basic.istio-system.svc:15012
        PILOT_ENABLE_STATUS:                             false
        PILOT_ENABLE_ANALYSIS:                           false
        CLUSTER_ID:                                      Kubernetes
      Mounts:
        /etc/cacerts from cacerts (ro)
        /var/run/secrets/istio-dns from local-certs (rw)
        /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-5d9tw (ro)
        /var/run/secrets/remote from istio-kubeconfig (ro)
  Conditions:
    Type              Status
    Initialized       True 
    Ready             False 
    ContainersReady   False 
    PodScheduled      True 
  Volumes:
    local-certs:
      Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
      Medium:     Memory
      SizeLimit:  <unset>
    cacerts:
      Type:        Secret (a volume populated by a Secret)
      SecretName:  cacerts
      Optional:    true
    istio-kubeconfig:
      Type:        Secret (a volume populated by a Secret)
      SecretName:  istio-kubeconfig
      Optional:    true
    kube-api-access-5d9tw:
      Type:                    Projected (a volume that contains injected data from multiple sources)
      TokenExpirationSeconds:  3607
      ConfigMapName:           kube-root-ca.crt
      ConfigMapOptional:       <nil>
      DownwardAPI:             true
      ConfigMapName:           openshift-service-ca.crt
      ConfigMapOptional:       <nil>
  QoS Class:                   Burstable
  Node-Selectors:              <none>
  Tolerations:                 node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                               node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                               node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
  Events:
    Type     Reason          Age                  From               Message
    ----     ------          ----                 ----               -------
    Normal   Scheduled       21m                  default-scheduler  Successfully assigned istio-system/istiod-basic-5749f77777-jxzlz to pbajjurinew-v7glr-worker-0-b82v7
    Normal   AddedInterface  21m                  multus             Add eth0 [10.129.2.166/23] from openshift-sdn
    Normal   Pulled          21m                  kubelet            Container image "registry.redhat.io/openshift-service-mesh/pilot-rhel8@sha256:cc9c994d867d4ab0125d3e8d61073c48562c7b3ac69429f6b69f9520b966279e" already present on machine
    Normal   Created         21m                  kubelet            Created container discovery
    Normal   Started         21m                  kubelet            Started container discovery
    Warning  Unhealthy       67s (x417 over 21m)  kubelet            Readiness probe failed: Get "http://10.129.2.166:8080/ready": dial tcp 10.129.2.166:8080: connect: connection refused