Uploaded image for project: 'OpenShift Bugs'
  1. OpenShift Bugs
  2. OCPBUGS-60848

pod with topology aware scheduler stuck in pending state due to invalid topology data

XMLWordPrintable

    • Quality / Stability / Reliability
    • False
    • Hide

      None

      Show
      None
    • None
    • Important
    • None
    • x86_64
    • None
    • Approved
    • CNF Compute Sprint 277
    • 1
    • None
    • None
    • None
    • None
    • None
    • None
    • None

      Description of problem:

      The bug here is after this bug was verified: https://issues.redhat.com/browse/OCPBUGS-56785
      
      The problem here is that after creating a scenario for which the secondary scheduler will become stalled creating a test pod to see if the pod will get scheduler is causing that pod to be stuck forever and is failing on invalid topology data.
      
      The environment that was used here is a compact cluster.

      Version-Release number of selected component (if applicable):

          

      How reproducible:

      Always

      Steps to Reproduce:

      1. Configure the kubelet to have a memory manager policy = static
      2. Run a job to qualify as for memory pinning
      3. Wait for the NRT data to settle
      4. Create a test pod to see that it is running

      Actual results:

      [root@registry bug-pod-resource-api]# oc get pods
      NAME                      READY   STATUS      RESTARTS   AGE
      generic-pause-pod-6ks7v   0/1     Completed   0          20m
      generic-pause-pod-x9kkf   0/1     Completed   0          20m
      test-pod                  0/1     Pending     0          17m
      

      Expected results:

       

      Additional info:

      [root@registry bug-pod-resource-api]# oc get clusterversion
      NAME      VERSION                              AVAILABLE   PROGRESSING   SINCE   STATUS
      version   4.20.0-0.nightly-2025-08-19-081012   True        False         5d22h   Cluster version is 4.20.0-0.nightly-2025-08-19-081012
      
      [root@registry bug-pod-resource-api]# oc get no
      NAME                                                STATUS   ROLES                         AGE     VERSION
      master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com   Ready    control-plane,master,worker   5d22h   v1.33.3
      master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com   Ready    control-plane,master,worker   5d22h   v1.33.3
      master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com   Ready    control-plane,master,worker   5d22h   v1.33.3
      
      [root@registry bug-pod-resource-api]# oc get mcp
      NAME     CONFIG                                             UPDATED   UPDATING   DEGRADED   MACHINECOUNT   READYMACHINECOUNT   UPDATEDMACHINECOUNT   DEGRADEDMACHINECOUNT   AGE
      master   rendered-master-652f84cbfe0a26f3625b34047cee88b6   True      False      False      3              3                   3                     0                      5d22h
      worker   rendered-worker-bcc58fa70929755e05487ae05b9b93b6   True      False      False      0              0                   0                     0                      5d22h
      

      Performance profile that was applied:

      apiVersion: performance.openshift.io/v2
      kind: PerformanceProfile
      metadata:
        name: performance
        annotations:
          kubeletconfig.experimental: |
            {"topologyManagerScope": "container",
            "cpuManagerPolicyOptions": {"full-pcpus-only": "false"}
            }
      spec:
        cpu:
          isolated: "16-103"
          reserved: "0-15"
        machineConfigPoolSelector:
          pools.operator.machineconfiguration.openshift.io/master: ""
        nodeSelector:
          node-role.kubernetes.io/master: ""
        numa:
          topologyPolicy: single-numa-node
        realTimeKernel:
          enabled: true
        workloadHints:
          highPowerConsumption: true
          perPodPowerManagement: false
          realTime: true
      

      Generic pause pod manifest

      apiVersion: batch/v1
      kind: Job
      metadata:
        labels:
          app: idle-gu-job-sched-stall
        name: generic-pause-pod
      spec:
        backoffLimit: 6
        completionMode: NonIndexed
        completions: 2
        manualSelector: false
        parallelism: 2
        podReplacementPolicy: TerminatingOrFailed
        suspend: false
        template:
          metadata:
            labels:
              app: idle-gu-job-sched-stall
          spec:
            containers:
            - args:
              - 1s
              command:
              - /bin/sleep
              image: quay.io/openshift-kni/pause:test-ci
              imagePullPolicy: IfNotPresent
              name: generic-job-idle
              resources:
                limits:
                  cpu: 100m
                  memory: 256Mi
              terminationMessagePath: /dev/termination-log
              terminationMessagePolicy: File
            dnsPolicy: ClusterFirst
            restartPolicy: Never
            schedulerName: default-scheduler
            terminationGracePeriodSeconds: 30
            topologySpreadConstraints:
            - labelSelector:
                matchLabels:
                  app: idle-gu-job-sched-stall
              matchLabelKeys:
              - pod-template-hash
              maxSkew: 1
              topologyKey: kubernetes.io/hostname
              whenUnsatisfiable: DoNotSchedule
      

      Generic test pod

      apiVersion: v1
      kind: Pod
      metadata:
        namespace: default
        name: test-pod
        labels:
          test-type: "scheduler-test-pod"
      spec:
        schedulerName: topo-aware-scheduler
        containers:
        - name: pause
          image: registry.k8s.io/pause:3.9
          resources:
            limits:
              cpu: "2"
              memory: 64Mi
            requests:
              cpu: "2"
              memory: 64Mi
      

       

      Results for the running pods

      [root@registry bug-pod-resource-api]# oc get pods
      NAME                      READY   STATUS      RESTARTS   AGE
      generic-pause-pod-6ks7v   0/1     Completed   0          20m
      generic-pause-pod-x9kkf   0/1     Completed   0          20m
      test-pod                  0/1     Pending     0          17m
      
      [root@registry bug-pod-resource-api]# oc describe pod/test-pod
      Name:             test-pod
      Namespace:        default
      Priority:         0
      Service Account:  default
      Node:             <none>
      Labels:           test-type=scheduler-test-pod
      Annotations:      <none>
      Status:           Pending
      IP:               
      IPs:              <none>
      Containers:
        pause:
          Image:      registry.k8s.io/pause:3.9
          Port:       <none>
          Host Port:  <none>
          Limits:
            cpu:     2
            memory:  64Mi
          Requests:
            cpu:        2
            memory:     64Mi
          Environment:  <none>
          Mounts:
            /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-v2rp5 (ro)
      Conditions:
        Type           Status
        PodScheduled   False 
      Volumes:
        kube-api-access-v2rp5:
          Type:                    Projected (a volume that contains injected data from multiple sources)
          TokenExpirationSeconds:  3607
          ConfigMapName:           kube-root-ca.crt
          Optional:                false
          DownwardAPI:             true
          ConfigMapName:           openshift-service-ca.crt
          Optional:                false
      QoS Class:                   Guaranteed
      Node-Selectors:              <none>
      Tolerations:                 node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                                   node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                                   node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
      Events:
        Type     Reason            Age               From                  Message
        ----     ------            ----              ----                  -------
        Warning  FailedScheduling  7s (x4 over 20s)  topo-aware-scheduler  0/3 nodes are available: 3 invalid node topology data. preemption: 0/3 nodes are available: 3 No preemption victims found for incoming pod.
      
      [kni@registry.kni-qe-52 bug-pod-resource-api]$ for node in `oc get nodes -l node-role.kubernetes.io/worker= --no-headers | awk '{print $1}'`; do echo -e "Node: $node\n";oc debug node/$node -- chroot /host ./tmp/knit-v0.2.1-linux-amd64 podres   ;done
      Node: master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com
      
      Starting pod/master-0kni-qe-52telco5granengrdu2redhatcom-debug-nmfqc ...
      To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
      {"pod_resources":[{"name":"network-metrics-daemon-x52mj","namespace":"openshift-multus","containers":[{"name":"network-metrics-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"apiserver-575c85d5d4-l6lx4","namespace":"openshift-oauth-apiserver","containers":[{"name":"oauth-apiserver"}]},{"name":"kube-controller-manager-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"kube-controller-manager"},{"name":"cluster-policy-controller"},{"name":"kube-controller-manager-cert-syncer"},{"name":"kube-controller-manager-recovery-controller"}]},{"name":"tuned-9gcnf","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"tuned"}]},{"name":"machine-config-server-9vkxz","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-server"}]},{"name":"ironic-proxy-2kx2d","namespace":"openshift-machine-api","containers":[{"name":"ironic-proxy"}]},{"name":"node-ca-nlcjw","namespace":"openshift-image-registry","containers":[{"name":"node-ca"}]},{"name":"metrics-server-dd7d49f48-lmkfq","namespace":"openshift-monitoring","containers":[{"name":"metrics-server"}]},{"name":"dns-default-gst5g","namespace":"openshift-dns","containers":[{"name":"dns"},{"name":"kube-rbac-proxy"}]},{"name":"insights-runtime-extractor-5ck74","namespace":"openshift-insights","containers":[{"name":"kube-rbac-proxy"},{"name":"exporter"},{"name":"extractor"}]},{"name":"secondary-scheduler-7f66f86cb7-nd4z6","namespace":"openshift-numaresources","containers":[{"name":"secondary-scheduler","memory":[{"memory_type":"memory","size":1258291200,"topology":{"nodes":[{"ID":0}]}}]}]},{"name":"oauth-openshift-68c5846dcd-llxwr","namespace":"openshift-authentication","containers":[{"name":"oauth-openshift"}]},{"name":"apiserver-b784b5f9-v7448","namespace":"openshift-apiserver","containers":[{"name":"openshift-apiserver"},{"name":"openshift-apiserver-check-endpoints"}]},{"name":"kube-controller-manager-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"guard"}]},{"name":"kube-rbac-proxy-crio-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-machine-config-operator","containers":[{"name":"kube-rbac-proxy-crio"}]},{"name":"multus-additional-cni-plugins-2h2qb","namespace":"openshift-multus","containers":[{"name":"kube-multus-additional-cni-plugins"}]},{"name":"node-resolver-4skmp","namespace":"openshift-dns","containers":[{"name":"dns-node-resolver"}]},{"name":"numaresources-controller-manager-5c4d7bfd45-qsw5f","namespace":"openshift-numaresources","containers":[{"name":"manager"}]},{"name":"etcd-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"etcdctl"},{"name":"etcd"},{"name":"etcd-metrics"},{"name":"etcd-readyz"},{"name":"etcd-rev"}]},{"name":"openshift-kube-scheduler-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"guard"}]},{"name":"numaresourcesoperator-master-w8md2","namespace":"openshift-numaresources","containers":[{"name":"resource-topology-exporter"},{"name":"shared-pool-container"}]},{"name":"iptables-alerter-5fmt4","namespace":"openshift-network-operator","containers":[{"name":"iptables-alerter"}]},{"name":"metal3-image-cache-9wppp","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"}]},{"name":"node-exporter-l5ghn","namespace":"openshift-monitoring","containers":[{"name":"node-exporter"},{"name":"kube-rbac-proxy"}]},{"name":"controller-manager-6b88548d64-8n4x9","namespace":"openshift-controller-manager","containers":[{"name":"controller-manager"}]},{"name":"keepalived-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"keepalived"},{"name":"keepalived-monitor"}]},{"name":"coredns-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"coredns"},{"name":"coredns-monitor"}]},{"name":"ovnkube-node-d5l8c","namespace":"openshift-ovn-kubernetes","containers":[{"name":"ovn-controller"},{"name":"ovn-acl-logging"},{"name":"kube-rbac-proxy-node"},{"name":"kube-rbac-proxy-ovn-metrics"},{"name":"northd"},{"name":"nbdb"},{"name":"sbdb"},{"name":"ovnkube-controller"}]},{"name":"route-controller-manager-66ff45df48-7sqls","namespace":"openshift-route-controller-manager","containers":[{"name":"route-controller-manager"}]},{"name":"openshift-kube-scheduler-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"kube-scheduler"},{"name":"kube-scheduler-cert-syncer"},{"name":"kube-scheduler-recovery-controller"}]},{"name":"network-check-target-lbpc2","namespace":"openshift-network-diagnostics","containers":[{"name":"network-check-target-container"}]},{"name":"machine-config-daemon-qcxnb","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"etcd-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"guard"}]},{"name":"master-0kni-qe-52telco5granengrdu2redhatcom-debug-nmfqc","namespace":"default","containers":[{"name":"container-00"}]},{"name":"ingress-canary-g4h8s","namespace":"openshift-ingress-canary","containers":[{"name":"serve-healthcheck-canary"}]},{"name":"kube-apiserver-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"kube-apiserver"},{"name":"kube-apiserver-cert-syncer"},{"name":"kube-apiserver-cert-regeneration-controller"},{"name":"kube-apiserver-insecure-readyz"},{"name":"kube-apiserver-check-endpoints"}]},{"name":"multus-gll45","namespace":"openshift-multus","containers":[{"name":"kube-multus"}]},{"name":"network-node-identity-5ft9p","namespace":"openshift-network-node-identity","containers":[{"name":"webhook"},{"name":"approver"}]},{"name":"kube-apiserver-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"guard"}]},{"name":"haproxy-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"haproxy"},{"name":"haproxy-monitor"}]}]}
      
      Removing debug pod ...
      Node: master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com
      
      Starting pod/master-1kni-qe-52telco5granengrdu2redhatcom-debug-lm8p2 ...
      To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
      {"pod_resources":[{"name":"monitoring-plugin-588f66cf97-vc56x","namespace":"openshift-monitoring","containers":[{"name":"monitoring-plugin"}]},{"name":"numaresourcesoperator-master-24nv7","namespace":"openshift-numaresources","containers":[{"name":"resource-topology-exporter"},{"name":"shared-pool-container"}]},{"name":"metrics-server-dd7d49f48-8x6bx","namespace":"openshift-monitoring","containers":[{"name":"metrics-server"}]},{"name":"master-1kni-qe-52telco5granengrdu2redhatcom-debug-lm8p2","namespace":"default","containers":[{"name":"container-00"}]},{"name":"metal3-image-cache-kp5qx","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"}]},{"name":"machine-config-controller-95c478994-2b6cb","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-controller"},{"name":"kube-rbac-proxy"}]},{"name":"machine-config-server-ggt8f","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-server"}]},{"name":"apiserver-575c85d5d4-pd4hg","namespace":"openshift-oauth-apiserver","containers":[{"name":"oauth-apiserver"}]},{"name":"console-559d876ff5-4qhcd","namespace":"openshift-console","containers":[{"name":"console"}]},{"name":"keepalived-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"keepalived"},{"name":"keepalived-monitor"}]},{"name":"cluster-version-operator-556f845b59-vd6j2","namespace":"openshift-cluster-version","containers":[{"name":"cluster-version-operator"}]},{"name":"etcd-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"guard"}]},{"name":"network-node-identity-wtlws","namespace":"openshift-network-node-identity","containers":[{"name":"webhook"},{"name":"approver"}]},{"name":"node-ca-wwjrt","namespace":"openshift-image-registry","containers":[{"name":"node-ca"}]},{"name":"csi-snapshot-controller-54d6b94fb5-fx52h","namespace":"openshift-cluster-storage-operator","containers":[{"name":"snapshot-controller"}]},{"name":"metal3-image-customization-6bfcd57c86-m7b6t","namespace":"openshift-machine-api","containers":[{"name":"machine-image-customization-controller"}]},{"name":"kni-nrop-lvrtd","namespace":"openshift-marketplace","containers":[{"name":"registry-server"}]},{"name":"router-default-6658bcc57-nk8g7","namespace":"openshift-ingress","containers":[{"name":"router"}]},{"name":"alertmanager-main-0","namespace":"openshift-monitoring","containers":[{"name":"alertmanager"},{"name":"config-reloader"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-metric"},{"name":"prom-label-proxy"}]},{"name":"coredns-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"coredns"},{"name":"coredns-monitor"}]},{"name":"multus-admission-controller-788794cf6f-fmnch","namespace":"openshift-multus","containers":[{"name":"multus-admission-controller"},{"name":"kube-rbac-proxy"}]},{"name":"ovnkube-control-plane-7d7c8cfc7-lp5r8","namespace":"openshift-ovn-kubernetes","containers":[{"name":"kube-rbac-proxy"},{"name":"ovnkube-cluster-manager"}]},{"name":"openshift-state-metrics-6d5466cd5b-js5qq","namespace":"openshift-monitoring","containers":[{"name":"kube-rbac-proxy-main"},{"name":"kube-rbac-proxy-self"},{"name":"openshift-state-metrics"}]},{"name":"route-controller-manager-66ff45df48-l6wl4","namespace":"openshift-route-controller-manager","containers":[{"name":"route-controller-manager"}]},{"name":"node-resolver-dvcj4","namespace":"openshift-dns","containers":[{"name":"dns-node-resolver"}]},{"name":"kube-apiserver-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"guard"}]},{"name":"kube-apiserver-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"kube-apiserver"},{"name":"kube-apiserver-cert-syncer"},{"name":"kube-apiserver-cert-regeneration-controller"},{"name":"kube-apiserver-insecure-readyz"},{"name":"kube-apiserver-check-endpoints"}]},{"name":"dns-default-w2zzp","namespace":"openshift-dns","containers":[{"name":"dns"},{"name":"kube-rbac-proxy"}]},{"name":"prometheus-k8s-1","namespace":"openshift-monitoring","containers":[{"name":"prometheus"},{"name":"config-reloader"},{"name":"thanos-sidecar"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-thanos"}]},{"name":"network-check-source-cddd7948c-8jcf4","namespace":"openshift-network-diagnostics","containers":[{"name":"check-endpoints"}]},{"name":"networking-console-plugin-9f4fb68bd-sk26n","namespace":"openshift-network-console","containers":[{"name":"networking-console-plugin"}]},{"name":"haproxy-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"haproxy"},{"name":"haproxy-monitor"}]},{"name":"network-metrics-daemon-nxww5","namespace":"openshift-multus","containers":[{"name":"network-metrics-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"network-check-target-xmmr9","namespace":"openshift-network-diagnostics","containers":[{"name":"network-check-target-container"}]},{"name":"machine-config-daemon-9rzwb","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"controller-manager-6b88548d64-5wmfc","namespace":"openshift-controller-manager","containers":[{"name":"controller-manager"}]},{"name":"metal3-baremetal-operator-7f5d87c4c6-zcn6q","namespace":"openshift-machine-api","containers":[{"name":"metal3-baremetal-operator"}]},{"name":"kube-controller-manager-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"kube-controller-manager"},{"name":"cluster-policy-controller"},{"name":"kube-controller-manager-cert-syncer"},{"name":"kube-controller-manager-recovery-controller"}]},{"name":"openshift-kube-scheduler-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"kube-scheduler"},{"name":"kube-scheduler-cert-syncer"},{"name":"kube-scheduler-recovery-controller"}]},{"name":"multus-additional-cni-plugins-7kq6s","namespace":"openshift-multus","containers":[{"name":"kube-multus-additional-cni-plugins"}]},{"name":"iptables-alerter-qjw2d","namespace":"openshift-network-operator","containers":[{"name":"iptables-alerter"}]},{"name":"thanos-querier-df477fb5-9p5hc","namespace":"openshift-monitoring","containers":[{"name":"thanos-query"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"prom-label-proxy"},{"name":"kube-rbac-proxy-rules"},{"name":"kube-rbac-proxy-metrics"}]},{"name":"downloads-67fc445bcb-dt9cn","namespace":"openshift-console","containers":[{"name":"download-server"}]},{"name":"packageserver-56d59bdf5b-mlwz5","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"packageserver"}]},{"name":"tuned-prpvs","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"tuned"}]},{"name":"ingress-canary-df5xl","namespace":"openshift-ingress-canary","containers":[{"name":"serve-healthcheck-canary"}]},{"name":"kube-controller-manager-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"guard"}]},{"name":"service-ca-69d5957859-sc8dn","namespace":"openshift-service-ca","containers":[{"name":"service-ca-controller"}]},{"name":"machine-api-controllers-5b948b4fff-cmkz6","namespace":"openshift-machine-api","containers":[{"name":"machineset-controller"},{"name":"machine-controller"},{"name":"nodelink-controller"},{"name":"machine-healthcheck-controller"},{"name":"kube-rbac-proxy-machineset-mtrc"},{"name":"kube-rbac-proxy-machine-mtrc"},{"name":"kube-rbac-proxy-mhc-mtrc"}]},{"name":"machine-api-operator-77f76fd656-mwkqg","namespace":"openshift-machine-api","containers":[{"name":"kube-rbac-proxy"},{"name":"machine-api-operator"}]},{"name":"secondary-scheduler-7f66f86cb7-rnntp","namespace":"openshift-numaresources","containers":[{"name":"secondary-scheduler","memory":[{"memory_type":"memory","size":1258291200,"topology":{"nodes":[{"ID":0}]}}]}]},{"name":"ovnkube-node-tpkfl","namespace":"openshift-ovn-kubernetes","containers":[{"name":"ovn-controller"},{"name":"ovn-acl-logging"},{"name":"kube-rbac-proxy-node"},{"name":"kube-rbac-proxy-ovn-metrics"},{"name":"northd"},{"name":"nbdb"},{"name":"sbdb"},{"name":"ovnkube-controller"}]},{"name":"oauth-openshift-68c5846dcd-65fvp","namespace":"openshift-authentication","containers":[{"name":"oauth-openshift"}]},{"name":"cluster-samples-operator-54997c887d-s5j9t","namespace":"openshift-cluster-samples-operator","containers":[{"name":"cluster-samples-operator"},{"name":"cluster-samples-operator-watch"}]},{"name":"kube-rbac-proxy-crio-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-machine-config-operator","containers":[{"name":"kube-rbac-proxy-crio"}]},{"name":"etcd-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"etcdctl"},{"name":"etcd"},{"name":"etcd-metrics"},{"name":"etcd-readyz"},{"name":"etcd-rev"}]},{"name":"node-exporter-58cgs","namespace":"openshift-monitoring","containers":[{"name":"node-exporter"},{"name":"kube-rbac-proxy"}]},{"name":"openshift-kube-scheduler-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"guard"}]},{"name":"prometheus-operator-7db86f56b5-mw8lc","namespace":"openshift-monitoring","containers":[{"name":"prometheus-operator"},{"name":"kube-rbac-proxy"}]},{"name":"ironic-proxy-p9tlr","namespace":"openshift-machine-api","containers":[{"name":"ironic-proxy"}]},{"name":"migrator-6b64d5cdcc-2hkg5","namespace":"openshift-kube-storage-version-migrator","containers":[{"name":"migrator"},{"name":"graceful-termination"}]},{"name":"multus-x2xjj","namespace":"openshift-multus","containers":[{"name":"kube-multus"}]},{"name":"insights-runtime-extractor-b6rxg","namespace":"openshift-insights","containers":[{"name":"kube-rbac-proxy"},{"name":"exporter"},{"name":"extractor"}]},{"name":"apiserver-b784b5f9-xw9dj","namespace":"openshift-apiserver","containers":[{"name":"openshift-apiserver"},{"name":"openshift-apiserver-check-endpoints"}]},{"name":"kube-state-metrics-67b87967bd-2p2kw","namespace":"openshift-monitoring","containers":[{"name":"kube-state-metrics"},{"name":"kube-rbac-proxy-main"},{"name":"kube-rbac-proxy-self"}]},{"name":"prometheus-operator-admission-webhook-6c76dfdf85-smqdb","namespace":"openshift-monitoring","containers":[{"name":"prometheus-operator-admission-webhook"}]},{"name":"metal3-6777cd8498-jj8rx","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"},{"name":"metal3-ironic"},{"name":"metal3-ramdisk-logs"}]}]}
      
      Removing debug pod ...
      Node: master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com
      
      Starting pod/master-2kni-qe-52telco5granengrdu2redhatcom-debug-pmntx ...
      To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
      {"pod_resources":[{"name":"oauth-openshift-68c5846dcd-7nm42","namespace":"openshift-authentication","containers":[{"name":"oauth-openshift"}]},{"name":"console-operator-5f556cc8f7-tr9c9","namespace":"openshift-console-operator","containers":[{"name":"console-operator"}]},{"name":"cluster-autoscaler-operator-697f7bd848-cvls4","namespace":"openshift-machine-api","containers":[{"name":"kube-rbac-proxy"},{"name":"cluster-autoscaler-operator"}]},{"name":"alertmanager-main-1","namespace":"openshift-monitoring","containers":[{"name":"alertmanager"},{"name":"config-reloader"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-metric"},{"name":"prom-label-proxy"}]},{"name":"route-controller-manager-66ff45df48-29hjg","namespace":"openshift-route-controller-manager","containers":[{"name":"route-controller-manager"}]},{"name":"thanos-querier-df477fb5-22r5d","namespace":"openshift-monitoring","containers":[{"name":"thanos-query"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"prom-label-proxy"},{"name":"kube-rbac-proxy-rules"},{"name":"kube-rbac-proxy-metrics"}]},{"name":"ingress-operator-6b9d98f45d-f2w8b","namespace":"openshift-ingress-operator","containers":[{"name":"ingress-operator"},{"name":"kube-rbac-proxy"}]},{"name":"cluster-olm-operator-58d5d4b49d-zdvlf","namespace":"openshift-cluster-olm-operator","containers":[{"name":"cluster-olm-operator"}]},{"name":"olm-operator-7c9d879469-vr2z6","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"olm-operator"}]},{"name":"metal3-image-cache-djkjg","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"}]},{"name":"csi-snapshot-controller-operator-596c645fb5-7cqld","namespace":"openshift-cluster-storage-operator","containers":[{"name":"csi-snapshot-controller-operator"}]},{"name":"prometheus-k8s-0","namespace":"openshift-monitoring","containers":[{"name":"prometheus"},{"name":"config-reloader"},{"name":"thanos-sidecar"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-thanos"}]},{"name":"control-plane-machine-set-operator-5cf4849d9d-xxxqt","namespace":"openshift-machine-api","containers":[{"name":"control-plane-machine-set-operator"}]},{"name":"cluster-monitoring-operator-5b9b49c94f-b7zb4","namespace":"openshift-monitoring","containers":[{"name":"cluster-monitoring-operator"}]},{"name":"kube-controller-manager-operator-86b9c868d8-4zvkr","namespace":"openshift-kube-controller-manager-operator","containers":[{"name":"kube-controller-manager-operator"}]},{"name":"secondary-scheduler-7f66f86cb7-qf4pv","namespace":"openshift-numaresources","containers":[{"name":"secondary-scheduler","memory":[{"memory_type":"memory","size":1258291200,"topology":{"nodes":[{"ID":0}]}}]}]},{"name":"network-metrics-daemon-2qdn2","namespace":"openshift-multus","containers":[{"name":"network-metrics-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"ovnkube-control-plane-7d7c8cfc7-22trz","namespace":"openshift-ovn-kubernetes","containers":[{"name":"kube-rbac-proxy"},{"name":"ovnkube-cluster-manager"}]},{"name":"kube-apiserver-operator-95bdb85b9-q79ht","namespace":"openshift-kube-apiserver-operator","containers":[{"name":"kube-apiserver-operator"}]},{"name":"openshift-config-operator-597989c67-l5rnp","namespace":"openshift-config-operator","containers":[{"name":"openshift-config-operator"}]},{"name":"catalogd-controller-manager-884c99498-kxspf","namespace":"openshift-catalogd","containers":[{"name":"manager"}]},{"name":"kube-rbac-proxy-crio-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-machine-config-operator","containers":[{"name":"kube-rbac-proxy-crio"}]},{"name":"downloads-67fc445bcb-xsn5f","namespace":"openshift-console","containers":[{"name":"download-server"}]},{"name":"cluster-cloud-controller-manager-operator-6c796f8dff-69wd8","namespace":"openshift-cloud-controller-manager-operator","containers":[{"name":"cluster-cloud-controller-manager"},{"name":"config-sync-controllers"},{"name":"kube-rbac-proxy"}]},{"name":"etcd-operator-57d79df4bd-f24lp","namespace":"openshift-etcd-operator","containers":[{"name":"etcd-operator"}]},{"name":"openshift-kube-scheduler-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"kube-scheduler"},{"name":"kube-scheduler-cert-syncer"},{"name":"kube-scheduler-recovery-controller"}]},{"name":"network-check-target-gpk98","namespace":"openshift-network-diagnostics","containers":[{"name":"network-check-target-container"}]},{"name":"machine-config-daemon-c2nwm","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"kube-apiserver-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"guard"}]},{"name":"cluster-image-registry-operator-974c6b758-d8xg9","namespace":"openshift-image-registry","containers":[{"name":"cluster-image-registry-operator"}]},{"name":"image-registry-796d5b566b-xmdrp","namespace":"openshift-image-registry","containers":[{"name":"registry"}]},{"name":"master-2kni-qe-52telco5granengrdu2redhatcom-debug-pmntx","namespace":"default","containers":[{"name":"container-00"}]},{"name":"openshift-controller-manager-operator-55f856587f-ppwhv","namespace":"openshift-controller-manager-operator","containers":[{"name":"openshift-controller-manager-operator"}]},{"name":"insights-operator-54f6b9ddc5-wxsks","namespace":"openshift-insights","containers":[{"name":"insights-operator"}]},{"name":"kube-storage-version-migrator-operator-6676b7fbc9-v2glx","namespace":"openshift-kube-storage-version-migrator-operator","containers":[{"name":"kube-storage-version-migrator-operator"}]},{"name":"ironic-proxy-rvfvx","namespace":"openshift-machine-api","containers":[{"name":"ironic-proxy"}]},{"name":"cluster-storage-operator-76cf476f8c-bl5px","namespace":"openshift-cluster-storage-operator","containers":[{"name":"cluster-storage-operator"}]},{"name":"multus-admission-controller-788794cf6f-vn4rq","namespace":"openshift-multus","containers":[{"name":"multus-admission-controller"},{"name":"kube-rbac-proxy"}]},{"name":"coredns-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"coredns"},{"name":"coredns-monitor"}]},{"name":"keepalived-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"keepalived"},{"name":"keepalived-monitor"}]},{"name":"package-server-manager-7b6bf69448-68ghr","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"kube-rbac-proxy"},{"name":"package-server-manager"}]},{"name":"catalog-operator-597fddd74-rxzkj","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"catalog-operator"}]},{"name":"machine-config-operator-6f4b598998-qjx4t","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-operator"},{"name":"kube-rbac-proxy"}]},{"name":"cluster-baremetal-operator-7d59475c8-x6njw","namespace":"openshift-machine-api","containers":[{"name":"cluster-baremetal-operator"},{"name":"baremetal-kube-rbac-proxy"}]},{"name":"etcd-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"etcdctl"},{"name":"etcd"},{"name":"etcd-metrics"},{"name":"etcd-readyz"},{"name":"etcd-rev"}]},{"name":"apiserver-575c85d5d4-n62p9","namespace":"openshift-oauth-apiserver","containers":[{"name":"oauth-apiserver"}]},{"name":"cluster-node-tuning-operator-78f47f8887-4sqlc","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"cluster-node-tuning-operator"}]},{"name":"multus-t8rp2","namespace":"openshift-multus","containers":[{"name":"kube-multus"}]},{"name":"dns-default-t8phv","namespace":"openshift-dns","containers":[{"name":"dns"},{"name":"kube-rbac-proxy"}]},{"name":"apiserver-b784b5f9-47fg6","namespace":"openshift-apiserver","containers":[{"name":"openshift-apiserver"},{"name":"openshift-apiserver-check-endpoints"}]},{"name":"openshift-kube-scheduler-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"guard"}]},{"name":"controller-manager-6b88548d64-5jc6z","namespace":"openshift-controller-manager","containers":[{"name":"controller-manager"}]},{"name":"ovnkube-node-pgxjg","namespace":"openshift-ovn-kubernetes","containers":[{"name":"ovn-controller"},{"name":"ovn-acl-logging"},{"name":"kube-rbac-proxy-node"},{"name":"kube-rbac-proxy-ovn-metrics"},{"name":"northd"},{"name":"nbdb"},{"name":"sbdb"},{"name":"ovnkube-controller"}]},{"name":"machine-config-server-8j64k","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-server"}]},{"name":"node-resolver-j89fb","namespace":"openshift-dns","containers":[{"name":"dns-node-resolver"}]},{"name":"console-559d876ff5-6vwns","namespace":"openshift-console","containers":[{"name":"console"}]},{"name":"network-operator-566d6954c9-6bvmv","namespace":"openshift-network-operator","containers":[{"name":"network-operator"}]},{"name":"kube-controller-manager-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"kube-controller-manager"},{"name":"cluster-policy-controller"},{"name":"kube-controller-manager-cert-syncer"},{"name":"kube-controller-manager-recovery-controller"}]},{"name":"kube-apiserver-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"kube-apiserver"},{"name":"kube-apiserver-cert-syncer"},{"name":"kube-apiserver-cert-regeneration-controller"},{"name":"kube-apiserver-insecure-readyz"},{"name":"kube-apiserver-check-endpoints"}]},{"name":"kube-controller-manager-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"guard"}]},{"name":"machine-approver-64f8f6b495-vb6v4","namespace":"openshift-cluster-machine-approver","containers":[{"name":"kube-rbac-proxy"},{"name":"machine-approver-controller"}]},{"name":"service-ca-operator-7d7d8684dc-82lf2","namespace":"openshift-service-ca-operator","containers":[{"name":"service-ca-operator"}]},{"name":"packageserver-56d59bdf5b-6zg9m","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"packageserver"}]},{"name":"dns-operator-596f54c5bb-npvz9","namespace":"openshift-dns-operator","containers":[{"name":"dns-operator"},{"name":"kube-rbac-proxy"}]},{"name":"insights-runtime-extractor-6jvb8","namespace":"openshift-insights","containers":[{"name":"kube-rbac-proxy"},{"name":"exporter"},{"name":"extractor"}]},{"name":"node-ca-wt5xl","namespace":"openshift-image-registry","containers":[{"name":"node-ca"}]},{"name":"csi-snapshot-controller-54d6b94fb5-x7qv6","namespace":"openshift-cluster-storage-operator","containers":[{"name":"snapshot-controller"}]},{"name":"authentication-operator-797c55ddf-jcs4r","namespace":"openshift-authentication-operator","containers":[{"name":"authentication-operator"}]},{"name":"networking-console-plugin-9f4fb68bd-cwpt5","namespace":"openshift-network-console","containers":[{"name":"networking-console-plugin"}]},{"name":"openshift-kube-scheduler-operator-558f48c74c-dc6t2","namespace":"openshift-kube-scheduler-operator","containers":[{"name":"kube-scheduler-operator-container"}]},{"name":"operator-controller-controller-manager-7967cc8894-7rbxq","namespace":"openshift-operator-controller","containers":[{"name":"manager"}]},{"name":"marketplace-operator-7df8b4fb6-m88pb","namespace":"openshift-marketplace","containers":[{"name":"marketplace-operator"}]},{"name":"iptables-alerter-f4xsk","namespace":"openshift-network-operator","containers":[{"name":"iptables-alerter"}]},{"name":"monitoring-plugin-588f66cf97-jmtsf","namespace":"openshift-monitoring","containers":[{"name":"monitoring-plugin"}]},{"name":"router-default-6658bcc57-jtqlm","namespace":"openshift-ingress","containers":[{"name":"router"}]},{"name":"numaresourcesoperator-master-wv8v9","namespace":"openshift-numaresources","containers":[{"name":"resource-topology-exporter"},{"name":"shared-pool-container"}]},{"name":"haproxy-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"haproxy"},{"name":"haproxy-monitor"}]},{"name":"network-node-identity-mln2t","namespace":"openshift-network-node-identity","containers":[{"name":"webhook"},{"name":"approver"}]},{"name":"tuned-qgcfl","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"tuned"}]},{"name":"node-exporter-752wt","namespace":"openshift-monitoring","containers":[{"name":"node-exporter"},{"name":"kube-rbac-proxy"}]},{"name":"etcd-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"guard"}]},{"name":"openshift-apiserver-operator-9fffdd54f-bvjnq","namespace":"openshift-apiserver-operator","containers":[{"name":"openshift-apiserver-operator"}]},{"name":"cloud-credential-operator-67c9777895-44x2l","namespace":"openshift-cloud-credential-operator","containers":[{"name":"kube-rbac-proxy"},{"name":"cloud-credential-operator"}]},{"name":"prometheus-operator-admission-webhook-6c76dfdf85-q5mlf","namespace":"openshift-monitoring","containers":[{"name":"prometheus-operator-admission-webhook"}]},{"name":"multus-additional-cni-plugins-t2vwg","namespace":"openshift-multus","containers":[{"name":"kube-multus-additional-cni-plugins"}]},{"name":"ingress-canary-mjnmn","namespace":"openshift-ingress-canary","containers":[{"name":"serve-healthcheck-canary"}]}]}
      

              fromani@redhat.com Francesco Romani
              rh-ee-rshemtov Roy Shemtov
              None
              None
              Roy Shemtov Roy Shemtov
              None
              Votes:
              0 Vote for this issue
              Watchers:
              7 Start watching this issue

                Created:
                Updated: