The bug here is after this bug was verified: https://issues.redhat.com/browse/OCPBUGS-56785

The problem here is that after creating a scenario for which the secondary scheduler will become stalled creating a test pod to see if the pod will get scheduler is causing that pod to be stuck forever and is failing on invalid topology data.

The environment that was used here is a compact cluster.

Always

1. Configure the kubelet to have a memory manager policy = static
2. Run a job to qualify as for memory pinning
3. Wait for the NRT data to settle
4. Create a test pod to see that it is running

[root@registry bug-pod-resource-api]# oc get pods
NAME                      READY   STATUS      RESTARTS   AGE
generic-pause-pod-6ks7v   0/1     Completed   0          20m
generic-pause-pod-x9kkf   0/1     Completed   0          20m
test-pod                  0/1     Pending     0          17m

[root@registry bug-pod-resource-api]# oc get clusterversion
NAME      VERSION                              AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.20.0-0.nightly-2025-08-19-081012   True        False         5d22h   Cluster version is 4.20.0-0.nightly-2025-08-19-081012

[root@registry bug-pod-resource-api]# oc get no
NAME                                                STATUS   ROLES                         AGE     VERSION
master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com   Ready    control-plane,master,worker   5d22h   v1.33.3
master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com   Ready    control-plane,master,worker   5d22h   v1.33.3
master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com   Ready    control-plane,master,worker   5d22h   v1.33.3

[root@registry bug-pod-resource-api]# oc get mcp
NAME     CONFIG                                             UPDATED   UPDATING   DEGRADED   MACHINECOUNT   READYMACHINECOUNT   UPDATEDMACHINECOUNT   DEGRADEDMACHINECOUNT   AGE
master   rendered-master-652f84cbfe0a26f3625b34047cee88b6   True      False      False      3              3                   3                     0                      5d22h
worker   rendered-worker-bcc58fa70929755e05487ae05b9b93b6   True      False      False      0              0                   0                     0                      5d22h

apiVersion: performance.openshift.io/v2
kind: PerformanceProfile
metadata:
  name: performance
  annotations:
    kubeletconfig.experimental: |
      {"topologyManagerScope": "container",
      "cpuManagerPolicyOptions": {"full-pcpus-only": "false"}
      }
spec:
  cpu:
    isolated: "16-103"
    reserved: "0-15"
  machineConfigPoolSelector:
    pools.operator.machineconfiguration.openshift.io/master: ""
  nodeSelector:
    node-role.kubernetes.io/master: ""
  numa:
    topologyPolicy: single-numa-node
  realTimeKernel:
    enabled: true
  workloadHints:
    highPowerConsumption: true
    perPodPowerManagement: false
    realTime: true

apiVersion: batch/v1
kind: Job
metadata:
  labels:
    app: idle-gu-job-sched-stall
  name: generic-pause-pod
spec:
  backoffLimit: 6
  completionMode: NonIndexed
  completions: 2
  manualSelector: false
  parallelism: 2
  podReplacementPolicy: TerminatingOrFailed
  suspend: false
  template:
    metadata:
      labels:
        app: idle-gu-job-sched-stall
    spec:
      containers:
      - args:
        - 1s
        command:
        - /bin/sleep
        image: quay.io/openshift-kni/pause:test-ci
        imagePullPolicy: IfNotPresent
        name: generic-job-idle
        resources:
          limits:
            cpu: 100m
            memory: 256Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
      dnsPolicy: ClusterFirst
      restartPolicy: Never
      schedulerName: default-scheduler
      terminationGracePeriodSeconds: 30
      topologySpreadConstraints:
      - labelSelector:
          matchLabels:
            app: idle-gu-job-sched-stall
        matchLabelKeys:
        - pod-template-hash
        maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: DoNotSchedule

apiVersion: v1
kind: Pod
metadata:
  namespace: default
  name: test-pod
  labels:
    test-type: "scheduler-test-pod"
spec:
  schedulerName: topo-aware-scheduler
  containers:
  - name: pause
    image: registry.k8s.io/pause:3.9
    resources:
      limits:
        cpu: "2"
        memory: 64Mi
      requests:
        cpu: "2"
        memory: 64Mi

[root@registry bug-pod-resource-api]# oc get pods
NAME                      READY   STATUS      RESTARTS   AGE
generic-pause-pod-6ks7v   0/1     Completed   0          20m
generic-pause-pod-x9kkf   0/1     Completed   0          20m
test-pod                  0/1     Pending     0          17m

[root@registry bug-pod-resource-api]# oc describe pod/test-pod
Name:             test-pod
Namespace:        default
Priority:         0
Service Account:  default
Node:             <none>
Labels:           test-type=scheduler-test-pod
Annotations:      <none>
Status:           Pending
IP:               
IPs:              <none>
Containers:
  pause:
    Image:      registry.k8s.io/pause:3.9
    Port:       <none>
    Host Port:  <none>
    Limits:
      cpu:     2
      memory:  64Mi
    Requests:
      cpu:        2
      memory:     64Mi
    Environment:  <none>
    Mounts:
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-v2rp5 (ro)
Conditions:
  Type           Status
  PodScheduled   False 
Volumes:
  kube-api-access-v2rp5:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    Optional:                false
    DownwardAPI:             true
    ConfigMapName:           openshift-service-ca.crt
    Optional:                false
QoS Class:                   Guaranteed
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason            Age               From                  Message
  ----     ------            ----              ----                  -------
  Warning  FailedScheduling  7s (x4 over 20s)  topo-aware-scheduler  0/3 nodes are available: 3 invalid node topology data. preemption: 0/3 nodes are available: 3 No preemption victims found for incoming pod.

[kni@registry.kni-qe-52 bug-pod-resource-api]$ for node in `oc get nodes -l node-role.kubernetes.io/worker= --no-headers | awk '{print $1}'`; do echo -e "Node: $node\n";oc debug node/$node -- chroot /host ./tmp/knit-v0.2.1-linux-amd64 podres   ;done
Node: master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com

Starting pod/master-0kni-qe-52telco5granengrdu2redhatcom-debug-nmfqc ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
{"pod_resources":[{"name":"network-metrics-daemon-x52mj","namespace":"openshift-multus","containers":[{"name":"network-metrics-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"apiserver-575c85d5d4-l6lx4","namespace":"openshift-oauth-apiserver","containers":[{"name":"oauth-apiserver"}]},{"name":"kube-controller-manager-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"kube-controller-manager"},{"name":"cluster-policy-controller"},{"name":"kube-controller-manager-cert-syncer"},{"name":"kube-controller-manager-recovery-controller"}]},{"name":"tuned-9gcnf","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"tuned"}]},{"name":"machine-config-server-9vkxz","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-server"}]},{"name":"ironic-proxy-2kx2d","namespace":"openshift-machine-api","containers":[{"name":"ironic-proxy"}]},{"name":"node-ca-nlcjw","namespace":"openshift-image-registry","containers":[{"name":"node-ca"}]},{"name":"metrics-server-dd7d49f48-lmkfq","namespace":"openshift-monitoring","containers":[{"name":"metrics-server"}]},{"name":"dns-default-gst5g","namespace":"openshift-dns","containers":[{"name":"dns"},{"name":"kube-rbac-proxy"}]},{"name":"insights-runtime-extractor-5ck74","namespace":"openshift-insights","containers":[{"name":"kube-rbac-proxy"},{"name":"exporter"},{"name":"extractor"}]},{"name":"secondary-scheduler-7f66f86cb7-nd4z6","namespace":"openshift-numaresources","containers":[{"name":"secondary-scheduler","memory":[{"memory_type":"memory","size":1258291200,"topology":{"nodes":[{"ID":0}]}}]}]},{"name":"oauth-openshift-68c5846dcd-llxwr","namespace":"openshift-authentication","containers":[{"name":"oauth-openshift"}]},{"name":"apiserver-b784b5f9-v7448","namespace":"openshift-apiserver","containers":[{"name":"openshift-apiserver"},{"name":"openshift-apiserver-check-endpoints"}]},{"name":"kube-controller-manager-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"guard"}]},{"name":"kube-rbac-proxy-crio-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-machine-config-operator","containers":[{"name":"kube-rbac-proxy-crio"}]},{"name":"multus-additional-cni-plugins-2h2qb","namespace":"openshift-multus","containers":[{"name":"kube-multus-additional-cni-plugins"}]},{"name":"node-resolver-4skmp","namespace":"openshift-dns","containers":[{"name":"dns-node-resolver"}]},{"name":"numaresources-controller-manager-5c4d7bfd45-qsw5f","namespace":"openshift-numaresources","containers":[{"name":"manager"}]},{"name":"etcd-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"etcdctl"},{"name":"etcd"},{"name":"etcd-metrics"},{"name":"etcd-readyz"},{"name":"etcd-rev"}]},{"name":"openshift-kube-scheduler-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"guard"}]},{"name":"numaresourcesoperator-master-w8md2","namespace":"openshift-numaresources","containers":[{"name":"resource-topology-exporter"},{"name":"shared-pool-container"}]},{"name":"iptables-alerter-5fmt4","namespace":"openshift-network-operator","containers":[{"name":"iptables-alerter"}]},{"name":"metal3-image-cache-9wppp","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"}]},{"name":"node-exporter-l5ghn","namespace":"openshift-monitoring","containers":[{"name":"node-exporter"},{"name":"kube-rbac-proxy"}]},{"name":"controller-manager-6b88548d64-8n4x9","namespace":"openshift-controller-manager","containers":[{"name":"controller-manager"}]},{"name":"keepalived-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"keepalived"},{"name":"keepalived-monitor"}]},{"name":"coredns-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"coredns"},{"name":"coredns-monitor"}]},{"name":"ovnkube-node-d5l8c","namespace":"openshift-ovn-kubernetes","containers":[{"name":"ovn-controller"},{"name":"ovn-acl-logging"},{"name":"kube-rbac-proxy-node"},{"name":"kube-rbac-proxy-ovn-metrics"},{"name":"northd"},{"name":"nbdb"},{"name":"sbdb"},{"name":"ovnkube-controller"}]},{"name":"route-controller-manager-66ff45df48-7sqls","namespace":"openshift-route-controller-manager","containers":[{"name":"route-controller-manager"}]},{"name":"openshift-kube-scheduler-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"kube-scheduler"},{"name":"kube-scheduler-cert-syncer"},{"name":"kube-scheduler-recovery-controller"}]},{"name":"network-check-target-lbpc2","namespace":"openshift-network-diagnostics","containers":[{"name":"network-check-target-container"}]},{"name":"machine-config-daemon-qcxnb","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"etcd-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"guard"}]},{"name":"master-0kni-qe-52telco5granengrdu2redhatcom-debug-nmfqc","namespace":"default","containers":[{"name":"container-00"}]},{"name":"ingress-canary-g4h8s","namespace":"openshift-ingress-canary","containers":[{"name":"serve-healthcheck-canary"}]},{"name":"kube-apiserver-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"kube-apiserver"},{"name":"kube-apiserver-cert-syncer"},{"name":"kube-apiserver-cert-regeneration-controller"},{"name":"kube-apiserver-insecure-readyz"},{"name":"kube-apiserver-check-endpoints"}]},{"name":"multus-gll45","namespace":"openshift-multus","containers":[{"name":"kube-multus"}]},{"name":"network-node-identity-5ft9p","namespace":"openshift-network-node-identity","containers":[{"name":"webhook"},{"name":"approver"}]},{"name":"kube-apiserver-guard-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"guard"}]},{"name":"haproxy-master-0.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"haproxy"},{"name":"haproxy-monitor"}]}]}

Removing debug pod ...
Node: master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com

Starting pod/master-1kni-qe-52telco5granengrdu2redhatcom-debug-lm8p2 ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
{"pod_resources":[{"name":"monitoring-plugin-588f66cf97-vc56x","namespace":"openshift-monitoring","containers":[{"name":"monitoring-plugin"}]},{"name":"numaresourcesoperator-master-24nv7","namespace":"openshift-numaresources","containers":[{"name":"resource-topology-exporter"},{"name":"shared-pool-container"}]},{"name":"metrics-server-dd7d49f48-8x6bx","namespace":"openshift-monitoring","containers":[{"name":"metrics-server"}]},{"name":"master-1kni-qe-52telco5granengrdu2redhatcom-debug-lm8p2","namespace":"default","containers":[{"name":"container-00"}]},{"name":"metal3-image-cache-kp5qx","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"}]},{"name":"machine-config-controller-95c478994-2b6cb","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-controller"},{"name":"kube-rbac-proxy"}]},{"name":"machine-config-server-ggt8f","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-server"}]},{"name":"apiserver-575c85d5d4-pd4hg","namespace":"openshift-oauth-apiserver","containers":[{"name":"oauth-apiserver"}]},{"name":"console-559d876ff5-4qhcd","namespace":"openshift-console","containers":[{"name":"console"}]},{"name":"keepalived-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"keepalived"},{"name":"keepalived-monitor"}]},{"name":"cluster-version-operator-556f845b59-vd6j2","namespace":"openshift-cluster-version","containers":[{"name":"cluster-version-operator"}]},{"name":"etcd-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"guard"}]},{"name":"network-node-identity-wtlws","namespace":"openshift-network-node-identity","containers":[{"name":"webhook"},{"name":"approver"}]},{"name":"node-ca-wwjrt","namespace":"openshift-image-registry","containers":[{"name":"node-ca"}]},{"name":"csi-snapshot-controller-54d6b94fb5-fx52h","namespace":"openshift-cluster-storage-operator","containers":[{"name":"snapshot-controller"}]},{"name":"metal3-image-customization-6bfcd57c86-m7b6t","namespace":"openshift-machine-api","containers":[{"name":"machine-image-customization-controller"}]},{"name":"kni-nrop-lvrtd","namespace":"openshift-marketplace","containers":[{"name":"registry-server"}]},{"name":"router-default-6658bcc57-nk8g7","namespace":"openshift-ingress","containers":[{"name":"router"}]},{"name":"alertmanager-main-0","namespace":"openshift-monitoring","containers":[{"name":"alertmanager"},{"name":"config-reloader"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-metric"},{"name":"prom-label-proxy"}]},{"name":"coredns-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"coredns"},{"name":"coredns-monitor"}]},{"name":"multus-admission-controller-788794cf6f-fmnch","namespace":"openshift-multus","containers":[{"name":"multus-admission-controller"},{"name":"kube-rbac-proxy"}]},{"name":"ovnkube-control-plane-7d7c8cfc7-lp5r8","namespace":"openshift-ovn-kubernetes","containers":[{"name":"kube-rbac-proxy"},{"name":"ovnkube-cluster-manager"}]},{"name":"openshift-state-metrics-6d5466cd5b-js5qq","namespace":"openshift-monitoring","containers":[{"name":"kube-rbac-proxy-main"},{"name":"kube-rbac-proxy-self"},{"name":"openshift-state-metrics"}]},{"name":"route-controller-manager-66ff45df48-l6wl4","namespace":"openshift-route-controller-manager","containers":[{"name":"route-controller-manager"}]},{"name":"node-resolver-dvcj4","namespace":"openshift-dns","containers":[{"name":"dns-node-resolver"}]},{"name":"kube-apiserver-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"guard"}]},{"name":"kube-apiserver-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"kube-apiserver"},{"name":"kube-apiserver-cert-syncer"},{"name":"kube-apiserver-cert-regeneration-controller"},{"name":"kube-apiserver-insecure-readyz"},{"name":"kube-apiserver-check-endpoints"}]},{"name":"dns-default-w2zzp","namespace":"openshift-dns","containers":[{"name":"dns"},{"name":"kube-rbac-proxy"}]},{"name":"prometheus-k8s-1","namespace":"openshift-monitoring","containers":[{"name":"prometheus"},{"name":"config-reloader"},{"name":"thanos-sidecar"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-thanos"}]},{"name":"network-check-source-cddd7948c-8jcf4","namespace":"openshift-network-diagnostics","containers":[{"name":"check-endpoints"}]},{"name":"networking-console-plugin-9f4fb68bd-sk26n","namespace":"openshift-network-console","containers":[{"name":"networking-console-plugin"}]},{"name":"haproxy-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"haproxy"},{"name":"haproxy-monitor"}]},{"name":"network-metrics-daemon-nxww5","namespace":"openshift-multus","containers":[{"name":"network-metrics-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"network-check-target-xmmr9","namespace":"openshift-network-diagnostics","containers":[{"name":"network-check-target-container"}]},{"name":"machine-config-daemon-9rzwb","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"controller-manager-6b88548d64-5wmfc","namespace":"openshift-controller-manager","containers":[{"name":"controller-manager"}]},{"name":"metal3-baremetal-operator-7f5d87c4c6-zcn6q","namespace":"openshift-machine-api","containers":[{"name":"metal3-baremetal-operator"}]},{"name":"kube-controller-manager-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"kube-controller-manager"},{"name":"cluster-policy-controller"},{"name":"kube-controller-manager-cert-syncer"},{"name":"kube-controller-manager-recovery-controller"}]},{"name":"openshift-kube-scheduler-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"kube-scheduler"},{"name":"kube-scheduler-cert-syncer"},{"name":"kube-scheduler-recovery-controller"}]},{"name":"multus-additional-cni-plugins-7kq6s","namespace":"openshift-multus","containers":[{"name":"kube-multus-additional-cni-plugins"}]},{"name":"iptables-alerter-qjw2d","namespace":"openshift-network-operator","containers":[{"name":"iptables-alerter"}]},{"name":"thanos-querier-df477fb5-9p5hc","namespace":"openshift-monitoring","containers":[{"name":"thanos-query"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"prom-label-proxy"},{"name":"kube-rbac-proxy-rules"},{"name":"kube-rbac-proxy-metrics"}]},{"name":"downloads-67fc445bcb-dt9cn","namespace":"openshift-console","containers":[{"name":"download-server"}]},{"name":"packageserver-56d59bdf5b-mlwz5","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"packageserver"}]},{"name":"tuned-prpvs","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"tuned"}]},{"name":"ingress-canary-df5xl","namespace":"openshift-ingress-canary","containers":[{"name":"serve-healthcheck-canary"}]},{"name":"kube-controller-manager-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"guard"}]},{"name":"service-ca-69d5957859-sc8dn","namespace":"openshift-service-ca","containers":[{"name":"service-ca-controller"}]},{"name":"machine-api-controllers-5b948b4fff-cmkz6","namespace":"openshift-machine-api","containers":[{"name":"machineset-controller"},{"name":"machine-controller"},{"name":"nodelink-controller"},{"name":"machine-healthcheck-controller"},{"name":"kube-rbac-proxy-machineset-mtrc"},{"name":"kube-rbac-proxy-machine-mtrc"},{"name":"kube-rbac-proxy-mhc-mtrc"}]},{"name":"machine-api-operator-77f76fd656-mwkqg","namespace":"openshift-machine-api","containers":[{"name":"kube-rbac-proxy"},{"name":"machine-api-operator"}]},{"name":"secondary-scheduler-7f66f86cb7-rnntp","namespace":"openshift-numaresources","containers":[{"name":"secondary-scheduler","memory":[{"memory_type":"memory","size":1258291200,"topology":{"nodes":[{"ID":0}]}}]}]},{"name":"ovnkube-node-tpkfl","namespace":"openshift-ovn-kubernetes","containers":[{"name":"ovn-controller"},{"name":"ovn-acl-logging"},{"name":"kube-rbac-proxy-node"},{"name":"kube-rbac-proxy-ovn-metrics"},{"name":"northd"},{"name":"nbdb"},{"name":"sbdb"},{"name":"ovnkube-controller"}]},{"name":"oauth-openshift-68c5846dcd-65fvp","namespace":"openshift-authentication","containers":[{"name":"oauth-openshift"}]},{"name":"cluster-samples-operator-54997c887d-s5j9t","namespace":"openshift-cluster-samples-operator","containers":[{"name":"cluster-samples-operator"},{"name":"cluster-samples-operator-watch"}]},{"name":"kube-rbac-proxy-crio-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-machine-config-operator","containers":[{"name":"kube-rbac-proxy-crio"}]},{"name":"etcd-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"etcdctl"},{"name":"etcd"},{"name":"etcd-metrics"},{"name":"etcd-readyz"},{"name":"etcd-rev"}]},{"name":"node-exporter-58cgs","namespace":"openshift-monitoring","containers":[{"name":"node-exporter"},{"name":"kube-rbac-proxy"}]},{"name":"openshift-kube-scheduler-guard-master-1.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"guard"}]},{"name":"prometheus-operator-7db86f56b5-mw8lc","namespace":"openshift-monitoring","containers":[{"name":"prometheus-operator"},{"name":"kube-rbac-proxy"}]},{"name":"ironic-proxy-p9tlr","namespace":"openshift-machine-api","containers":[{"name":"ironic-proxy"}]},{"name":"migrator-6b64d5cdcc-2hkg5","namespace":"openshift-kube-storage-version-migrator","containers":[{"name":"migrator"},{"name":"graceful-termination"}]},{"name":"multus-x2xjj","namespace":"openshift-multus","containers":[{"name":"kube-multus"}]},{"name":"insights-runtime-extractor-b6rxg","namespace":"openshift-insights","containers":[{"name":"kube-rbac-proxy"},{"name":"exporter"},{"name":"extractor"}]},{"name":"apiserver-b784b5f9-xw9dj","namespace":"openshift-apiserver","containers":[{"name":"openshift-apiserver"},{"name":"openshift-apiserver-check-endpoints"}]},{"name":"kube-state-metrics-67b87967bd-2p2kw","namespace":"openshift-monitoring","containers":[{"name":"kube-state-metrics"},{"name":"kube-rbac-proxy-main"},{"name":"kube-rbac-proxy-self"}]},{"name":"prometheus-operator-admission-webhook-6c76dfdf85-smqdb","namespace":"openshift-monitoring","containers":[{"name":"prometheus-operator-admission-webhook"}]},{"name":"metal3-6777cd8498-jj8rx","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"},{"name":"metal3-ironic"},{"name":"metal3-ramdisk-logs"}]}]}

Removing debug pod ...
Node: master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com

Starting pod/master-2kni-qe-52telco5granengrdu2redhatcom-debug-pmntx ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
{"pod_resources":[{"name":"oauth-openshift-68c5846dcd-7nm42","namespace":"openshift-authentication","containers":[{"name":"oauth-openshift"}]},{"name":"console-operator-5f556cc8f7-tr9c9","namespace":"openshift-console-operator","containers":[{"name":"console-operator"}]},{"name":"cluster-autoscaler-operator-697f7bd848-cvls4","namespace":"openshift-machine-api","containers":[{"name":"kube-rbac-proxy"},{"name":"cluster-autoscaler-operator"}]},{"name":"alertmanager-main-1","namespace":"openshift-monitoring","containers":[{"name":"alertmanager"},{"name":"config-reloader"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-metric"},{"name":"prom-label-proxy"}]},{"name":"route-controller-manager-66ff45df48-29hjg","namespace":"openshift-route-controller-manager","containers":[{"name":"route-controller-manager"}]},{"name":"thanos-querier-df477fb5-22r5d","namespace":"openshift-monitoring","containers":[{"name":"thanos-query"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"prom-label-proxy"},{"name":"kube-rbac-proxy-rules"},{"name":"kube-rbac-proxy-metrics"}]},{"name":"ingress-operator-6b9d98f45d-f2w8b","namespace":"openshift-ingress-operator","containers":[{"name":"ingress-operator"},{"name":"kube-rbac-proxy"}]},{"name":"cluster-olm-operator-58d5d4b49d-zdvlf","namespace":"openshift-cluster-olm-operator","containers":[{"name":"cluster-olm-operator"}]},{"name":"olm-operator-7c9d879469-vr2z6","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"olm-operator"}]},{"name":"metal3-image-cache-djkjg","namespace":"openshift-machine-api","containers":[{"name":"metal3-httpd"}]},{"name":"csi-snapshot-controller-operator-596c645fb5-7cqld","namespace":"openshift-cluster-storage-operator","containers":[{"name":"csi-snapshot-controller-operator"}]},{"name":"prometheus-k8s-0","namespace":"openshift-monitoring","containers":[{"name":"prometheus"},{"name":"config-reloader"},{"name":"thanos-sidecar"},{"name":"kube-rbac-proxy-web"},{"name":"kube-rbac-proxy"},{"name":"kube-rbac-proxy-thanos"}]},{"name":"control-plane-machine-set-operator-5cf4849d9d-xxxqt","namespace":"openshift-machine-api","containers":[{"name":"control-plane-machine-set-operator"}]},{"name":"cluster-monitoring-operator-5b9b49c94f-b7zb4","namespace":"openshift-monitoring","containers":[{"name":"cluster-monitoring-operator"}]},{"name":"kube-controller-manager-operator-86b9c868d8-4zvkr","namespace":"openshift-kube-controller-manager-operator","containers":[{"name":"kube-controller-manager-operator"}]},{"name":"secondary-scheduler-7f66f86cb7-qf4pv","namespace":"openshift-numaresources","containers":[{"name":"secondary-scheduler","memory":[{"memory_type":"memory","size":1258291200,"topology":{"nodes":[{"ID":0}]}}]}]},{"name":"network-metrics-daemon-2qdn2","namespace":"openshift-multus","containers":[{"name":"network-metrics-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"ovnkube-control-plane-7d7c8cfc7-22trz","namespace":"openshift-ovn-kubernetes","containers":[{"name":"kube-rbac-proxy"},{"name":"ovnkube-cluster-manager"}]},{"name":"kube-apiserver-operator-95bdb85b9-q79ht","namespace":"openshift-kube-apiserver-operator","containers":[{"name":"kube-apiserver-operator"}]},{"name":"openshift-config-operator-597989c67-l5rnp","namespace":"openshift-config-operator","containers":[{"name":"openshift-config-operator"}]},{"name":"catalogd-controller-manager-884c99498-kxspf","namespace":"openshift-catalogd","containers":[{"name":"manager"}]},{"name":"kube-rbac-proxy-crio-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-machine-config-operator","containers":[{"name":"kube-rbac-proxy-crio"}]},{"name":"downloads-67fc445bcb-xsn5f","namespace":"openshift-console","containers":[{"name":"download-server"}]},{"name":"cluster-cloud-controller-manager-operator-6c796f8dff-69wd8","namespace":"openshift-cloud-controller-manager-operator","containers":[{"name":"cluster-cloud-controller-manager"},{"name":"config-sync-controllers"},{"name":"kube-rbac-proxy"}]},{"name":"etcd-operator-57d79df4bd-f24lp","namespace":"openshift-etcd-operator","containers":[{"name":"etcd-operator"}]},{"name":"openshift-kube-scheduler-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"kube-scheduler"},{"name":"kube-scheduler-cert-syncer"},{"name":"kube-scheduler-recovery-controller"}]},{"name":"network-check-target-gpk98","namespace":"openshift-network-diagnostics","containers":[{"name":"network-check-target-container"}]},{"name":"machine-config-daemon-c2nwm","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-daemon"},{"name":"kube-rbac-proxy"}]},{"name":"kube-apiserver-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"guard"}]},{"name":"cluster-image-registry-operator-974c6b758-d8xg9","namespace":"openshift-image-registry","containers":[{"name":"cluster-image-registry-operator"}]},{"name":"image-registry-796d5b566b-xmdrp","namespace":"openshift-image-registry","containers":[{"name":"registry"}]},{"name":"master-2kni-qe-52telco5granengrdu2redhatcom-debug-pmntx","namespace":"default","containers":[{"name":"container-00"}]},{"name":"openshift-controller-manager-operator-55f856587f-ppwhv","namespace":"openshift-controller-manager-operator","containers":[{"name":"openshift-controller-manager-operator"}]},{"name":"insights-operator-54f6b9ddc5-wxsks","namespace":"openshift-insights","containers":[{"name":"insights-operator"}]},{"name":"kube-storage-version-migrator-operator-6676b7fbc9-v2glx","namespace":"openshift-kube-storage-version-migrator-operator","containers":[{"name":"kube-storage-version-migrator-operator"}]},{"name":"ironic-proxy-rvfvx","namespace":"openshift-machine-api","containers":[{"name":"ironic-proxy"}]},{"name":"cluster-storage-operator-76cf476f8c-bl5px","namespace":"openshift-cluster-storage-operator","containers":[{"name":"cluster-storage-operator"}]},{"name":"multus-admission-controller-788794cf6f-vn4rq","namespace":"openshift-multus","containers":[{"name":"multus-admission-controller"},{"name":"kube-rbac-proxy"}]},{"name":"coredns-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"coredns"},{"name":"coredns-monitor"}]},{"name":"keepalived-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"keepalived"},{"name":"keepalived-monitor"}]},{"name":"package-server-manager-7b6bf69448-68ghr","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"kube-rbac-proxy"},{"name":"package-server-manager"}]},{"name":"catalog-operator-597fddd74-rxzkj","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"catalog-operator"}]},{"name":"machine-config-operator-6f4b598998-qjx4t","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-operator"},{"name":"kube-rbac-proxy"}]},{"name":"cluster-baremetal-operator-7d59475c8-x6njw","namespace":"openshift-machine-api","containers":[{"name":"cluster-baremetal-operator"},{"name":"baremetal-kube-rbac-proxy"}]},{"name":"etcd-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"etcdctl"},{"name":"etcd"},{"name":"etcd-metrics"},{"name":"etcd-readyz"},{"name":"etcd-rev"}]},{"name":"apiserver-575c85d5d4-n62p9","namespace":"openshift-oauth-apiserver","containers":[{"name":"oauth-apiserver"}]},{"name":"cluster-node-tuning-operator-78f47f8887-4sqlc","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"cluster-node-tuning-operator"}]},{"name":"multus-t8rp2","namespace":"openshift-multus","containers":[{"name":"kube-multus"}]},{"name":"dns-default-t8phv","namespace":"openshift-dns","containers":[{"name":"dns"},{"name":"kube-rbac-proxy"}]},{"name":"apiserver-b784b5f9-47fg6","namespace":"openshift-apiserver","containers":[{"name":"openshift-apiserver"},{"name":"openshift-apiserver-check-endpoints"}]},{"name":"openshift-kube-scheduler-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-scheduler","containers":[{"name":"guard"}]},{"name":"controller-manager-6b88548d64-5jc6z","namespace":"openshift-controller-manager","containers":[{"name":"controller-manager"}]},{"name":"ovnkube-node-pgxjg","namespace":"openshift-ovn-kubernetes","containers":[{"name":"ovn-controller"},{"name":"ovn-acl-logging"},{"name":"kube-rbac-proxy-node"},{"name":"kube-rbac-proxy-ovn-metrics"},{"name":"northd"},{"name":"nbdb"},{"name":"sbdb"},{"name":"ovnkube-controller"}]},{"name":"machine-config-server-8j64k","namespace":"openshift-machine-config-operator","containers":[{"name":"machine-config-server"}]},{"name":"node-resolver-j89fb","namespace":"openshift-dns","containers":[{"name":"dns-node-resolver"}]},{"name":"console-559d876ff5-6vwns","namespace":"openshift-console","containers":[{"name":"console"}]},{"name":"network-operator-566d6954c9-6bvmv","namespace":"openshift-network-operator","containers":[{"name":"network-operator"}]},{"name":"kube-controller-manager-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"kube-controller-manager"},{"name":"cluster-policy-controller"},{"name":"kube-controller-manager-cert-syncer"},{"name":"kube-controller-manager-recovery-controller"}]},{"name":"kube-apiserver-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-apiserver","containers":[{"name":"kube-apiserver"},{"name":"kube-apiserver-cert-syncer"},{"name":"kube-apiserver-cert-regeneration-controller"},{"name":"kube-apiserver-insecure-readyz"},{"name":"kube-apiserver-check-endpoints"}]},{"name":"kube-controller-manager-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kube-controller-manager","containers":[{"name":"guard"}]},{"name":"machine-approver-64f8f6b495-vb6v4","namespace":"openshift-cluster-machine-approver","containers":[{"name":"kube-rbac-proxy"},{"name":"machine-approver-controller"}]},{"name":"service-ca-operator-7d7d8684dc-82lf2","namespace":"openshift-service-ca-operator","containers":[{"name":"service-ca-operator"}]},{"name":"packageserver-56d59bdf5b-6zg9m","namespace":"openshift-operator-lifecycle-manager","containers":[{"name":"packageserver"}]},{"name":"dns-operator-596f54c5bb-npvz9","namespace":"openshift-dns-operator","containers":[{"name":"dns-operator"},{"name":"kube-rbac-proxy"}]},{"name":"insights-runtime-extractor-6jvb8","namespace":"openshift-insights","containers":[{"name":"kube-rbac-proxy"},{"name":"exporter"},{"name":"extractor"}]},{"name":"node-ca-wt5xl","namespace":"openshift-image-registry","containers":[{"name":"node-ca"}]},{"name":"csi-snapshot-controller-54d6b94fb5-x7qv6","namespace":"openshift-cluster-storage-operator","containers":[{"name":"snapshot-controller"}]},{"name":"authentication-operator-797c55ddf-jcs4r","namespace":"openshift-authentication-operator","containers":[{"name":"authentication-operator"}]},{"name":"networking-console-plugin-9f4fb68bd-cwpt5","namespace":"openshift-network-console","containers":[{"name":"networking-console-plugin"}]},{"name":"openshift-kube-scheduler-operator-558f48c74c-dc6t2","namespace":"openshift-kube-scheduler-operator","containers":[{"name":"kube-scheduler-operator-container"}]},{"name":"operator-controller-controller-manager-7967cc8894-7rbxq","namespace":"openshift-operator-controller","containers":[{"name":"manager"}]},{"name":"marketplace-operator-7df8b4fb6-m88pb","namespace":"openshift-marketplace","containers":[{"name":"marketplace-operator"}]},{"name":"iptables-alerter-f4xsk","namespace":"openshift-network-operator","containers":[{"name":"iptables-alerter"}]},{"name":"monitoring-plugin-588f66cf97-jmtsf","namespace":"openshift-monitoring","containers":[{"name":"monitoring-plugin"}]},{"name":"router-default-6658bcc57-jtqlm","namespace":"openshift-ingress","containers":[{"name":"router"}]},{"name":"numaresourcesoperator-master-wv8v9","namespace":"openshift-numaresources","containers":[{"name":"resource-topology-exporter"},{"name":"shared-pool-container"}]},{"name":"haproxy-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-kni-infra","containers":[{"name":"haproxy"},{"name":"haproxy-monitor"}]},{"name":"network-node-identity-mln2t","namespace":"openshift-network-node-identity","containers":[{"name":"webhook"},{"name":"approver"}]},{"name":"tuned-qgcfl","namespace":"openshift-cluster-node-tuning-operator","containers":[{"name":"tuned"}]},{"name":"node-exporter-752wt","namespace":"openshift-monitoring","containers":[{"name":"node-exporter"},{"name":"kube-rbac-proxy"}]},{"name":"etcd-guard-master-2.kni-qe-52.telco5gran.eng.rdu2.redhat.com","namespace":"openshift-etcd","containers":[{"name":"guard"}]},{"name":"openshift-apiserver-operator-9fffdd54f-bvjnq","namespace":"openshift-apiserver-operator","containers":[{"name":"openshift-apiserver-operator"}]},{"name":"cloud-credential-operator-67c9777895-44x2l","namespace":"openshift-cloud-credential-operator","containers":[{"name":"kube-rbac-proxy"},{"name":"cloud-credential-operator"}]},{"name":"prometheus-operator-admission-webhook-6c76dfdf85-q5mlf","namespace":"openshift-monitoring","containers":[{"name":"prometheus-operator-admission-webhook"}]},{"name":"multus-additional-cni-plugins-t2vwg","namespace":"openshift-multus","containers":[{"name":"kube-multus-additional-cni-plugins"}]},{"name":"ingress-canary-mjnmn","namespace":"openshift-ingress-canary","containers":[{"name":"serve-healthcheck-canary"}]}]}