oc get ibu upgrade -o yaml                  
                                                                                                                                                                       
apiVersion: lca.openshift.io/v1alpha1
kind: ImageBasedUpgrade
metadata:
  creationTimestamp: "2024-04-10T04:59:06Z"
  generation: 12
  name: upgrade
  resourceVersion: "125014"
  uid: c2f8a588-cf90-40a6-b4b8-c06cac9b7f66
spec:
  additionalImages:
    name: ""
    namespace: ""
  autoRollbackOnFailure:
    disabledForPostRebootConfig: true
    disabledForUpgradeCompletion: true
    disabledInitMonitor: true
    initMonitorTimeoutSeconds: 1801
  extraManifests:
  - name: lca-catalog-source-configmap
    namespace: openshift-marketplace
  oadpContent:
  - name: oadp-cm
    namespace: openshift-adp
  seedImageRef:
    image: quay.io/ocp-edge-qe/ib-seedimage-public:ci-4.15kmm-no-rollback
    version: 4.15.6
  stage: Prep
status:
  conditions:
  - lastTransitionTime: "2024-04-10T13:35:41Z"
    message: In progress
    observedGeneration: 12
    reason: InProgress
    status: "False"
    type: Idle
  - lastTransitionTime: "2024-04-10T13:35:41Z"
    message: |-
      Waiting for system to stabilize before Prep stage can continue: one or more health checks failed
        - one or more ClusterServiceVersions not yet ready: kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2
    observedGeneration: 12
    reason: InProgress
    status: "True"
    type: PrepInProgress
  observedGeneration: 12
  validNextStages:
  - Idle

2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   csv not ready: kernel-module-management.v2.0.2
2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   csv health check failure        {"error": "one or more ClusterServiceVersions not yet ready: kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2"}
2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   Cluster version is ready
2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   All CSRs are approved
2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   One or more health checks failed
2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   Waiting for system to stabilize before Prep stage can continue: one or more health checks failed                                                                             
  - one or more ClusterServiceVersions not yet ready: kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2,
kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2, kernel-module-management.v2.0.2                                                                                          
2024-04-10T14:16:47Z    INFO    controllers.ImageBasedUpgrade   Finish reconciling IBU  {"name": {"name":"upgrade"}, "requeueAfter": 20}     

 oc get csv -A                                                                                                                                                                                                      
NAMESPACE                                          NAME                              DISPLAY                    VERSION          REPLACES                          PHASE                                                                      
assisted-installer                                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
default                                            kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
ibu-workload-app                                   kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
kube-node-lease                                    kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
kube-public                                        kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
kube-system                                        kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
open-cluster-management-agent-addon                kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
open-cluster-management-agent                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-adp                                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded 
openshift-adp                                      oadp-operator.v1.3.1              OADP Operator              1.3.1            oadp-operator.v1.3.0              Succeeded                                                                  
openshift-apiserver-operator                       kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-apiserver                                kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-authentication-operator                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-authentication                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cloud-controller-manager-operator        kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cloud-controller-manager                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cloud-credential-operator                kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cloud-network-config-controller          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cluster-csi-drivers                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cluster-machine-approver                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cluster-node-tuning-operator             kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cluster-samples-operator                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cluster-storage-operator                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-cluster-version                          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-config-managed                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded                                                                  
openshift-config-operator                          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-config                                   kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-console-operator                         kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-console-user-settings                    kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-console                                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-controller-manager-operator              kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-controller-manager                       kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-dns-operator                             kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-dns                                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-etcd-operator                            kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-etcd                                     kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-host-network                             kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-image-registry                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-infra                                    kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-ingress-canary                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-ingress-operator                         kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-ingress                                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-insights                                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kmm                                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kni-infra                                kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-apiserver-operator                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-apiserver                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-controller-manager-operator         kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-controller-manager                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-scheduler-operator                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-scheduler                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-storage-version-migrator-operator   kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-kube-storage-version-migrator            kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-lifecycle-agent                          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded


openshift-lifecycle-agent                          lifecycle-agent.v4.16.0           Lifecycle Agent            4.16.0                                             Succeeded
openshift-machine-api                              kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-machine-config-operator                  kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-marketplace                              kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-monitoring                               kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-multus                                   kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-network-diagnostics                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-network-node-identity                    kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-network-operator                         kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-node                                     kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-nutanix-infra                            kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-oauth-apiserver                          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-openstack-infra                          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-operator-lifecycle-manager               kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-operator-lifecycle-manager               packageserver                     Package Server             0.0.1-snapshot                                     Succeeded
openshift-operators                                kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-ovirt-infra                              kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-ovn-kubernetes                           kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-route-controller-manager                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-service-ca-operator                      kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-service-ca                               kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-user-workload-monitoring                 kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift-vsphere-infra                            kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
openshift                                          kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded
simple-kmod                                        kernel-module-management.v2.0.2   Kernel Module Management   2.0.2            kernel-module-management.v2.0.1   Succeeded