While upgrading 3519 SNOs using the ZTP process from 4.12.16 to 4.13.0-rc.7 24 clusters failed to even begin upgrading because the clusterversion object reported 

Preconditions failed for payload loaded version="4.13.0-rc.7" image="quay.io/openshift-release-dev/ocp-release@sha256:aae5131ec824c301c11d0bf11d81b3996a222be8b49ce4716e9d464229a2f92b":
        Precondition "ClusterVersionUpgradeable" failed because of "AsExpected": Cluster
        operator cloud-controller-manager should not be upgraded between minor versions: '

Hub 4.12.14
SNOs upgrading from 4.12.16 to 4.13.0-rc.7
ACM version - 2.8.0-DOWNSTREAM-2023-04-30-18-44-29

24 out of 3519 or ~0.7%

1.
2.
3.

# oc --kubeconfig /root/hv-vm/kc/vm00488/kubeconfig get no
NAME      STATUS   ROLES                         AGE     VERSION
vm00488   Ready    control-plane,master,worker   2d20h   v1.25.8+37a9a08
# oc --kubeconfig /root/hv-vm/kc/vm00488/kubeconfig get clusterversion
NAME      VERSION   AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.12.16   True        False         2d      Cluster version is 4.12.16
# oc --kubeconfig /root/hv-vm/kc/vm00488/kubeconfig get clusterversion version -o yaml
apiVersion: config.openshift.io/v1
kind: ClusterVersion
metadata:
  creationTimestamp: "2023-05-16T23:18:55Z"
  generation: 6
  name: version
  resourceVersion: "647935"
  uid: c70fe530-450f-453d-9350-c0ac9e533d54
spec:
  channel: candidate-4.13
  clusterID: bc98b215-fd2e-47ba-acb7-42de6c5b4d4e
  desiredUpdate:
    version: 4.13.0-rc.7
  upstream: http://e27-h01-000-r650.rdu2.scalelab.redhat.com:8081/upgrade/upgrade-graph_candidate-4.13
status:
  availableUpdates:
  - channels:
    - candidate-4.13
    - candidate-4.14
    - fast-4.13
    - stable-4.13
    image: quay.io/openshift-release-dev/ocp-release@sha256:74b23ed4bbb593195a721373ed6693687a9b444c97065ce8ac653ba464375711
    url: https://access.redhat.com/errata/RHSA-2023:1326
    version: 4.13.0
  - channels:
    - candidate-4.13
    - candidate-4.14
    image: quay.io/openshift-release-dev/ocp-release@sha256:28e51227251e0b196fc51c62b57d9181d19b104ff63784d4bde7ba59260132fd
    version: 4.13.0-rc.8
  - channels:
    - candidate-4.13
    - candidate-4.14
    image: quay.io/openshift-release-dev/ocp-release@sha256:aae5131ec824c301c11d0bf11d81b3996a222be8b49ce4716e9d464229a2f92b
    version: 4.13.0-rc.7
  - channels:
    - candidate-4.12
    - candidate-4.13
    image: quay.io/openshift-release-dev/ocp-release@sha256:7ca5f8aa44bbc537c5a985a523d87365eab3f6e72abc50b7be4caae741e093f4
    url: https://access.redhat.com/errata/RHBA-2023:2699
    version: 4.12.17
  capabilities:
    enabledCapabilities:
    - CSISnapshot
    - Console
    - Insights
    - Storage
    - baremetal
    - marketplace
    - openshift-samples
    knownCapabilities:
    - CSISnapshot
    - Console
    - Insights
    - Storage
    - baremetal
    - marketplace
    - openshift-samples
  conditions:
  - lastTransitionTime: "2023-05-17T19:01:03Z"
    status: "True"
    type: RetrievedUpdates
  - lastTransitionTime: "2023-05-16T23:18:59Z"
    message: Capabilities match configured spec
    reason: AsExpected
    status: "False"
    type: ImplicitlyEnabledCapabilities
  - lastTransitionTime: "2023-05-18T21:15:01Z"
    message: 'Preconditions failed for payload loaded version="4.13.0-rc.7" image="quay.io/openshift-release-dev/ocp-release@sha256:aae5131ec824c301c11d0bf11d81b3996a222be8b49ce4716e9d464229a2f92b":
      Precondition "ClusterVersionUpgradeable" failed because of "AsExpected": Cluster
      operator cloud-controller-manager should not be upgraded between minor versions: '
    reason: PreconditionChecks
    status: "False"
    type: ReleaseAccepted
  - lastTransitionTime: "2023-05-16T23:52:46Z"
    message: Done applying 4.12.16
    status: "True"
    type: Available
  - lastTransitionTime: "2023-05-18T18:15:53Z"
    status: "False"
    type: Failing
  - lastTransitionTime: "2023-05-17T19:56:23Z"
    message: Cluster version is 4.12.16
    status: "False"
    type: Progressing
  - lastTransitionTime: "2023-05-18T21:08:26Z"
    message: 'Cluster operator cloud-controller-manager should not be upgraded between
      minor versions: '
    reason: AsExpected
    status: "False"
    type: Upgradeable
  desired:
    channels:
    - candidate-4.12
    - candidate-4.13
    - eus-4.12
    - fast-4.12
    - fast-4.13
    - stable-4.12
    image: quay.io/openshift-release-dev/ocp-release@sha256:5339b3c4686010dc42990e0addce5aa4fddd071d6d9504dffe08a4b5059f6f38
    url: https://access.redhat.com/errata/RHSA-2023:2110
    version: 4.12.16
  history:
  - completionTime: "2023-05-17T19:56:23Z"
    image: quay.io/openshift-release-dev/ocp-release@sha256:5339b3c4686010dc42990e0addce5aa4fddd071d6d9504dffe08a4b5059f6f38
    startedTime: "2023-05-17T19:03:27Z"
    state: Completed
    verified: true
    version: 4.12.16
  - completionTime: "2023-05-16T23:52:46Z"
    image: e27-h01-000-r650.rdu2.scalelab.redhat.com:5000/ocp4/openshift4@sha256:0398462b1c54758fe83619022a98bbe65f6deed71663b6665224d3ba36e43f03
    startedTime: "2023-05-16T23:18:59Z"
    state: Completed
    verified: false
    version: 4.12.11
  observedGeneration: 6
  versionHash: Ir-CNe7dSao=