When upgrade ocp 4.12.64 to 4.13.0-0.nightly-2024-09-05-234759, image registry pod which is using pvc of thin sc as storge backend failed to be running with error `AttachVolume.Attach failed for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" : rpc error: code = Internal desc = failed to get VolumeID from volumeMigrationService for volumePath: "[vsanDatastore] 4e19ad64-9cc0-eeab-0e48-507c6f3b7674/ci-op-ridmlqn8-da2a0-q-pvc-cd241968-241b-49bb-a6ed-f64c08a07392.vmdk`.

    4.13

    always

    1.
omc get pvc -n openshift-image-registry -o yaml
apiVersion: v1
items:
- apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    annotations:
      pv.kubernetes.io/bind-completed: "yes"
      pv.kubernetes.io/bound-by-controller: "yes"
      volume.beta.kubernetes.io/storage-provisioner: kubernetes.io/vsphere-volume
      volume.kubernetes.io/storage-provisioner: kubernetes.io/vsphere-volume
    creationTimestamp: "2024-09-10T07:05:09Z"
    finalizers:
    - kubernetes.io/pvc-protection
    name: registry-pvc
    namespace: openshift-image-registry
    resourceVersion: "29599"
    uid: cd241968-241b-49bb-a6ed-f64c08a07392
  spec:
    accessModes:
    - ReadWriteOnce
    resources:
      requests:
        storage: 10Gi
    storageClassName: thin
    volumeMode: Filesystem
    volumeName: pvc-cd241968-241b-49bb-a6ed-f64c08a07392
  status:
    accessModes:
    - ReadWriteOnce
    capacity:
      storage: 10Gi
    phase: Bound
kind: List
 % omc get sc
NAME         PROVISIONER                    RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
mysc-62753   kubernetes.io/vsphere-volume   Delete          WaitForFirstConsumer   false                  3h
thin-csi     csi.vsphere.vmware.com         Delete          WaitForFirstConsumer   true                   4h
thin         kubernetes.io/vsphere-volume   Delete          Immediate              false                  4h
% omc get sc  thin -o yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
  creationTimestamp: "2024-09-10T06:44:26Z"
  name: thin
  resourceVersion: "4635"
  uid: 841bbde4-0bfd-42a2-a76d-d75ce6b0ffd9
parameters:
  diskformat: thin
provisioner: kubernetes.io/vsphere-volume
reclaimPolicy: Delete
volumeBindingMode: Immediate     

omc get pods -n openshift-image-registry -l docker-registry=default
NAME                              READY   STATUS              RESTARTS   AGE
image-registry-75466fb689-rnz7f   0/1     ContainerCreating   0          1h

omc get events -n openshift-image-registry  | grep -i error --color
1h46m       Warning   FailedAttachVolume       pod/image-registry-75466fb689-bt87t                     Multi-Attach error for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" Volume is already exclusively attached to one node and can't be attached to another
1h13m       Warning   ProbeError               pod/image-registry-75466fb689-bt87t                     Readiness probe error: Get "https://10.131.2.44:5000/healthz": dial tcp 10.131.2.44:5000: connect: connection refused...
1h10m       Warning   FailedAttachVolume       pod/image-registry-75466fb689-rnz7f                     AttachVolume.Attach failed for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" : rpc error: code = Internal desc = failed to get VolumeID from volumeMigrationService for volumePath: "[vsanDatastore] 4e19ad64-9cc0-eeab-0e48-507c6f3b7674/ci-op-ridmlqn8-da2a0-q-pvc-cd241968-241b-49bb-a6ed-f64c08a07392.vmdk"
6m          Warning   FailedAttachVolume       pod/image-registry-75466fb689-rnz7f                     AttachVolume.Attach failed for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" : rpc error: code = Internal desc = failed to get VolumeID from volumeMigrationService for volumePath: "[vsanDatastore] 4e19ad64-9cc0-eeab-0e48-507c6f3b7674/ci-op-ridmlqn8-da2a0-q-pvc-cd241968-241b-49bb-a6ed-f64c08a07392.vmdk"
2h54m       Warning   FailedAttachVolume       pod/image-registry-8479b778df-62xxj                     Multi-Attach error for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" Volume is already used by pod(s) image-registry-8479b778df-jcl5s
3h46m       Warning   FailedAttachVolume       pod/image-registry-8479b778df-j2pls                     Multi-Attach error for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" Volume is already exclusively attached to one node and can't be attached to another
3h1m        Warning   FailedAttachVolume       pod/image-registry-8479b778df-jcl5s                     Multi-Attach error for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" Volume is already used by pod(s) image-registry-8479b778df-vmssw
2h51m       Warning   FailedAttachVolume       pod/image-registry-8479b778df-lkrzc                     Multi-Attach error for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" Volume is already used by pod(s) image-registry-8479b778df-62xxj
3h26m       Warning   FailedAttachVolume       pod/image-registry-8479b778df-vmssw                     Multi-Attach error for volume "pvc-cd241968-241b-49bb-a6ed-f64c08a07392" Volume is already used by pod(s) image-registry-8479b778df-j2pls     

    Pods with pvc failed to be running

     Pods with pvc could be running

must gather log https://gcsweb-qe-private-deck-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/qe-private-deck/logs/periodic-ci-openshift-openshift-tests-private-release-4.13-amd64-nightly-4.13-upgrade-from-stable-4.12-vsphere-ipi-proxy-workers-rhel8-f28/1833389441058607104/artifacts/vsphere-ipi-proxy-workers-rhel8-f28/gather-must-gather/artifacts/must-gather.tar