Importing a VM from RHV using MTV fails with these kind of errors:

      error:
        phase: CopyDisks
        reasons:
        - 'Unable to process data: Failure cleaning up temporary scratch space: openfdat
          /scratch/lost+found: permission denied'


The importer pod uses a scratch PVC in Filesystem mode.

OCP 4.13.17
Openshift Virtualization 4.13.5
MTV 2.5.5
Containerized Data Importer v1.56.1-12-g16ff45d2

Reproduced only in customer environment.

1. StorageClass and StorageProfile:

--- apiVersion: cdi.kubevirt.io/v1beta1
kind: StorageProfile
metadata:
  creationTimestamp: "2024-04-03T10:42:24Z"
  generation: 3
  labels:
    app: containerized-data-importer
    app.kubernetes.io/component: storage
    app.kubernetes.io/managed-by: cdi-controller
    app.kubernetes.io/part-of: hyperconverged-cluster
    app.kubernetes.io/version: 4.13.5
    cdi.kubevirt.io: ""
  name: k8s-wok-iscsi-noreplica-halma
  ownerReferences:
  - apiVersion: cdi.kubevirt.io/v1beta1
    blockOwnerDeletion: true
    controller: true
    kind: CDI
    name: cdi-kubevirt-hyperconverged
    uid: e5578e77-68c0-4973-a87d-4c79d65d15fa
  resourceVersion: "209194592"
  uid: 4d2babc7-a762-4f58-a16c-1d57b6abbe7a
spec:
  claimPropertySets:
  - accessModes:
    - ReadWriteMany
    volumeMode: Block
status:
  claimPropertySets:
  - accessModes:
    - ReadWriteMany
    volumeMode: Block
  provisioner: csi.huawei.com
  storageClass: k8s-wok-iscsi-noreplica-halma

 2. PVC created for the importer scratch space is in filesystem mode:  oc describe pvc -n if-nvxsw mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-scratch
Name:          mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-scratch
Namespace:     if-nvxsw
StorageClass:  k8s-wok-iscsi-noreplica-halma
Status:        Bound
Volume:        pvc-e0999a4d-8031-4bef-be05-908b659ae89c
Labels:        app=containerized-data-importer
               app.kubernetes.io/component=storage
               app.kubernetes.io/managed-by=cdi-controller
               app.kubernetes.io/part-of=hyperconverged-cluster
               app.kubernetes.io/version=4.13.5
Annotations:   pv.kubernetes.io/bind-completed: yes
               pv.kubernetes.io/bound-by-controller: yes
               volume.beta.kubernetes.io/storage-provisioner: csi.huawei.com
               volume.kubernetes.io/storage-provisioner: csi.huawei.com
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      30Gi
Access Modes:  RWO
VolumeMode:    Filesystem
Used By:       importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-10815197-bd5e-405f-b156-ea906b799596
               importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-63073548-416d-4ed5-b54c-55f5a81c5b26
Events:        <none> oc get pvc -n if-nvxsw mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-scratch -o yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    pv.kubernetes.io/bind-completed: "yes"
    pv.kubernetes.io/bound-by-controller: "yes"
    volume.beta.kubernetes.io/storage-provisioner: csi.huawei.com
    volume.kubernetes.io/storage-provisioner: csi.huawei.com
  creationTimestamp: "2024-04-05T09:12:32Z"
  finalizers:
  - kubernetes.io/pvc-protection
  labels:
    app: containerized-data-importer
    app.kubernetes.io/component: storage
    app.kubernetes.io/managed-by: cdi-controller
    app.kubernetes.io/part-of: hyperconverged-cluster
    app.kubernetes.io/version: 4.13.5
  name: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-scratch
  namespace: if-nvxsw
  ownerReferences:
  - apiVersion: v1
    blockOwnerDeletion: true
    controller: true
    kind: Pod
    name: importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-63073548-416d-4ed5-b54c-55f5a81c5b26
    uid: 6fa4278b-80a8-4fa6-9c4c-ff0f7821b71f
  resourceVersion: "210382846"
  uid: e0999a4d-8031-4bef-be05-908b659ae89c
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: "32212254720"
  storageClassName: k8s-wok-iscsi-noreplica-halma
  volumeMode: Filesystem
  volumeName: pvc-e0999a4d-8031-4bef-be05-908b659ae89c
status:
  accessModes:
  - ReadWriteOnce
  capacity:
    storage: 30Gi
  phase: Bound  

- State of the importer POD oc describe pod -n if-nvxsw importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-63073548-416d-4ed5-b54c-55f5a81c5b26
Name:             importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-63073548-416d-4ed5-b54c-55f5a81c5b26
Namespace:        if-nvxsw
Priority:         0
Service Account:  default
Node:             skslc-k8s-wok-105.prod-01.k8s.lan/10.160.1.105
Start Time:       Fri, 05 Apr 2024 11:12:33 +0200
Labels:           app=containerized-data-importer
                  app.kubernetes.io/component=storage
                  app.kubernetes.io/managed-by=cdi-controller
                  app.kubernetes.io/part-of=hyperconverged-cluster
                  app.kubernetes.io/version=4.13.5
                  cdi.kubevirt.io=importer
                  prometheus.cdi.kubevirt.io=true
Annotations:      cdi.kubevirt.io/storage.createdByController: yes
                  k8s.ovn.org/pod-networks:
                    {"default":{"ip_addresses":["172.16.24.59/23"],"mac_address":"0a:58:ac:10:18:3b","gateway_ips":["172.16.24.1"],"ip_address":"172.16.24.59/...
                  k8s.v1.cni.cncf.io/network-status:
                    [{
                        "name": "ovn-kubernetes",
                        "interface": "eth0",
                        "ips": [
                            "172.16.24.59"
                        ],
                        "mac": "0a:58:ac:10:18:3b",
                        "default": true,
                        "dns": {}
                    }]
                  openshift.io/scc: privileged
                  seccomp.security.alpha.kubernetes.io/pod: runtime/default
                  sidecar.istio.io/inject: false
Status:           Succeeded
IP:               172.16.24.59
IPs:
  IP:           172.16.24.59
Controlled By:  PersistentVolumeClaim/mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh
Containers:
  importer:
    Container ID:  cri-o://f6d0579083646adba350c0ae7eef0121a99fd94947c6d2ad09be16bdc723107b
    Image:         registry.redhat.io/container-native-virtualization/virt-cdi-importer-rhel9@sha256:eb66bfabd67d153e1e5a9537b1fe9a94f8e7fc101ec10f86d70a76c0324da082
    Image ID:      registry.redhat.io/container-native-virtualization/virt-cdi-importer-rhel9@sha256:64fa194703392dd4048a9f508f7872e04864000b8728773504a09661a14684cb
    Port:          8443/TCP
    Host Port:     0/TCP
    Args:
      -v=1
    State:          Terminated
      Reason:       Completed
      Message:      Import Complete
      Exit Code:    0
      Started:      Fri, 05 Apr 2024 11:12:46 +0200
      Finished:     Fri, 05 Apr 2024 11:15:38 +0200
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     750m
      memory:  600M
    Requests:
      cpu:     100m
      memory:  60M
    Environment:
      IMPORTER_SOURCE:               imageio
      IMPORTER_ENDPOINT:             https://rhev-hq.afbag.com/ovirt-engine/api
      IMPORTER_CONTENTTYPE:          kubevirt
      IMPORTER_IMAGE_SIZE:           32212254720
      OWNER_UID:                     b45118e9-3419-4c55-829c-7899cb0e27ca
      FILESYSTEM_OVERHEAD:           0
      INSECURE_TLS:                  false
      IMPORTER_DISK_ID:              232d53f3-6814-40ab-a60b-e57e156978e4
      IMPORTER_UUID:
      IMPORTER_READY_FILE:
      IMPORTER_DONE_FILE:
      IMPORTER_BACKING_FILE:
      IMPORTER_THUMBPRINT:
      http_proxy:
      https_proxy:
      no_proxy:
      IMPORTER_CURRENT_CHECKPOINT:   63073548-416d-4ed5-b54c-55f5a81c5b26
      IMPORTER_PREVIOUS_CHECKPOINT:
      IMPORTER_FINAL_CHECKPOINT:     false
      PREALLOCATION:                 false
      IMPORTER_ACCESS_KEY_ID:        <set to the key 'accessKeyId' in secret 'mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-xxqqn'>  Optional: false
      IMPORTER_SECRET_KEY:           <set to the key 'secretKey' in secret 'mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-xxqqn'>    Optional: false
      IMPORTER_CERT_DIR:             /certs
      IMPORTER_PROXY_CERT_DIR:       /proxycerts/
    Mounts:
      /certs from cdi-cert-vol (rw)
      /proxycerts/ from cdi-proxy-cert-vol (rw)
      /scratch from cdi-scratch-vol (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-w7wr8 (ro)
    Devices:
      /dev/cdi-block-volume from cdi-data-vol
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  cdi-data-vol:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh
    ReadOnly:   false
  cdi-scratch-vol:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-scratch
    ReadOnly:   false
  cdi-cert-vol:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-p959s
    Optional:  false
  cdi-proxy-cert-vol:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      import-proxy-cm-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh
    Optional:  false
  kube-api-access-w7wr8:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
    ConfigMapName:           openshift-service-ca.crt
    ConfigMapOptional:       <nil>
QoS Class:                   Burstable
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:                      <none>
oc get pod -n if-nvxsw importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-63073548-416d-4ed5-b54c-55f5a81c5b26 -o yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    cdi.kubevirt.io/storage.createdByController: "yes"
    k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["172.16.24.59/23"],"mac_address":"0a:58:ac:10:18:3b","gateway_ips":["172.16.24.1"],"ip_address":"172.16.24.59/23","gateway_ip":"172.16.24.1"}}'
    k8s.v1.cni.cncf.io/network-status: |-
      [{
          "name": "ovn-kubernetes",
          "interface": "eth0",
          "ips": [
              "172.16.24.59"
          ],
          "mac": "0a:58:ac:10:18:3b",
          "default": true,
          "dns": {}
      }]
    openshift.io/scc: privileged
    seccomp.security.alpha.kubernetes.io/pod: runtime/default
    sidecar.istio.io/inject: "false"
  creationTimestamp: "2024-04-05T09:12:32Z"
  labels:
    app: containerized-data-importer
    app.kubernetes.io/component: storage
    app.kubernetes.io/managed-by: cdi-controller
    app.kubernetes.io/part-of: hyperconverged-cluster
    app.kubernetes.io/version: 4.13.5
    cdi.kubevirt.io: importer
    prometheus.cdi.kubevirt.io: "true"
  name: importer-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-checkpoint-63073548-416d-4ed5-b54c-55f5a81c5b26
  namespace: if-nvxsw
  ownerReferences:
  - apiVersion: v1
    blockOwnerDeletion: true
    controller: true
    kind: PersistentVolumeClaim
    name: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh
    uid: 5b80a799-60e1-4029-a149-e5e8a98fea14
  resourceVersion: "210386401"
  uid: 6fa4278b-80a8-4fa6-9c4c-ff0f7821b71f
spec:
  containers:
  - args:
    - -v=1
    env:
    - name: IMPORTER_SOURCE
      value: imageio
    - name: IMPORTER_ENDPOINT
      value: https://rhev-hq.afbag.com/ovirt-engine/api
    - name: IMPORTER_CONTENTTYPE
      value: kubevirt
    - name: IMPORTER_IMAGE_SIZE
      value: "32212254720"
    - name: OWNER_UID
      value: b45118e9-3419-4c55-829c-7899cb0e27ca
    - name: FILESYSTEM_OVERHEAD
      value: "0"
    - name: INSECURE_TLS
      value: "false"
    - name: IMPORTER_DISK_ID
      value: 232d53f3-6814-40ab-a60b-e57e156978e4
    - name: IMPORTER_UUID
    - name: IMPORTER_READY_FILE
    - name: IMPORTER_DONE_FILE
    - name: IMPORTER_BACKING_FILE
    - name: IMPORTER_THUMBPRINT
    - name: http_proxy
    - name: https_proxy
    - name: no_proxy
    - name: IMPORTER_CURRENT_CHECKPOINT
      value: 63073548-416d-4ed5-b54c-55f5a81c5b26
    - name: IMPORTER_PREVIOUS_CHECKPOINT
    - name: IMPORTER_FINAL_CHECKPOINT
      value: "false"
    - name: PREALLOCATION
      value: "false"
    - name: IMPORTER_ACCESS_KEY_ID
      valueFrom:
        secretKeyRef:
          key: accessKeyId
          name: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-xxqqn
    - name: IMPORTER_SECRET_KEY
      valueFrom:
        secretKeyRef:
          key: secretKey
          name: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-xxqqn
    - name: IMPORTER_CERT_DIR
      value: /certs
    - name: IMPORTER_PROXY_CERT_DIR
      value: /proxycerts/
    image: registry.redhat.io/container-native-virtualization/virt-cdi-importer-rhel9@sha256:eb66bfabd67d153e1e5a9537b1fe9a94f8e7fc101ec10f86d70a76c0324da082
    imagePullPolicy: IfNotPresent
    name: importer
    ports:
    - containerPort: 8443
      name: metrics
      protocol: TCP
    resources:
      limits:
        cpu: 750m
        memory: 600M
      requests:
        cpu: 100m
        memory: 60M
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 107
      seccompProfile:
        type: RuntimeDefault
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeDevices:
    - devicePath: /dev/cdi-block-volume
      name: cdi-data-vol
    volumeMounts:
    - mountPath: /scratch
      name: cdi-scratch-vol
    - mountPath: /certs
      name: cdi-cert-vol
    - mountPath: /proxycerts/
      name: cdi-proxy-cert-vol
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: kube-api-access-w7wr8
      readOnly: true
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  imagePullSecrets:
  - name: default-dockercfg-mcrnh
  nodeName: skslc-k8s-wok-105.prod-01.k8s.lan
  preemptionPolicy: PreemptLowerPriority
  priority: 0
  restartPolicy: OnFailure
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 107
    seLinuxOptions:
      level: s0:c30,c25
    seccompProfile:
      type: RuntimeDefault
    supplementalGroups:
    - 1000920000
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  - effect: NoSchedule
    key: node.kubernetes.io/memory-pressure
    operator: Exists
  volumes:
  - name: cdi-data-vol
    persistentVolumeClaim:
      claimName: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh
  - name: cdi-scratch-vol
    persistentVolumeClaim:
      claimName: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh-scratch
  - configMap:
      defaultMode: 420
      name: mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-p959s
    name: cdi-cert-vol
  - configMap:
      defaultMode: 420
      name: import-proxy-cm-mbu-test-20240404-5b3e93a7-d079-4424-ba69-a405492e5405-txwvh
    name: cdi-proxy-cert-vol
  - name: kube-api-access-w7wr8
    projected:
      defaultMode: 420
      sources:
      - serviceAccountToken:
          expirationSeconds: 3607
          path: token
      - configMap:
          items:
          - key: ca.crt
            path: ca.crt
          name: kube-root-ca.crt
      - downwardAPI:
          items:
          - fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
            path: namespace
      - configMap:
          items:
          - key: service-ca.crt
            path: service-ca.crt
          name: openshift-service-ca.crt
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2024-04-05T09:12:33Z"
    reason: PodCompleted
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: "2024-04-05T09:15:38Z"
    reason: PodCompleted
    status: "False"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: "2024-04-05T09:15:38Z"
    reason: PodCompleted
    status: "False"
    type: ContainersReady
  - lastProbeTime: null
    lastTransitionTime: "2024-04-05T09:12:33Z"
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: cri-o://f6d0579083646adba350c0ae7eef0121a99fd94947c6d2ad09be16bdc723107b
    image: registry.redhat.io/container-native-virtualization/virt-cdi-importer-rhel9@sha256:eb66bfabd67d153e1e5a9537b1fe9a94f8e7fc101ec10f86d70a76c0324da082
    imageID: registry.redhat.io/container-native-virtualization/virt-cdi-importer-rhel9@sha256:64fa194703392dd4048a9f508f7872e04864000b8728773504a09661a14684cb
    lastState: {}
    name: importer
    ready: false
    restartCount: 0
    started: false
    state:
      terminated:
        containerID: cri-o://f6d0579083646adba350c0ae7eef0121a99fd94947c6d2ad09be16bdc723107b
        exitCode: 0
        finishedAt: "2024-04-05T09:15:38Z"
        message: Import Complete
        reason: Completed
        startedAt: "2024-04-05T09:12:46Z"
  hostIP: 10.160.1.105
  phase: Succeeded
  podIP: 172.16.24.59
  podIPs:
  - ip: 172.16.24.59
  qosClass: Burstable
  startTime: "2024-04-05T09:12:33Z"
 

The import of the VM fails.

No error importing the disk.

I see this issue addressed by this PR:

https://github.com/kubevirt/containerized-data-importer/pull/2676

But I guess the fix is not included in 4.13. Can it be backported?