time="2024-05-20T13:47:38Z" level=error msg="Namespace mysql-persistent, resource restore error: error preparing pods/mysql-persistent/mysql-7f77d47fd7-bvkwc: rpc error: code = Unknown desc = resource name may not be empty" logSource="pkg/controller/restore_controller.go:591" restore=velero/test-restore

 ./velero install --provider gcp --image velero/velero:main --plugins velero/velero-plugin-for-gcp:latest,quay.io/konveyor/openshift-velero-plugin:latest --bucket oadp8078177bwp --secret-file /home/fedora/workspace/cam/oadp-pipeline/credentials --use-node-agent --use-volume-snapshots=false --features=EnableCSI 

$ oc get pod -n mysql-persistent
NAME                     READY   STATUS    RESTARTS   AGE
mysql-7f77d47fd7-bvkwc   1/1     Running   0          66s

$ oc get backup test-backup -o yaml
apiVersion: velero.io/v1
kind: Backup
metadata:
  annotations:
    velero.io/resource-timeout: 10m0s
    velero.io/source-cluster-k8s-gitversion: v1.29.4+4a87b53
    velero.io/source-cluster-k8s-major-version: "1"
    velero.io/source-cluster-k8s-minor-version: "29"
  creationTimestamp: "2024-05-20T13:44:34Z"
  generation: 6
  labels:
    velero.io/storage-location: default
  name: test-backup
  namespace: velero
  resourceVersion: "57014"
  uid: 38cef544-ef28-4257-bb16-42fc46e95dcf
spec:
  csiSnapshotTimeout: 10m0s
  defaultVolumesToFsBackup: true
  includedNamespaces:
  - mysql-persistent
  itemOperationTimeout: 4h0m0s
  snapshotMoveData: false
  storageLocation: default
  ttl: 720h0m0s
status:
  completionTimestamp: "2024-05-20T13:44:54Z"
  expiration: "2024-06-19T13:44:34Z"
  formatVersion: 1.1.0
  hookStatus: {}
  phase: Completed
  progress:
    itemsBackedUp: 47
    totalItems: 47
  startTimestamp: "2024-05-20T13:44:34Z"
  version: 1

$ oc delete ns mysql-persistent
namespace "mysql-persistent" deleted

apiVersion: velero.io/v1
kind: Restore
metadata:
  name: test-restore
  namespace: velero
spec:
  backupName: test-backup

time="2024-05-20T13:47:38Z" level=error msg="Namespace mysql-persistent, resource restore error: error preparing pods/mysql-persistent/mysql-7f77d47fd7-bvkwc: rpc error: code = Unknown desc = resource name may not be empty" logSource="pkg/controller/restore_controller.go:591" restore=velero/test-restore

$ oc get pod -n mysql-persistent
NAME                     READY   STATUS    RESTARTS   AGE
mysql-7f77d47fd7-hkhkg   1/1     Running   0          13m

$ oc get pod -n mysql-persistent -o yaml
apiVersion: v1
items:
- apiVersion: v1
  kind: Pod
  metadata:
    annotations:
      k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.131.0.35/23"],"mac_address":"0a:58:0a:83:00:23","gateway_ips":["10.131.0.1"],"routes":[{"dest":"10.128.0.0/14","nextHop":"10.131.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.131.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.131.0.1"}],"ip_address":"10.131.0.35/23","gateway_ip":"10.131.0.1"}}'
      k8s.v1.cni.cncf.io/network-status: |-
        [{
            "name": "ovn-kubernetes",
            "interface": "eth0",
            "ips": [
                "10.131.0.35"
            ],
            "mac": "0a:58:0a:83:00:23",
            "default": true,
            "dns": {}
        }]
      openshift.io/scc: restricted-v2
      seccomp.security.alpha.kubernetes.io/pod: runtime/default
    creationTimestamp: "2024-05-20T13:47:37Z"
    generateName: mysql-7f77d47fd7-
    labels:
      app: mysql
      pod-template-hash: 7f77d47fd7
      testlabel: selectors
      testlabel2: foo
    name: mysql-7f77d47fd7-hkhkg
    namespace: mysql-persistent
    ownerReferences:
    - apiVersion: apps/v1
      blockOwnerDeletion: true
      controller: true
      kind: ReplicaSet
      name: mysql-7f77d47fd7
      uid: c5c7eb2e-bf43-4861-9ac0-e05e957100b3
    resourceVersion: "58173"
    uid: 5709dc6a-9b9b-4ee3-a4a8-43fc70b676dc
  spec:
    containers:
    - env:
      - name: MYSQL_USER
        valueFrom:
          secretKeyRef:
            key: database-user
            name: mysql
      - name: MYSQL_PASSWORD
        valueFrom:
          secretKeyRef:
            key: database-password
            name: mysql
      - name: MYSQL_ROOT_PASSWORD
        valueFrom:
          secretKeyRef:
            key: database-root-password
            name: mysql
      - name: MYSQL_DATABASE
        valueFrom:
          secretKeyRef:
            key: database-name
            name: mysql
      image: quay.io/migqe/rhel8-mariadb:10.5
      imagePullPolicy: IfNotPresent
      name: mysql
      ports:
      - containerPort: 3306
        name: mysql
        protocol: TCP
      resources:
        limits:
          memory: 512Mi
        requests:
          memory: 512Mi
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop:
          - ALL
        runAsNonRoot: true
        runAsUser: 1000750000
      terminationMessagePath: /dev/termination-log
      terminationMessagePolicy: File
      volumeMounts:
      - mountPath: /var/lib/mysql
        name: mysql-data
      - mountPath: /test-data
        name: mysql-data1
      - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
        name: kube-api-access-kf9pc
        readOnly: true
    dnsPolicy: ClusterFirst
    enableServiceLinks: true
    imagePullSecrets:
    - name: default-dockercfg-wgfxp
    nodeName: oadp-80781-77bwp-worker-c-w9m7g
    preemptionPolicy: PreemptLowerPriority
    priority: 0
    restartPolicy: Always
    schedulerName: default-scheduler
    securityContext:
      fsGroup: 1000750000
      seLinuxOptions:
        level: s0:c27,c24
      seccompProfile:
        type: RuntimeDefault
    serviceAccount: default
    serviceAccountName: default
    terminationGracePeriodSeconds: 30
    tolerations:
    - effect: NoExecute
      key: node.kubernetes.io/not-ready
      operator: Exists
      tolerationSeconds: 300
    - effect: NoExecute
      key: node.kubernetes.io/unreachable
      operator: Exists
      tolerationSeconds: 300
    - effect: NoSchedule
      key: node.kubernetes.io/memory-pressure
      operator: Exists
    volumes:
    - name: mysql-data
      persistentVolumeClaim:
        claimName: mysql
    - name: mysql-data1
      persistentVolumeClaim:
        claimName: mysql-1
    - name: kube-api-access-kf9pc
      projected:
        defaultMode: 420
        sources:
        - serviceAccountToken:
            expirationSeconds: 3607
            path: token
        - configMap:
            items:
            - key: ca.crt
              path: ca.crt
            name: kube-root-ca.crt
        - downwardAPI:
            items:
            - fieldRef:
                apiVersion: v1
                fieldPath: metadata.namespace
              path: namespace
        - configMap:
            items:
            - key: service-ca.crt
              path: service-ca.crt
            name: openshift-service-ca.crt
  status:
    conditions:
    - lastProbeTime: null
      lastTransitionTime: "2024-05-20T13:47:54Z"
      status: "True"
      type: PodReadyToStartContainers
    - lastProbeTime: null
      lastTransitionTime: "2024-05-20T13:47:41Z"
      status: "True"
      type: Initialized
    - lastProbeTime: null
      lastTransitionTime: "2024-05-20T13:47:54Z"
      status: "True"
      type: Ready
    - lastProbeTime: null
      lastTransitionTime: "2024-05-20T13:47:54Z"
      status: "True"
      type: ContainersReady
    - lastProbeTime: null
      lastTransitionTime: "2024-05-20T13:47:41Z"
      status: "True"
      type: PodScheduled
    containerStatuses:
    - containerID: cri-o://1c6fcec15451bbf2b3d5e99e0cc7ea7d4fdf283ae31201bd4006c1056fb735b1
      image: quay.io/migqe/rhel8-mariadb:10.5
      imageID: quay.io/migqe/rhel8-mariadb@sha256:c88f3c34c97f3739dcab17d015184c53834e668c32c072743e6e8308d22a0893
      lastState: {}
      name: mysql
      ready: true
      restartCount: 0
      started: true
      state:
        running:
          startedAt: "2024-05-20T13:47:53Z"
    hostIP: 10.0.128.4
    hostIPs:
    - ip: 10.0.128.4
    phase: Running
    podIP: 10.131.0.35
    podIPs:
    - ip: 10.131.0.35
    qosClass: Burstable
    startTime: "2024-05-20T13:47:41Z"

$ oc version
Client Version: 4.12.13
Kustomize Version: v4.5.7
Server Version: 4.16.0-0.nightly-2024-05-19-083311
Kubernetes Version: v1.29.4+4a87b53

$ oc get pod 
NAME                     READY   STATUS    RESTARTS   AGE
node-agent-5g4n2         1/1     Running   0          50m
node-agent-6hgff         1/1     Running   0          50m
node-agent-cpx8g         1/1     Running   0          50m
velero-9b55cf9ff-g224q   1/1     Running   0          16m

$ velero version
Client:
    Version: v1.12.1
    Git commit: 5c4fdfe147357ec7b908339f4516cd96d6b97c61
Server:
    Version: main
# WARNING: the client version does not match the server version. Please update server

$ velero describe restore test-restore --details
Name:         test-restore
Namespace:    velero
Labels:       <none>
Annotations:  <none>
Phase:                       PartiallyFailed (run 'velero restore logs test-restore' for more information)
Total items to be restored:  30
Items restored:              30
Started:    2024-05-20 19:17:34 +0530 IST
Completed:  2024-05-20 19:17:49 +0530 IST
Warnings:
  Velero:     <none>
  Cluster:  could not restore, CustomResourceDefinition "clusterserviceversions.operators.coreos.com" already exists. Warning: the in-cluster version is different than the backed-up version
  Namespaces:
    mysql-persistent:  could not restore, ConfigMap "kube-root-ca.crt" already exists. Warning: the in-cluster version is different than the backed-up version
                       could not restore, ConfigMap "openshift-service-ca.crt" already exists. Warning: the in-cluster version is different than the backed-up version
                       error getting resource from lister for rolebindings.authorization.openshift.io, mysql-persistent/system:deployers: rolebindings.authorization.openshift.io "system:deployers" not found
                       error getting resource from lister for rolebindings.authorization.openshift.io, mysql-persistent/system:image-builders: rolebindings.authorization.openshift.io "system:image-builders" not found
                       error getting resource from lister for rolebindings.authorization.openshift.io, mysql-persistent/system:image-pullers: rolebindings.authorization.openshift.io "system:image-pullers" not found
                       could not restore, RoleBinding "admin" already exists. Warning: the in-cluster version is different than the backed-up version
                       could not restore, RoleBinding "system:deployers" already exists. Warning: the in-cluster version is different than the backed-up version
                       could not restore, RoleBinding "system:image-builders" already exists. Warning: the in-cluster version is different than the backed-up version
                       could not restore, RoleBinding "system:image-pullers" already exists. Warning: the in-cluster version is different than the backed-up version
Errors:
  Velero:     <none>
  Cluster:    <none>
  Namespaces:
    mysql-persistent:  error preparing pods/mysql-persistent/mysql-7f77d47fd7-bvkwc: rpc error: code = Unknown desc = resource name may not be empty
Backup:  test-backup
Namespaces:
  Included:  all namespaces found in the backup
  Excluded:  <none>
Resources:
  Included:        *
  Excluded:        nodes, events, events.events.k8s.io, backups.velero.io, restores.velero.io, resticrepositories.velero.io, csinodes.storage.k8s.io, volumeattachments.storage.k8s.io, backuprepositories.velero.io
  Cluster-scoped:  auto
Namespace mappings:  <none>
Label selector:  <none>
Or label selector:  <none>
Restore PVs:  auto
Existing Resource Policy:   <none>
ItemOperationTimeout:       4h0m0s
Preserve Service NodePorts:  auto
Resource List:
  apiextensions.k8s.io/v1/CustomResourceDefinition:
    - clusterserviceversions.operators.coreos.com(failed)
  apps/v1/Deployment:
    - mysql-persistent/mysql(created)
  apps/v1/ReplicaSet:
    - mysql-persistent/mysql-7f77d47fd7(created)
  authorization.openshift.io/v1/RoleBinding:
    - mysql-persistent/admin(created)
    - mysql-persistent/system:deployers(failed)
    - mysql-persistent/system:image-builders(failed)
    - mysql-persistent/system:image-pullers(failed)
  discovery.k8s.io/v1/EndpointSlice:
    - mysql-persistent/mysql-wr6vf(created)
  operators.coreos.com/v1alpha1/ClusterServiceVersion:
    - mysql-persistent/volsync-product.v0.9.1(created)
  rbac.authorization.k8s.io/v1/RoleBinding:
    - mysql-persistent/admin(failed)
    - mysql-persistent/system:deployers(failed)
    - mysql-persistent/system:image-builders(failed)
    - mysql-persistent/system:image-pullers(failed)
  v1/ConfigMap:
    - mysql-persistent/kube-root-ca.crt(failed)
    - mysql-persistent/openshift-service-ca.crt(failed)
  v1/Endpoints:
    - mysql-persistent/mysql(created)
  v1/Namespace:
    - mysql-persistent(created)
  v1/PersistentVolume:
    - pvc-9f69e356-fe03-4578-8765-47706d60e844(skipped)
    - pvc-ffae3fff-a30c-441e-ad09-51cfa838c726(skipped)
  v1/PersistentVolumeClaim:
    - mysql-persistent/mysql(created)
    - mysql-persistent/mysql-1(created)
  v1/Pod:
    - mysql-persistent/mysql-7f77d47fd7-bvkwc(failed)
  v1/Secret:
    - mysql-persistent/builder-dockercfg-6cdkz(created)
    - mysql-persistent/default-dockercfg-d8prp(created)
    - mysql-persistent/deployer-dockercfg-pdtjl(created)
    - mysql-persistent/mysql(created)
  v1/Service:
    - mysql-persistent/mysql(created)
  v1/ServiceAccount:
    - mysql-persistent/builder(skipped)
    - mysql-persistent/default(skipped)
    - mysql-persistent/deployer(skipped)