time="2024-12-23T15:16:20Z" level=info msg="[imagecopy] Error copying image: writing blob: initiating layer upload to /v2/test-oadp-394/django-psql-persistent/blobs/uploads/ in registry-1.docker.io: received unexpected HTTP status: 500 Internal Server Error" backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins  

time="2024-12-23T17:26:22.660996771Z" level=error msg="response completed with error"
environment=development 
err.code=unknown 
err.detail="s3aws: InvalidArgument: Invalid argument.\n\tstatus code: 400, request id: , host id: "
err.message="unknown error" 
go.version="go1.22.9 (Red Hat 1.22.9-2.el9_5) X:strictfipsruntime" 
http.response.status=500
http.response.written=123 
service=registry 
vars.name=test-oadp-394/django-psql-persistent 
version=v3.0.0+unknown"

$ velero backup logs  django-manual-do-backup-images |grep copy
time="2024-12-23T15:14:31Z" level=info msg="[imagecopy] Copying tag: \"latest\"" backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins
time="2024-12-23T15:14:31Z" level=info msg="[imagecopy] copying image to BSL registry: udistribution-s3-62b1b046-a9c3-4d1f-98eb-3b81430f3dc7" backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins
time="2024-12-23T15:14:31Z" level=info msg="[imagecopy] copying from: docker://image-registry.openshift-image-registry.svc:5000/test-oadp-394/django-psql-persistent@sha256:b88abafbe0a30a93982b3332e7478b1fcda9470714a9ed6ac0651f03aa667be9" backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins
time="2024-12-23T15:14:31Z" level=info msg="[imagecopy] copying to: udistribution-s3-62b1b046-a9c3-4d1f-98eb-3b81430f3dc7://test-oadp-394/django-psql-persistent:latest" backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins
time="2024-12-23T15:14:31Z" level=info msg="copying image: docker://image-registry.openshift-image-registry.svc:5000/test-oadp-394/django-psql-persistent@sha256:b88abafbe0a30a93982b3332e7478b1fcda9470714a9ed6ac0651f03aa667be9; will attempt up to 7 times..." backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins
time="2024-12-23T15:16:20Z" level=info msg="[imagecopy] Error copying image: writing blob: initiating layer upload to /v2/test-oadp-394/django-psql-persistent/blobs/uploads/ in registry-1.docker.io: received unexpected HTTP status: 500 Internal Server Error" backup=openshift-adp/django-manual-do-backup-images cmd=/plugins/velero-plugins logSource="/remote-source/deps/gomod/pkg/mod/github.com/bombsimon/logrusr/v3@v3.0.0/logrusr.go:108" pluginName=velero-plugins 

apiVersion: oadp.openshift.io/v1alpha1
kind: DataProtectionApplication
metadata:
  creationTimestamp: '2024-12-23T13:47:07Z'
  generation: 7
  managedFields:
    - apiVersion: oadp.openshift.io/v1alpha1
      fieldsType: FieldsV1
      fieldsV1:
        'f:spec':
          .: {}
          'f:configuration':
            .: {}
            'f:velero': {}
          'f:podDnsConfig': {}
          'f:snapshotLocations': {}
      manager: __debug_bin1972371061
      operation: Update
      time: '2024-12-23T13:47:07Z'
    - apiVersion: oadp.openshift.io/v1alpha1
      fieldsType: FieldsV1
      fieldsV1:
        'f:status':
          .: {}
          'f:conditions': {}
      manager: manager
      operation: Update
      subresource: status
      time: '2024-12-23T13:47:08Z'
    - apiVersion: oadp.openshift.io/v1alpha1
      fieldsType: FieldsV1
      fieldsV1:
        'f:spec':
          'f:backupLocations': {}
          'f:configuration':
            'f:nodeAgent':
              .: {}
              'f:enable': {}
              'f:podConfig':
                .: {}
                'f:resourceAllocations': {}
              'f:uploaderType': {}
            'f:velero':
              'f:defaultPlugins': {}
      manager: __debug_bin3595348414
      operation: Update
      time: '2024-12-23T14:11:30Z'
    - apiVersion: oadp.openshift.io/v1alpha1
      fieldsType: FieldsV1
      fieldsV1:
        'f:spec':
          'f:backupImages': {}
      manager: Mozilla
      operation: Update
      time: '2024-12-23T15:10:33Z'
  name: ts-dpa
  namespace: openshift-adp
  resourceVersion: '202346'
  uid: 484bebf8-7d81-4970-967a-32f83037aa63
spec:
  backupImages: true
  backupLocations:
    - velero:
        config:
          insecureSkipTLSVerify: 'true'
          profile: default
          region: us-west1
          s3ForcePathStyle: 'true'
          s3Url: 'https://storage.googleapis.com'
        credential:
          key: cloud
          name: cloud-credentials
        default: true
        objectStorage:
          bucket: oadp105641ss42c
          prefix: velero-e2e-cde7c4ef-c137-11ef-ac9a-0c9a3c9340c2
        provider: aws
  configuration:
    nodeAgent:
      enable: true
      podConfig:
        resourceAllocations: {}
      uploaderType: kopia
    velero:
      defaultPlugins:
        - openshift
        - legacy-aws
        - kubevirt
  podDnsConfig: {}
  snapshotLocations: []
status:
  conditions:
    - lastTransitionTime: '2024-12-23T13:47:08Z'
      message: Reconcile complete
      reason: Complete
      status: 'True'
      type: Reconciled 

 $ velero backup describe  django-manual-do-backup-images
Name:         django-manual-do-backup-images
Namespace:    openshift-adp
Labels:       velero.io/storage-location=ts-dpa-1
Annotations:  velero.io/resource-timeout=10m0s
              velero.io/source-cluster-k8s-gitversion=v1.30.7
              velero.io/source-cluster-k8s-major-version=1
              velero.io/source-cluster-k8s-minor-version=30Phase:  PartiallyFailed (run `velero backup logs django-manual-do-backup-images` for more information)
Warnings:
  Velero:     <none>
  Cluster:    <none>
  Namespaces:
    test-oadp-394:   resource: /pods name: /django-psql-persistent-1-build message: /Skip pod volume buildworkdir error: /pod is not in the expected status, name=django-psql-persistent-1-build, namespace=test-oadp-394, phase=Succeeded: pod is not running
                     resource: /pods name: /django-psql-persistent-1-build message: /Skip pod volume container-storage-root error: /pod is not in the expected status, name=django-psql-persistent-1-build, namespace=test-oadp-394, phase=Succeeded: pod is not running
                     resource: /pods name: /django-psql-persistent-1-build message: /Skip pod volume container-storage-run error: /pod is not in the expected status, name=django-psql-persistent-1-build, namespace=test-oadp-394, phase=Succeeded: pod is not running
                     resource: /pods name: /django-psql-persistent-1-build message: /Skip pod volume build-blob-cache error: /pod is not in the expected status, name=django-psql-persistent-1-build, namespace=test-oadp-394, phase=Succeeded: pod is not runningErrors:
  Velero:    name: /django-psql-persistent message: /Error backing up item error: /error executing custom action (groupResource=imagestreams.image.openshift.io, namespace=test-oadp-394, name=django-psql-persistent): rpc error: code = Unknown desc = writing blob: initiating layer upload to /v2/test-oadp-394/django-psql-persistent/blobs/uploads/ in registry-1.docker.io: received unexpected HTTP status: 500 Internal Server Error
  Cluster:    <none>
  Namespaces: <none>