Uploaded image for project: 'OpenShift API for Data Protection'
  1. OpenShift API for Data Protection
  2. OADP-634

create backup using restic plugin fails ( performance cases on baremetal )

XMLWordPrintable

    • True
    • Hide

      None

      Show
      None
    • False
    • ToDo
    • 0
    • 0
    • Very Likely
    • 0
    • None
    • Unset
    • Unknown
    • No

      Description of problem:

      running a backup using restic as a plugin fails 

      on cloud20 with ocp 4.11 and oadp version 1.1 .0 iib:276825  

      [root@f03-h20-000-r640 mpqe-scale-scripts]# velero backup logs perf-datagen-case2.1.1-restic-ocs --insecure-skip-tls-verify |grep -i error
      time="2022-07-21T08:29:04Z" level=info msg="1 errors encountered backup up item" backup=openshift-adp/perf-datagen-case2.1.1-restic-ocs logSource="pkg/backup/backup.go:413" name=perf-datagen-case1-5f9d4f7758-955kh
      time="2022-07-21T08:29:04Z" level=error msg="Error backing up item" backup=openshift-adp/perf-datagen-case2.1.1-restic-ocs error="pod volume backup failed: running Restic backup, stderr=: signal: killed" error.file="/remote-source/src/github.com/vmware-tanzu/velero/pkg/restic/backupper.go:199" error.function="github.com/vmware-tanzu/velero/pkg/restic.(*backupper).BackupPodVolumes" logSource="pkg/backup/backup.go:417" name=perf-datagen-case1-5f9d4f7758-955kh
      

      tested  using  both sc  sources   nfs & ocs  

       

      podvolumebackups :

      Namespace:    openshift-adp
      Labels:       velero.io/backup-name=perf-datagen-case2.1.1-restic-ocs5
                    velero.io/backup-uid=c0a246dd-a91d-4f20-86c3-9a4841a02ad7
                    velero.io/pvc-uid=bf15c123-4cdf-4a79-b3a4-a1574a6c8ab7
      Annotations:  velero.io/pvc-name: perf-datagen-case1
      API Version:  velero.io/v1
      Kind:         PodVolumeBackup
      Metadata:
        Creation Timestamp:  2022-07-21T12:09:33Z
        Generate Name:       perf-datagen-case2.1.1-restic-ocs5-
        Generation:          4
        Managed Fields:
          API Version:  velero.io/v1
          Fields Type:  FieldsV1
          fieldsV1:
            f:metadata:
              f:annotations:
                .:
                f:velero.io/pvc-name:
              f:generateName:
              f:labels:
                .:
                f:velero.io/backup-name:
                f:velero.io/backup-uid:
                f:velero.io/pvc-uid:
              f:ownerReferences:
                .:
                k:{"uid":"c0a246dd-a91d-4f20-86c3-9a4841a02ad7"}:
            f:spec:
              .:
              f:backupStorageLocation:
              f:node:
              f:pod:
                .:
                f:kind:
                f:name:
                f:namespace:
                f:uid:
              f:repoIdentifier:
              f:tags:
                .:
                f:backup:
                f:backup-uid:
                f:ns:
                f:pod:
                f:pod-uid:
                f:pvc-uid:
                f:volume:
              f:volume:
            f:status:
              .:
              f:progress:
          Manager:      velero-server
          Operation:    Update
          Time:         2022-07-21T12:09:33Z
          API Version:  velero.io/v1
          Fields Type:  FieldsV1
          fieldsV1:
            f:status:
              f:completionTimestamp:
              f:message:
              f:phase:
              f:progress:
                f:bytesDone:
                f:totalBytes:
              f:startTimestamp:
          Manager:    restic-server
          Operation:  Update
          Time:       2022-07-21T12:09:43Z
        Owner References:
          API Version:     velero.io/v1
          Controller:      true
          Kind:            Backup
          Name:            perf-datagen-case2.1.1-restic-ocs5
          UID:             c0a246dd-a91d-4f20-86c3-9a4841a02ad7
        Resource Version:  122346654
        UID:               c3972342-d5f0-4266-9f94-5ecdb96d7d91
      Spec:
        Backup Storage Location:  example-velero-1
        Node:                     worker002-r640
        Pod:
          Kind:           Pod
          Name:           perf-datagen-case1-5f9d4f7758-955kh
          Namespace:      perf-datagen-case1
          UID:            c961ecf7-dbc7-4a8c-ae4b-4e28948f6bf8
        Repo Identifier:  s3:https://s3-openshift-storage.apps.vlan603.rdu2.scalelab.redhat.com/oadp-bucket/velero/restic/perf-datagen-case1
        Tags:
          Backup:        perf-datagen-case2.1.1-restic-ocs5
          Backup - UID:  c0a246dd-a91d-4f20-86c3-9a4841a02ad7
          Ns:            perf-datagen-case1
          Pod:           perf-datagen-case1-5f9d4f7758-955kh
          Pod - UID:     c961ecf7-dbc7-4a8c-ae4b-4e28948f6bf8
          Pvc - UID:     bf15c123-4cdf-4a79-b3a4-a1574a6c8ab7
          Volume:        vol-0
        Volume:          vol-0
      Status:
        Completion Timestamp:  2022-07-21T12:09:38Z
        Message:               running Restic backup, stderr=: signal: killed
        Phase:                 Failed
        Progress:
          Bytes Done:     2500700
          Total Bytes:    14524600
        Start Timestamp:  2022-07-21T12:09:33Z
      Events:             <none>
      
       

      backup CR 

      [root@f03-h20-000-r640 mpqe-scale-scripts]# oc describe backups/perf-datagen-case2.1.1-restic-ocs1 -nopenshift-adp
      Name:         perf-datagen-case2.1.1-restic-ocs1
      Namespace:    openshift-adp
      Labels:       velero.io/storage-location=example-velero-1
      Annotations:  velero.io/source-cluster-k8s-gitversion: v1.24.0+9ddc8b1
                    velero.io/source-cluster-k8s-major-version: 1
                    velero.io/source-cluster-k8s-minor-version: 24
      API Version:  velero.io/v1
      Kind:         Backup
      Metadata:
        Creation Timestamp:  2022-07-21T08:44:37Z
        Generation:          5
        Managed Fields:
          API Version:  velero.io/v1
          Fields Type:  FieldsV1
          fieldsV1:
            f:spec:
              .:
              f:defaultVolumesToRestic:
              f:hooks:
              f:includedNamespaces:
              f:metadata:
              f:snapshotVolumes:
            f:status:
          Manager:      velero
          Operation:    Update
          Time:         2022-07-21T08:44:37Z
          API Version:  velero.io/v1
          Fields Type:  FieldsV1
          fieldsV1:
            f:metadata:
              f:annotations:
                .:
                f:velero.io/source-cluster-k8s-gitversion:
                f:velero.io/source-cluster-k8s-major-version:
                f:velero.io/source-cluster-k8s-minor-version:
              f:labels:
                .:
                f:velero.io/storage-location:
            f:spec:
              f:storageLocation:
              f:ttl:
            f:status:
              f:completionTimestamp:
              f:errors:
              f:expiration:
              f:formatVersion:
              f:phase:
              f:progress:
                .:
                f:itemsBackedUp:
                f:totalItems:
              f:startTimestamp:
              f:version:
          Manager:         velero-server
          Operation:       Update
          Time:            2022-07-21T08:45:40Z
        Resource Version:  121244847
        UID:               486154c9-6694-48a4-84c3-1768b6e6ea53
      Spec:
        Default Volumes To Restic:  true
        Hooks:
        Included Namespaces:
          perf-datagen-case1
        Metadata:
        Snapshot Volumes:  false
        Storage Location:  example-velero-1
        Ttl:               720h0m0s
      Status:
        Completion Timestamp:  2022-07-21T08:45:40Z
        Errors:                1
        Expiration:            2022-08-20T08:44:37Z
        Format Version:        1.1.0
        Phase:                 PartiallyFailed
        Progress:
          Items Backed Up:  26
          Total Items:      26
        Start Timestamp:    2022-07-21T08:44:42Z
        Version:            1
      Events:               <none>
       

       

      DPA: 

      [root@f03-h20-000-r640 mpqe-scale-scripts]# oc get   dpa/example-velero  -nopenshift-adp -oyaml
      apiVersion: oadp.openshift.io/v1alpha1
      kind: DataProtectionApplication
      metadata:
        annotations:
          kubectl.kubernetes.io/last-applied-configuration: |
            {"apiVersion":"oadp.openshift.io/v1alpha1","kind":"DataProtectionApplication","metadata":{"annotations":{},"name":"example-velero","namespace":"openshift-adp"},"spec":{"backupLocations":[{"name":"default","velero":{"config":{"insecureSkipTLSVerify":"true","profile":"noobaa","region":"noobaa","s3ForcePathStyle":"true","s3Url":"https://s3-openshift-storage.apps.vlan603.rdu2.scalelab.redhat.com"},"credential":{"key":"cloud","name":"cloud-credentials"},"default":true,"objectStorage":{"bucket":"oadp-bucket","prefix":"velero"},"provider":"aws"}}],"configuration":{"velero":{"defaultPlugins":["openshift","aws","csi"],"featureFlags":["EnableCSI"]}}}}
        creationTimestamp: "2022-07-20T09:41:24Z"
        generation: 6
        name: example-velero
        namespace: openshift-adp
        resourceVersion: "122796589"
        uid: c88bcbb3-3867-46ed-9b2f-f70a6a1f60c3
      spec:
        backupLocations:
        - velero:
            config:
              insecureSkipTLSVerify: "true"
              profile: noobaa
              region: noobaa
              s3ForcePathStyle: "true"
              s3Url: https://s3-openshift-storage.apps.vlan603.rdu2.scalelab.redhat.com
            credential:
              key: cloud
              name: cloud-credentials
            default: true
            objectStorage:
              bucket: oadp-bucket
              prefix: velero
            provider: aws
        configuration:
          restic:
            enable: true
          velero:
            defaultPlugins:
            - openshift
            - aws
            - csi
      status:
        conditions:
        - lastTransitionTime: "2022-07-20T09:41:25Z"
          message: Reconcile complete
          reason: Complete
          status: "True"
          type: Reconciled
       

       

       

      Version-Release number of selected component (if applicable):

      ocp 4.11 

      oadp 1.1.0 

      iib:276825

       

              sseago Scott Seago
              tzahia Tzahi Ashkenazi
              Tzahi Ashkenazi Tzahi Ashkenazi
              Votes:
              0 Vote for this issue
              Watchers:
              4 Start watching this issue

                Created:
                Updated:
                Resolved: