# Prerequisite
## Cluster with 4 worker instances

## OADP installed in the openshift-adp namespace
# To install OADP from master:
#    a) Clone the OADP project
#    b) Ensure you have admin rights and you are logged in to your OpenShift cluster
#    c) $ make build-deploy && make deploy-olm

## Installing CephFS with OpenShift Data Foundation
 1. Install OpenShift Data Foundation from OperatorHub
    https://access.redhat.com/documentation/en-us/red_hat_openshift_data_foundation/4.12/html-single/deploying_and_managing_openshift_data_foundation_using_red_hat_openstack_platform/index#installing-openshift-data-foundation-operator-using-the-operator-hub_internal-osp

    # NOTE: During install ensure that the Enable option is selected for the Console plugin

 2. Create Storage System
    https://access.redhat.com/documentation/en-us/red_hat_openshift_data_foundation/4.12/html-single/deploying_and_managing_openshift_data_foundation_using_red_hat_openstack_platform/index#creating-an-openshift-data-foundation-service_internal-osp

    # NOTE: Select all 4 nodes, the capacity can be set to minimum, no need to enable encryption

 3. For the CephFS and CephRBD follow the instructions to annotate relevant:
    a) VolumeSnapshotClass: https://github.com/openshift/oadp-operator/blob/master/docs/examples/datamover_advanced_voloptions.md#cephfs-volumesnapshotclass-and-storageclass
    b) StorageClass: https://github.com/openshift/oadp-operator/blob/master/docs/examples/datamover_advanced_voloptions.md#cephfs-volumesnapshotclass-and-storageclass

 4. Install the latest VolSync from the Marketplace

 5. Create secret which will be used later by the DPA:

  $ export OADP_NS="openshift-adp"
  $ export SECRET_NAME="my-restic-secret"
  $ cat <<EOF | oc apply -n $OADP_NS -f -
apiVersion: v1
kind: Secret
metadata:
  name: ${SECRET_NAME}
type: Opaque
stringData:
  RESTIC_PASSWORD: my-secure-restic-password
EOF


6. Download and install noobaa from MCG (that includes noobaa CLI)
   https://access.redhat.com/documentation/en-us/red_hat_openshift_container_storage/4.8/html/managing_hybrid_and_multicloud_resources/accessing-the-multicloud-object-gateway-with-your-applications_rhocs
   https://access.redhat.com/downloads/content/547/ver=4/rhel---8/4/x86_64/packages

7. Deploy noobaa in the adp namespace
  $ export OADP_NS=openshift-adp
  $ /usr/bin/noobaa install --namespace "$OADP_NS"

8. Confirm deployment:
  $  describe noobaa -n "$OADP_NS"

9. Get credentials:
 $ AWS_ACCESS_KEY=$(oc get secret noobaa-admin -n "$OADP_NS" -o json | jq '.data|map_values(@base64d)|.AWS_ACCESS_KEY_ID')
 $ AWS_SECRET_ACCESS_KEY=$(oc get secret noobaa-admin -n "$OADP_NS" -o json | jq '.data|map_values(@base64d)|.AWS_SECRET_ACCESS_KEY')

10. Create secret 'mycloud':
 $ TMP_DIR="$(mktemp -d)"
 $ cat <<EOF > "$TMP_DIR/credentials-velero.yaml"
[default]
aws_access_key_id=${AWS_ACCESS_KEY}
aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
EOF

 $ oc create secret generic cloud-credentials -n "$OADP_NS" --from-file mycloud="${TMP_DIR}/credentials-velero.yaml"

11. Create new namespace to be backed up, with many PVCs

NAMESPACE_PVCS="test"

NO_PVCS=30
PVC_LABEL_NAME="oadp-test"
PVC_LABEL_VALUE="tobedeleted"
PVC_PREFIX_NAME="oadp-test"
PVC_MIN_REQUESTED_SIZE="15Mi"
storageClassName="ocs-storagecluster-cephfs"

oc create ns "${NAMESPACE_PVCS}"
oc create sa oadp-test-sa -n "${NAMESPACE_PVCS}"
oc adm policy add-scc-to-user anyuid -z oadp-test-sa -n "${NAMESPACE_PVCS}"

for ((i=1; i<=$NO_PVCS; i++)); do
  claimName="$PVC_PREFIX_NAME-$i"
  cat <<EOF | oc apply -n $NAMESPACE_PVCS -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: $claimName
  labels:
    $PVC_LABEL_NAME: $PVC_LABEL_VALUE
spec:
  storageClassName: $storageClassName
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: $PVC_MIN_REQUESTED_SIZE
EOF
  echo "Created PVC \"$claimName\" in namespace: \"$NAMESPACE_PVCS\""
done

12. Optionally create deployments that are using those PVs, note that one test should be without the pods


for ((i=1; i<=$NO_PVCS; i++)); do
  deploymentName="$PVC_PREFIX_NAME-$i"
  cat <<EOF | oc apply -n $NAMESPACE_PVCS -f -
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $deploymentName
  labels:
    $PVC_LABEL_NAME: $PVC_LABEL_VALUE
spec:
  replicas: 1
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
        $PVC_LABEL_NAME: $PVC_LABEL_VALUE
    spec:
      serviceAccountName: oadp-test-sa
      containers:
      - name: my-container
        image: nginx:latest
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: data-volume
          mountPath: /data
      volumes:
      - name: data-volume
        persistentVolumeClaim:
          claimName: ${PVC_PREFIX_NAME}-$((i % NO_PVCS + 1))
EOF
  echo "Created Deployment \"$deploymentName\" in namespace: \"$NAMESPACE_PVCS\" using PVC claim-$((i % NO_PVCS + 1))"
done


13. Create DPA:

  $ cat <<EOF | oc apply -n $OADP_NS -f -
apiVersion: oadp.openshift.io/v1alpha1
kind: DataProtectionApplication
metadata:
  name:  dpa-sample
spec:
  backupLocations:
    - velero:
        config:
          profile: default
          region: eu-central-1
          insecureSkipTLSVerify: 'true'
          s3ForcePathStyle: "true"
          s3Url: 'https://s3.openshift-adp.svc:443'          
        credential:
          key: mycloud
          name: cloud-credentials
        default: true
        objectStorage:
          bucket: first.bucket
          prefix: cephoadptest
        provider: aws
  configuration:
    nodeAgent:
      enable: false
      uploaderType: restic
    velero:
      defaultPlugins:
        - openshift
        - aws
        - csi
        - vsm
  features:
    dataMover:
      credentialName: ${SECRET_NAME}
      enable: true
      volumeOptionsForStorageClasses:
        ocs-storagecluster-cephfs:
          sourceVolumeOptions:
            accessMode: ReadOnlyMany
            cacheAccessMode: ReadWriteMany
            cacheStorageClassName: ocs-storagecluster-cephfs
            storageClassName: ocs-storagecluster-cephfs-shallow
EOF


15. Create backup CR:

  $ cat <<EOF | oc apply -n $OADP_NS -f -
apiVersion: velero.io/v1
kind: Backup
metadata:
  name: test-backup
spec:
  includedNamespaces:
  - ${NAMESPACE_PVCS}
  storageLocation: dpa-sample-1
EOF

16. Once backup is complete delete the pvc's and deployments:

 $ oc delete deployments -n "$NAMESPACE_PVCS" -l "$PVC_LABEL_NAME=$PVC_LABEL_VALUE"
 $ oc delete pvc -n "$NAMESPACE_PVCS" -l "$PVC_LABEL_NAME=$PVC_LABEL_VALUE"

17. Create restore CR:
  $ cat <<EOF | oc apply -n $OADP_NS -f -
apiVersion: velero.io/v1
kind: Restore
metadata:
  name: test-backup
spec:
  backupName: test-backup
  restorePVs: true
EOF


############################# Cleanup:
 $ oc delete pvc -n "$NAMESPACE_PVCS" -l "$PVC_LABEL_NAME=$PVC_LABEL_VALUE"
 $ oc delete secret cloud-credentials -n "$OADP_NS"
 $ oc delete deployments -n "$NAMESPACE_PVCS" -l "$PVC_LABEL_NAME=$PVC_LABEL_VALUE"
 $ oc delete ns -n "$NAMESPACE_PVCS"