Uploaded image for project: 'Red Hat OpenStack Services on OpenShift'
  1. Red Hat OpenStack Services on OpenShift
  2. OSPRH-9309

Fernet key rotation - operator implementation

XMLWordPrintable

    • Icon: Story Story
    • Resolution: Done
    • Icon: Major Major
    • rhos-18.0 FR 1 (Nov 2024)
    • None
    • None
    • None
    • DFG Security: UC Sprint 101, DFG Security: UC Sprint 102, DFG Security: UC Sprint 103
    • 3

      The result of the story should be a pull request opened in the keystone-operator repository, containing a complete implementation of fernet key rotation.

      The cadence of the fernet token keys rotation should be configurable. There are 3 settings in total: the number of fernet-keys = token-validity(hours) / rotation-time(hours) + 2

      The old documentation instructs the operator to set these individually, but maybe there is a way to have only one or 2 settings for the end user. https://docs.redhat.com/en/documentation/red_hat_openstack_platform/10/html/deploy_fernet_on_the_overcloud/rotate_the_fernet_keys#rotate_the_fernet_keys

       

      This is a set of working CRs that can be applied to an install_yamls cluster to rotate the keys:

       

      ---
      kind: Role
      apiVersion: rbac.authorization.k8s.io/v1
      metadata: 
        name: keystone-fernet-rotation-role-example
        namespace: openstack
      rules: 
      - apiGroups: [""]
        resources: 
        - secrets
        verbs: 
        - 'delete'
        - 'create'
        - 'patch'
        - 'get'
      ---
      kind: ServiceAccount
      apiVersion: v1
      metadata: 
        name: keystone-fernet-rotation-example
        namespace: openstack
      ---
      kind: RoleBinding
      apiVersion: rbac.authorization.k8s.io/v1
      metadata: 
        name: keystone-fernet-rotation-rolebinding-example
        namespace: openstack
      subjects: 
      - kind: ServiceAccount
        name: keystone-fernet-rotation-example
        namespace: openstack
      roleRef: 
        kind: Role
        name: keystone-fernet-rotation-role-example
        apiGroup: ""
      ---
      kind: CronJob
      apiVersion: batch/v1
      metadata: 
        annotations: 
        name: keystone-fernet-rotation-job-example
        namespace: openstack
      spec: 
        concurrencyPolicy: Forbid
        failedJobsHistoryLimit: 1
        jobTemplate: 
          metadata: 
            creationTimestamp: null
          spec: 
            backoffLimit: 0
            template: 
              metadata: 
                creationTimestamp: null
              spec: 
                serviceAccountName: keystone-fernet-rotation-example
                containers: 
                - name: keystone-fernet-rotation-cron-example
                  image: registry.redhat.io/openshift4/ose-cli
                  imagePullPolicy: IfNotPresent
                  command: 
                  - /bin/bash
                  - -c
                  - |-
                    echo `date -u` Starting...
                    case $MAX_ACTIVE_KEYS in
                      ''|*[!0-9]*)
                        echo "MAX_ACTIVE_KEYS is not a number, exiting."
                        exit 1
                        ;;
                      [01])
                        echo "MAX_ACTIVE_KEYS ($MAX_ACTIVE_KEYS) -lt 2, exiting."
                        exit 1
                    esac
                    cd /run/keys
                    mkdir /tmp/keys
                    for file in FernetKeys[0-9]*;
                    do
                      cat "$file" > /tmp/keys/"${file#FernetKeys}"
                    done
                    cd /tmp/keys
                    number_of_keys=`ls -1 | wc -l`
                    max_key=`ls -1 | sort -n | tail -1`
                    
                    if [ $((number_of_keys - 1)) != $max_key ]; then
                      echo "Corrupted FernetKeys secret, exiting."
                      exit 1
                    fi
                    
                    mv 0 $((max_key  + 1))
                    dd if=/dev/urandom bs=32 count=1 2>/dev/null | base64 > 0
                    while [ -f "$MAX_ACTIVE_KEYS" ]; do
                      i=2
                      while [ -f "$i" ]; do
                        mv $i $((i-1))
                        i=$((i+1))
                      done
                    done
                    echo '{"stringData": {' > /tmp/patch_file.json
                    i=0
                    while [ -f "$((i+1))" ]; do
                      echo '"FernetKeys'$i'": "'`cat $i`'",' >> /tmp/patch_file.json
                      i=$((i+1))
                    done
                    echo '"FernetKeys'$i'": "'`cat $i`'"' >> /tmp/patch_file.json
                    echo '}}' >> /tmp/patch_file.json
                    kubectl patch secret -n $NAMESPACE $SECRET_NAME \
                      --patch-file=/tmp/patch_file.json
                    echo `date -u` $((i+1)) keys rotated.
                    cd /run/keys
                    if [ -f "FernetKeys$MAX_ACTIVE_KEYS" ]; then
                      echo '[' > /tmp/patch_file.json
                      i=$((MAX_ACTIVE_KEYS-1))
                      while [ -f "FernetKeys$((i+1))" ]; do
                        echo '{"op": "remove", "path": "/data/FernetKeys'$i'"},' \
                        >> /tmp/patch_file.json
                        i=$((i+1))
                      done
                        echo '{"op": "remove", "path": "/data/FernetKeys'$i'"}' \
                        >> /tmp/patch_file.json
                      echo ']' >> /tmp/patch_file.json
                      kubectl patch secret -n $NAMESPACE $SECRET_NAME \
                         --type=json --patch-file=/tmp/patch_file.json
                      echo `date -u` $MAX_ACTIVE_KEYS through $i keys deleted.
                    fi
                  env: 
                  - name: SECRET_NAME
                    value: keystone
                  - name: NAMESPACE
                    value: openstack
                  - name: MAX_ACTIVE_KEYS
                    value: "3"
                  volumeMounts: 
                    - name: keys
                      mountPath: "/run/keys"
                      readOnly: true
                volumes: 
                  - name: keys
                    secret: 
                      secretName: keystone
                restartPolicy: Never
                terminationGracePeriodSeconds: 30
        schedule: "1 0 * * *"
        successfulJobsHistoryLimit: 3
        suspend: false
      

      Trigger the job with:

      oc create job --from cronjob/keystone-fernet-rotation-job-example rotation-job-example

      See logs with:

      oc logs job/rotation-job-example

              ggrasza@redhat.com Grzegorz Grasza
              ggrasza@redhat.com Grzegorz Grasza
              rhos-dfg-security
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

                Created:
                Updated:
                Resolved: