apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    deprecated.daemonset.template.generation: "3"
    kubernetes.io/description: |
      This daemonset launches the ovn-kubernetes per node networking components.
    networkoperator.openshift.io/cluster-network-cidr: 10.128.0.0/14
    networkoperator.openshift.io/hybrid-overlay-status: disabled
    networkoperator.openshift.io/ip-family-mode: single-stack
    networkoperator.openshift.io/rollout-hung: ""
    release.openshift.io/version: 4.14.0-rc.5
  creationTimestamp: "2023-10-11T06:08:13Z"
  generation: 3
  labels:
    networkoperator.openshift.io/generates-operator-status: stand-alone
  name: ovnkube-node
  namespace: openshift-ovn-kubernetes
  ownerReferences:
  - apiVersion: operator.openshift.io/v1
    blockOwnerDeletion: true
    controller: true
    kind: Network
    name: cluster
    uid: 8148fe49-f299-438a-959f-50eda73d01f3
  resourceVersion: "120223"
  uid: f70db9ff-9b8f-446b-bbd0-4e29b77676b0
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: ovnkube-node
  template:
    metadata:
      annotations:
        networkoperator.openshift.io/cluster-network-cidr: 10.128.0.0/14
        networkoperator.openshift.io/hybrid-overlay-status: disabled
        networkoperator.openshift.io/ip-family-mode: single-stack
        target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
      creationTimestamp: null
      labels:
        app: ovnkube-node
        component: network
        kubernetes.io/os: linux
        openshift.io/component: network
        ovn-db-pod: "true"
        type: infra
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: network.operator.openshift.io/dpu-host
                operator: DoesNotExist
              - key: network.operator.openshift.io/smart-nic
                operator: DoesNotExist
              - key: network.operator.openshift.io/dpu
                operator: DoesNotExist
      containers:
      - command:
        - /bin/bash
        - -c
        - |
          set -e
          if [[ -f "/env/${K8S_NODE}" ]]; then
            set -o allexport
            source "/env/${K8S_NODE}"
            set +o allexport
          fi

          echo "$(date -Iseconds) - starting ovn-controller"
          exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \
            --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \
            --syslog-method="null" \
            --log-file=/var/log/ovn/acl-audit-log.log \
            -vFACILITY:"local0" \
            -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \
            -vconsole:"${OVN_LOG_LEVEL}" -vconsole:"acl_log:off" \
            -vPATTERN:console:"%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \
            -vsyslog:"acl_log:info" \
            -vfile:"acl_log:info"
        env:
        - name: OVN_LOG_LEVEL
          value: info
        - name: K8S_NODE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        name: ovn-controller
        resources:
          requests:
            cpu: 10m
            memory: 300Mi
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /run/openvswitch
          name: run-openvswitch
        - mountPath: /run/ovn/
          name: run-ovn
        - mountPath: /etc/openvswitch
          name: etc-openvswitch
        - mountPath: /etc/ovn/
          name: etc-openvswitch
        - mountPath: /var/lib/openvswitch
          name: var-lib-openvswitch
        - mountPath: /env
          name: env-overrides
        - mountPath: /ovn-cert
          name: ovn-cert
        - mountPath: /ovn-ca
          name: ovn-ca
        - mountPath: /var/log/ovn/
          name: node-log
        - mountPath: /dev/log
          name: log-socket
      - command:
        - /bin/bash
        - -c
        - |
          set -euo pipefail

          # Rotate audit log files when then get to max size (in bytes)
          MAXFILESIZE=$(( "50"*1000000 ))
          MAXLOGFILES="5"
          LOGDIR=/var/log/ovn
          LOGFILE=${LOGDIR}/acl-audit-log.log
          CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid)

          # Redirect err to null so no messages are shown upon rotation
          tail -F ${LOGFILE} 2> /dev/null &

          while true
          do
            # Make sure ovn-controller's logfile exists, and get current size in bytes
            if [ -f "$LOGFILE" ]; then
              file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1`
            else
              ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen
              file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1`
            fi

            if [ $file_size -gt $MAXFILESIZE ];then
              echo "Rotating OVN ACL Log File"
              timestamp=`date '+%Y-%m-%dT%H-%M-%S'`
              mv ${LOGFILE} /var/log/ovn/acl-audit-log.$timestamp.log
              ovs-appctl -t /run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen
              CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid)
            fi

            # Ensure total number of log files does not exceed the maximum configured from OVNPolicyAuditMaxLogFiles
            num_files=$(ls -1 ${LOGDIR}/acl-audit-log* 2>/dev/null | wc -l)
            if [ "$num_files" -gt "$MAXLOGFILES" ]; then
              num_to_delete=$(( num_files - ${MAXLOGFILES} ))
              ls -1t ${LOGDIR}/acl-audit-log* 2>/dev/null | tail -$num_to_delete | xargs -I {} rm {}
            fi

            # sleep for 30 seconds to avoid wasting CPU
            sleep 30
          done
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        name: ovn-acl-logging
        resources:
          requests:
            cpu: 10m
            memory: 20Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /var/log/ovn/
          name: node-log
        - mountPath: /run/ovn/
          name: run-ovn
      - command:
        - /bin/bash
        - -c
        - |
          #!/bin/bash
          set -euo pipefail

          TLS_PK=/etc/pki/tls/metrics-cert/tls.key
          TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt
          # As the secret mount is optional we must wait for the files to be present.
          # The service is created in monitor.yaml and this is created in sdn.yaml.
          # If it isn't created there is probably an issue so we want to crashloop.
          retries=0
          TS=$(date +%s)
          WARN_TS=$(( ${TS} + $(( 20 * 60)) ))
          HAS_LOGGED_INFO=0

          log_missing_certs(){
              CUR_TS=$(date +%s)
              if [[ "${CUR_TS}" -gt "WARN_TS"  ]]; then
                echo $(date -Iseconds) WARN: ovn-node-metrics-cert not mounted after 20 minutes.
              elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then
                echo $(date -Iseconds) INFO: ovn-node-metrics-cert not mounted. Waiting one hour.
                HAS_LOGGED_INFO=1
              fi
          }
          while [[ ! -f "${TLS_PK}" ||  ! -f "${TLS_CERT}" ]] ; do
            log_missing_certs
            sleep 5
          done

          echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy
          exec /usr/bin/kube-rbac-proxy \
            --logtostderr \
            --secure-listen-address=:9103 \
            --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \
            --upstream=http://127.0.0.1:29103/ \
            --tls-private-key-file=${TLS_PK} \
            --tls-cert-file=${TLS_CERT}
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ed9d3463f9f56e26db83203fc8f6c5056de3aa1ef7fabdb7664bda1492d5761e
        imagePullPolicy: IfNotPresent
        name: kube-rbac-proxy-node
        ports:
        - containerPort: 9103
          hostPort: 9103
          name: https
          protocol: TCP
        resources:
          requests:
            cpu: 10m
            memory: 20Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /etc/pki/tls/metrics-cert
          name: ovn-node-metrics-cert
          readOnly: true
      - command:
        - /bin/bash
        - -c
        - |
          #!/bin/bash
          set -euo pipefail

          TLS_PK=/etc/pki/tls/metrics-cert/tls.key
          TLS_CERT=/etc/pki/tls/metrics-cert/tls.crt
          # As the secret mount is optional we must wait for the files to be present.
          # The service is created in monitor.yaml and this is created in sdn.yaml.
          # If it isn't created there is probably an issue so we want to crashloop.
          retries=0
          TS=$(date +%s)
          WARN_TS=$(( ${TS} + $(( 20 * 60)) ))
          HAS_LOGGED_INFO=0

          log_missing_certs(){
              CUR_TS=$(date +%s)
              if [[ "${CUR_TS}" -gt "WARN_TS"  ]]; then
                echo $(date -Iseconds) WARN: ovn-node-metrics-cert not mounted after 20 minutes.
              elif [[ "${HAS_LOGGED_INFO}" -eq 0 ]] ; then
                echo $(date -Iseconds) INFO: ovn-node-metrics-cert not mounted. Waiting one hour.
                HAS_LOGGED_INFO=1
              fi
          }
          while [[ ! -f "${TLS_PK}" ||  ! -f "${TLS_CERT}" ]] ; do
            log_missing_certs
            sleep 5
          done

          echo $(date -Iseconds) INFO: ovn-node-metrics-certs mounted, starting kube-rbac-proxy
          exec /usr/bin/kube-rbac-proxy \
            --logtostderr \
            --secure-listen-address=:9105 \
            --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 \
            --upstream=http://127.0.0.1:29105/ \
            --tls-private-key-file=${TLS_PK} \
            --tls-cert-file=${TLS_CERT}
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ed9d3463f9f56e26db83203fc8f6c5056de3aa1ef7fabdb7664bda1492d5761e
        imagePullPolicy: IfNotPresent
        name: kube-rbac-proxy-ovn-metrics
        ports:
        - containerPort: 9105
          hostPort: 9105
          name: https
          protocol: TCP
        resources:
          requests:
            cpu: 10m
            memory: 20Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /etc/pki/tls/metrics-cert
          name: ovn-node-metrics-cert
          readOnly: true
      - command:
        - /bin/bash
        - -c
        - |
          set -xem
          if [[ -f /env/_master ]]; then
            set -o allexport
            source /env/_master
            set +o allexport
          fi

          quit() {
            echo "$(date -Iseconds) - stopping ovn-northd"
            OVN_MANAGE_OVSDB=no /usr/share/ovn/scripts/ovn-ctl stop_northd
            echo "$(date -Iseconds) - ovn-northd stopped"
            rm -f /var/run/ovn/ovn-northd.pid
            exit 0
          }
          # end of quit
          trap quit TERM INT

          echo "$(date -Iseconds) - starting ovn-northd"
          exec ovn-northd \
            --no-chdir "-vconsole:${OVN_LOG_LEVEL}" -vfile:off "-vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \
            --pidfile /var/run/ovn/ovn-northd.pid \
            --n-threads=1 &
          wait $!
        env:
        - name: OVN_LOG_LEVEL
          value: info
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        name: northd
        resources:
          requests:
            cpu: 10m
            memory: 70Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /etc/openvswitch/
          name: etc-openvswitch
        - mountPath: /var/lib/openvswitch/
          name: var-lib-openvswitch
        - mountPath: /run/openvswitch/
          name: run-openvswitch
        - mountPath: /run/ovn/
          name: run-ovn
        - mountPath: /env
          name: env-overrides
        - mountPath: /ovn-cert
          name: ovn-cert
        - mountPath: /ovn-ca
          name: ovn-ca
      - command:
        - /bin/bash
        - -c
        - |
          set -xem
          if [[ -f /env/_master ]]; then
            set -o allexport
            source /env/_master
            set +o allexport
          fi

          quit() {
            echo "$(date -Iseconds) - stopping nbdb"
            /usr/share/ovn/scripts/ovn-ctl stop_nb_ovsdb
            echo "$(date -Iseconds) - nbdb stopped"
            rm -f /var/run/ovn/ovnnb_db.pid
            exit 0
          }
          # end of quit
          trap quit TERM INT

          bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac }

          OVN_ARGS="--no-monitor"
          exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \
            --ovn-nb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \
            run_nb_ovsdb &

          wait $!
        env:
        - name: OVN_LOG_LEVEL
          value: info
        - name: OVN_NORTHD_PROBE_INTERVAL
          value: "10000"
        - name: K8S_NODE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
            exec:
              command:
              - /bin/bash
              - -c
              - |
                set -x
                rm -f /var/run/ovn/ovnnb_db.pid
                #configure northd_probe_interval
                OVN_NB_CTL="ovn-nbctl"

                # retry an operation a number of times, sleeping 2 seconds between each try
                retry() {
                  local tries=${1}
                  local desc=${2}
                  local cmd=${3}

                  local retries=0
                  while ! ${cmd}; do
                    (( retries += 1 ))
                    if [[ "${retries}" -gt ${tries} ]]; then
                      echo "$(date -Iseconds) - ERROR - nbdb ${desc} - too many failed attempts, giving up"
                      return 1
                    fi
                    echo "$(date -Iseconds) - WARN - nbdb ${desc} - failed try ${retries}, retrying..."
                    sleep 2
                  done
                  echo "$(date -Iseconds) - INFO - nbdb ${desc} - success"
                  return 0
                }

                # set the connection and inactivity probe
                if ! retry 60 "inactivity-probe" "ovn-nbctl --no-leader-only -t 5 set-connection pssl:9641 -- set connection . inactivity_probe=60000"; then
                  exit 1
                fi

                # Upgrade the db if required.
                DB_SCHEMA="/usr/share/ovn/ovn-nb.ovsschema"
                DB_SERVER="unix:/var/run/ovn/ovnnb_db.sock"
                schema_name=$(ovsdb-tool schema-name $DB_SCHEMA)
                db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name")
                target_version=$(ovsdb-tool schema-version "$DB_SCHEMA")

                if ovsdb-tool compare-versions "$db_version" == "$target_version"; then
                  :
                elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then
                    echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?"
                else
                    echo "Upgrading database $schema_name from schema version $db_version to $target_version"
                    ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA"
                fi

                echo "Setting the IC zone to ${K8S_NODE}"
                retries=0
                current_probe_interval=0
                while [[ "${retries}" -lt 10 ]]; do
                  current_probe_interval=$(${OVN_NB_CTL} set NB_Global . name="${K8S_NODE}" options:name="${K8S_NODE}")
                  if [[ $? == 0 ]]; then
                    current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"')
                    break
                  else
                    sleep 2
                    (( retries += 1 ))
                  fi
                done

                northd_probe_interval=${OVN_NORTHD_PROBE_INTERVAL:-10000}
                echo "Setting northd probe interval to ${northd_probe_interval} ms"
                retries=0
                current_probe_interval=0
                while [[ "${retries}" -lt 10 ]]; do
                  current_probe_interval=$(${OVN_NB_CTL} --if-exists get NB_GLOBAL . options:northd_probe_interval)
                  if [[ $? == 0 ]]; then
                    current_probe_interval=$(echo ${current_probe_interval} | tr -d '\"')
                    break
                  else
                    sleep 2
                    (( retries += 1 ))
                  fi
                done

                if [[ "${current_probe_interval}" != "${northd_probe_interval}" ]]; then
                  retries=0
                  while [[ "${retries}" -lt 10 ]]; do
                    ${OVN_NB_CTL} set NB_GLOBAL . options:northd_probe_interval=${northd_probe_interval}
                    if [[ $? != 0 ]]; then
                      echo "Failed to set northd probe interval to ${northd_probe_interval}. retrying..."
                      sleep 2
                      (( retries += 1 ))
                    else
                      echo "Successfully set northd probe interval to ${northd_probe_interval} ms"
                      break
                    fi
                  done
                fi

                # Enable/disable IPsec

                ipsec=false

                ipsec_encapsulation=false
                if [ "${ipsec}" == "true" ]; then
                    # IBMCloud does not forward ESP (IP proto 50)
                    # Instead, force IBMCloud IPsec to always use NAT-T
                    if [ "GCP" == "IBMCloud" ]; then
                        ipsec_encapsulation=true
                    fi
                fi
                if ! retry 20 "ipsec" "${OVN_NB_CTL} set nb_global . ipsec=${ipsec} options:ipsec_encapsulation=${ipsec_encapsulation}"; then
                  exit 1
                fi

                # Tell northd to sleep a bit so it takes less CPU
                if ! retry 20 "northd-backoff" "${OVN_NB_CTL} set nb_global . options:northd-backoff-interval-ms=300"; then
                  exit 1
                fi
        name: nbdb
        readinessProbe:
          exec:
            command:
            - /bin/bash
            - -c
            - |
              set -xeo pipefail
              status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=3 ovsdb-server/sync-status  2>/dev/null | { grep "state: active" || false; })
              if [[  -z "${status}" ]]; then
                echo "NB DB is not running or active."
                exit 1
              fi

              # set trim-on-compaction
              /usr/bin/ovn-appctl -t /var/run/ovn/ovnnb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null
          failureThreshold: 3
          initialDelaySeconds: 90
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        resources:
          requests:
            cpu: 10m
            memory: 300Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /etc/openvswitch/
          name: etc-openvswitch
        - mountPath: /etc/ovn/
          name: etc-openvswitch
        - mountPath: /var/lib/openvswitch/
          name: var-lib-openvswitch
        - mountPath: /run/openvswitch/
          name: run-openvswitch
        - mountPath: /run/ovn/
          name: run-ovn
        - mountPath: /env
          name: env-overrides
        - mountPath: /ovn-cert
          name: ovn-cert
        - mountPath: /ovn-ca
          name: ovn-ca
      - command:
        - /bin/bash
        - -c
        - |
          set -xem
          if [[ -f /env/_master ]]; then
            set -o allexport
            source /env/_master
            set +o allexport
          fi

          quit() {
            echo "$(date -Iseconds) - stopping sbdb"
            /usr/share/ovn/scripts/ovn-ctl stop_sb_ovsdb
            echo "$(date -Iseconds) - sbdb stopped"
            rm -f /var/run/ovn/ovnsb_db.pid
            exit 0
          }
          # end of quit
          trap quit TERM INT

          bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac }

          OVN_ARGS="--no-monitor"
          exec /usr/share/ovn/scripts/ovn-ctl ${OVN_ARGS} \
            --ovn-sb-log="-vconsole:${OVN_LOG_LEVEL} -vfile:off -vPATTERN:console:%D{%Y-%m-%dT%H:%M:%S.###Z}|%05N|%c%T|%p|%m" \
            run_sb_ovsdb &

          wait $!
        env:
        - name: OVN_LOG_LEVEL
          value: info
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        lifecycle:
          postStart:
            exec:
              command:
              - /bin/bash
              - -c
              - |
                set -x
                rm -f /var/run/ovn/ovnsb_db.pid

                # retry an operation a number of times, sleeping 2 seconds between each try
                retry() {
                  local tries=${1}
                  local desc=${2}
                  local cmd=${3}

                  local retries=0
                  while ! ${cmd}; do
                    (( retries += 1 ))
                    if [[ "${retries}" -gt ${tries} ]]; then
                      echo "$(date -Iseconds) - ERROR - sbdb ${desc} - too many failed attempts, giving up"
                      return 1
                    fi
                    echo "$(date -Iseconds) - WARN - sbdb ${desc} - failed try ${retries}, retrying..."
                    sleep 2
                  done
                  echo "$(date -Iseconds) - INFO - sbdb ${desc} - success"
                  return 0
                }

                # set the connection and inactivity probe
                if ! retry 60 "inactivity-probe" "ovn-sbctl --no-leader-only -t 5 set-connection pssl:9642 -- set connection . inactivity_probe=180000"; then
                  exit 1
                fi

                # Upgrade the db if required.
                DB_SCHEMA="/usr/share/ovn/ovn-sb.ovsschema"
                DB_SERVER="unix:/var/run/ovn/ovnsb_db.sock"
                schema_name=$(ovsdb-tool schema-name $DB_SCHEMA)
                db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" "$schema_name")
                target_version=$(ovsdb-tool schema-version "$DB_SCHEMA")
                if ovsdb-tool compare-versions "$db_version" == "$target_version"; then
                  :
                elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then
                    echo "Database $schema_name has newer schema version ($db_version) than our local schema ($target_version), possibly an upgrade is partially complete?"
                else
                    echo "Upgrading database $schema_name from schema version $db_version to $target_version"
                    ovsdb-client -t 30 convert "$DB_SERVER" "$DB_SCHEMA"
                fi

                # Kill some time while the cluster converges by checking IPsec status
                OVN_SB_CTL="ovn-sbctl"
                if ! retry 20 "ipsec" "${OVN_SB_CTL} get sb_global . ipsec"; then
                  exit 1
                fi
        name: sbdb
        readinessProbe:
          exec:
            command:
            - /bin/bash
            - -c
            - |
              set -xeo pipefail
              status=$(/usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=3 ovsdb-server/sync-status  2>/dev/null | { grep "state: active" || false; })
              if [[  -z "${status}" ]]; then
                echo "SB DB is not running or active."
                exit 1
              fi
              # set trim-on-compaction
              /usr/bin/ovn-appctl -t /var/run/ovn/ovnsb_db.ctl --timeout=5 ovsdb-server/memory-trim-on-compaction on 2>/dev/null
          failureThreshold: 3
          initialDelaySeconds: 90
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        resources:
          requests:
            cpu: 10m
            memory: 300Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /etc/openvswitch/
          name: etc-openvswitch
        - mountPath: /etc/ovn/
          name: etc-openvswitch
        - mountPath: /var/lib/openvswitch/
          name: var-lib-openvswitch
        - mountPath: /run/openvswitch/
          name: run-openvswitch
        - mountPath: /run/ovn/
          name: run-ovn
        - mountPath: /env
          name: env-overrides
        - mountPath: /ovn-cert
          name: ovn-cert
        - mountPath: /ovn-ca
          name: ovn-ca
      - command:
        - /bin/bash
        - -c
        - |
          set -xe
          if [[ -f "/env/${K8S_NODE}" ]]; then
            set -o allexport
            source "/env/${K8S_NODE}"
            set +o allexport
          fi

          function log()
          {
              echo "$(date --iso-8601=seconds) [cnibincopy] ${1}"
          }

          # collect host os information
          . /host/etc/os-release
          rhelmajor=
          # detect which version we're using in order to copy the proper binaries
          case "${ID}" in
            rhcos|scos)
              RHEL_VERSION=$(echo "${CPE_NAME}" | cut -f 5 -d :)
              rhelmajor=$(echo $RHEL_VERSION | sed -E 's/([0-9]+)\.{1}[0-9]+(\.[0-9]+)?/\1/')
            ;;
            rhel) rhelmajor=$(echo "${VERSION_ID}" | cut -f 1 -d .)
            ;;
            fedora)
              if [ "${VARIANT_ID}" == "coreos" ]; then
                rhelmajor=8
              else
                log "FATAL ERROR: Unsupported Fedora variant=${VARIANT_ID}"
                exit 1
              fi
            ;;
            *) log "FATAL ERROR: Unsupported OS ID=${ID}"; exit 1
            ;;
          esac

          # Set which directory we'll copy from, detect if it exists
          sourcedir=/usr/libexec/cni/
          case "${rhelmajor}" in
          8)
            sourcedir=/usr/libexec/cni/rhel8
          ;;
          9)
            sourcedir=/usr/libexec/cni/rhel9
          ;;
          *)
            log "ERROR: RHEL Major Version Unsupported, rhelmajor=${rhelmajor}"
          ;;
          esac

          cp -f "$sourcedir/ovn-k8s-cni-overlay" /cni-bin-dir/

          ovn_config_namespace=openshift-ovn-kubernetes
          echo "I$(date "+%m%d %H:%M:%S.%N") - disable conntrack on geneve port"
          iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK
          iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK
          ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK
          ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK
          echo "I$(date "+%m%d %H:%M:%S.%N") - starting ovnkube-node"

          if [ "shared" == "shared" ]; then
            gateway_mode_flags="--gateway-mode shared --gateway-interface br-ex"
          elif [ "shared" == "local" ]; then
            gateway_mode_flags="--gateway-mode local --gateway-interface br-ex"
          else
            echo "Invalid OVN_GATEWAY_MODE: \"shared\". Must be \"local\" or \"shared\"."
            exit 1
          fi

          export_network_flows_flags=
          if [[ -n "${NETFLOW_COLLECTORS}" ]] ; then
            export_network_flows_flags="--netflow-targets ${NETFLOW_COLLECTORS}"
          fi
          if [[ -n "${SFLOW_COLLECTORS}" ]] ; then
            export_network_flows_flags="$export_network_flows_flags --sflow-targets ${SFLOW_COLLECTORS}"
          fi
          if [[ -n "${IPFIX_COLLECTORS}" ]] ; then
            export_network_flows_flags="$export_network_flows_flags --ipfix-targets ${IPFIX_COLLECTORS}"
          fi
          if [[ -n "${IPFIX_CACHE_MAX_FLOWS}" ]] ; then
            export_network_flows_flags="$export_network_flows_flags --ipfix-cache-max-flows ${IPFIX_CACHE_MAX_FLOWS}"
          fi
          if [[ -n "${IPFIX_CACHE_ACTIVE_TIMEOUT}" ]] ; then
            export_network_flows_flags="$export_network_flows_flags --ipfix-cache-active-timeout ${IPFIX_CACHE_ACTIVE_TIMEOUT}"
          fi
          if [[ -n "${IPFIX_SAMPLING}" ]] ; then
            export_network_flows_flags="$export_network_flows_flags --ipfix-sampling ${IPFIX_SAMPLING}"
          fi
          gw_interface_flag=
          # if br-ex1 is configured on the node, we want to use it for external gateway traffic
          if [ -d /sys/class/net/br-ex1 ]; then
            gw_interface_flag="--exgw-interface=br-ex1"
          fi

          node_mgmt_port_netdev_flags=
          if [[ -n "${OVNKUBE_NODE_MGMT_PORT_NETDEV}" ]] ; then
            node_mgmt_port_netdev_flags="--ovnkube-node-mgmt-port-netdev ${OVNKUBE_NODE_MGMT_PORT_NETDEV}"
          fi
          if [[ -n "${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}" ]] ; then
            node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${OVNKUBE_NODE_MGMT_PORT_DP_RESOURCE_NAME}"
          fi

          multi_network_enabled_flag=
          if [[ "true" == "true" ]]; then
            multi_network_enabled_flag="--enable-multi-network"
          fi

          multi_network_policy_enabled_flag=
          if [[ "false" == "true" ]]; then
            multi_network_policy_enabled_flag="--enable-multi-networkpolicy"
          fi

          admin_network_policy_enabled_flag=
          if [[ "false" == "true" ]]; then
            admin_network_policy_enabled_flag="--enable-admin-network-policy"
          fi

          # If IP Forwarding mode is global set it in the host here.
          ip_forwarding_flag=
          if [ "Global" == "Global" ]; then
            sysctl -w net.ipv4.ip_forward=1
            sysctl -w net.ipv6.conf.all.forwarding=1
          else
            ip_forwarding_flag="--disable-forwarding"
          fi

          NETWORK_NODE_IDENTITY_ENABLE=
          if [[ "true" == "true" ]]; then
            NETWORK_NODE_IDENTITY_ENABLE="
              --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig
              --cert-dir=/etc/ovn/ovnkube-node-certs
              --cert-duration=24h
            "
          fi

          exec /usr/bin/ovnkube --init-ovnkube-controller "${K8S_NODE}" --init-node "${K8S_NODE}" \
            --config-file=/run/ovnkube-config/ovnkube.conf \
            --ovn-empty-lb-events \
            --loglevel "${OVN_KUBE_LOG_LEVEL}" \
            --inactivity-probe="${OVN_CONTROLLER_INACTIVITY_PROBE}" \
            ${gateway_mode_flags} \
            ${node_mgmt_port_netdev_flags} \
            --metrics-bind-address "127.0.0.1:29103" \
            --ovn-metrics-bind-address "127.0.0.1:29105" \
            --metrics-enable-pprof \
            --metrics-enable-config-duration \
            --export-ovs-metrics \
            --disable-snat-multiple-gws \
            ${export_network_flows_flags} \
            ${multi_network_enabled_flag} \
            ${multi_network_policy_enabled_flag} \
            ${admin_network_policy_enabled_flag} \
            --enable-multicast \
            --zone ${K8S_NODE} \
            --enable-interconnect \
            --acl-logging-rate-limit "20" \
            ${gw_interface_flag} \
            --enable-multi-external-gateway=true \
            ${ip_forwarding_flag} \
            ${NETWORK_NODE_IDENTITY_ENABLE}
        env:
        - name: KUBERNETES_SERVICE_PORT
          value: "6443"
        - name: KUBERNETES_SERVICE_HOST
          value: api-int.mbabushk-gcp.gcp.dev09.red-chesterfield.com
        - name: OVN_CONTROLLER_INACTIVITY_PROBE
          value: "180000"
        - name: OVN_KUBE_LOG_LEVEL
          value: "4"
        - name: K8S_NODE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        lifecycle:
          preStop:
            exec:
              command:
              - rm
              - -f
              - /etc/cni/net.d/10-ovn-kubernetes.conf
        name: ovnkube-controller
        ports:
        - containerPort: 29103
          hostPort: 29103
          name: metrics-port
          protocol: TCP
        readinessProbe:
          exec:
            command:
            - test
            - -f
            - /etc/cni/net.d/10-ovn-kubernetes.conf
          failureThreshold: 3
          initialDelaySeconds: 5
          periodSeconds: 30
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          requests:
            cpu: 10m
            memory: 600Mi
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - mountPath: /var/lib/kubelet
          name: host-kubelet
          readOnly: true
        - mountPath: /etc/systemd/system
          name: systemd-units
          readOnly: true
        - mountPath: /host
          mountPropagation: HostToContainer
          name: host-slash
          readOnly: true
        - mountPath: /run/ovn-kubernetes/
          name: host-run-ovn-kubernetes
        - mountPath: /run/netns
          mountPropagation: HostToContainer
          name: host-run-netns
          readOnly: true
        - mountPath: /cni-bin-dir
          name: host-cni-bin
        - mountPath: /etc/cni/net.d
          name: host-cni-netd
        - mountPath: /var/lib/cni/networks/ovn-k8s-cni-overlay
          name: host-var-lib-cni-networks-ovn-kubernetes
        - mountPath: /run/openvswitch
          name: run-openvswitch
        - mountPath: /var/log/ovnkube/
          name: etc-openvswitch
        - mountPath: /run/ovn/
          name: run-ovn
        - mountPath: /etc/openvswitch
          name: etc-openvswitch
        - mountPath: /etc/ovn/
          name: etc-openvswitch
        - mountPath: /var/lib/openvswitch
          name: var-lib-openvswitch
        - mountPath: /run/ovnkube-config/
          name: ovnkube-config
        - mountPath: /env
          name: env-overrides
        - mountPath: /ovn-cert
          name: ovn-cert
        - mountPath: /ovn-ca
          name: ovn-ca
      dnsPolicy: Default
      hostNetwork: true
      hostPID: true
      initContainers:
      - command:
        - /bin/bash
        - -c
        - |
          cat << EOF > /etc/ovn/kubeconfig
          apiVersion: v1
          clusters:
            - cluster:
                certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                server: https://api-int.mbabushk-gcp.gcp.dev09.red-chesterfield.com:6443
              name: default-cluster
          contexts:
            - context:
                cluster: default-cluster
                namespace: default
                user: default-auth
              name: default-context
          current-context: default-context
          kind: Config
          preferences: {}
          users:
            - name: default-auth
              user:
                client-certificate: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem
                client-key: /etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem
          EOF
        image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b3ce1f8a0dd37203ae125d4a99d95847d17fd3b0b3fff15f776aec108c6e0fc3
        imagePullPolicy: IfNotPresent
        name: kubecfg-setup
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/ovn/
          name: etc-openvswitch
      nodeSelector:
        beta.kubernetes.io/os: linux
      priorityClassName: system-node-critical
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: ovn-kubernetes-node
      serviceAccountName: ovn-kubernetes-node
      terminationGracePeriodSeconds: 30
      tolerations:
      - operator: Exists
      volumes:
      - hostPath:
          path: /var/lib/kubelet
          type: ""
        name: host-kubelet
      - hostPath:
          path: /etc/systemd/system
          type: ""
        name: systemd-units
      - hostPath:
          path: /
          type: ""
        name: host-slash
      - hostPath:
          path: /run/netns
          type: ""
        name: host-run-netns
      - hostPath:
          path: /var/lib/openvswitch/data
          type: ""
        name: var-lib-openvswitch
      - hostPath:
          path: /var/lib/ovn-ic/etc
          type: ""
        name: etc-openvswitch
      - hostPath:
          path: /var/run/openvswitch
          type: ""
        name: run-openvswitch
      - hostPath:
          path: /var/run/ovn-ic
          type: ""
        name: run-ovn
      - hostPath:
          path: /var/log/ovn
          type: ""
        name: node-log
      - hostPath:
          path: /dev/log
          type: ""
        name: log-socket
      - hostPath:
          path: /run/ovn-kubernetes
          type: ""
        name: host-run-ovn-kubernetes
      - hostPath:
          path: /var/lib/cni/bin
          type: ""
        name: host-cni-bin
      - hostPath:
          path: /var/run/multus/cni/net.d
          type: ""
        name: host-cni-netd
      - hostPath:
          path: /var/lib/cni/networks/ovn-k8s-cni-overlay
          type: ""
        name: host-var-lib-cni-networks-ovn-kubernetes
      - configMap:
          defaultMode: 420
          name: ovnkube-config
        name: ovnkube-config
      - configMap:
          defaultMode: 420
          name: env-overrides
          optional: true
        name: env-overrides
      - configMap:
          defaultMode: 420
          name: ovn-ca
        name: ovn-ca
      - name: ovn-cert
        secret:
          defaultMode: 420
          secretName: ovn-cert
      - name: ovn-node-metrics-cert
        secret:
          defaultMode: 420
          optional: true
          secretName: ovn-node-metrics-cert
  updateStrategy:
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 10%
    type: RollingUpdate
status:
  currentNumberScheduled: 7
  desiredNumberScheduled: 7
  numberAvailable: 6
  numberMisscheduled: 0
  numberReady: 6
  numberUnavailable: 1
  observedGeneration: 3
  updatedNumberScheduled: 1
