#!/bin/bash
# multipath test script and command line utility
# Author:  <qinwang@redhat.com>
#

USAGE="Usage: MPATH  [TESTCASE] [TIMEOUT] [INTERVAL]

  MPATH:     Test multipath name, e.g. mpatha
  TESTCASE:  Testcase index, 0-9, 
      0: Display multipath
      1: Switch the active path by offline
      2: Online and switch the active path
      3: Online all paths
      4: Offline all paths
      6: Enable paths by iptables
      7: Switch the active path by iptables
      8: Periodic switch by iptables
      9: Periodic switch by offline
  TIMEOUT:   Timeout for switch, default 180s
  INTERVAL:  Interval of each switch, default 5s

  Example: $0 mpatha 9 300 5

"

MPATH=${1:-""}
TESTCASE=${2:-2}
TIMEOUT=${3:-180}
INTERVAL=${4:-5}

(($# < 1)) && {
  echo -e "Miss Parameter \n$USAGE"
  exit 1
}

multipath -ll "${MPATH}" || {
  echo "Wrong mpath: ${MPATH}"
  exit 1
}
echo "Multipath:${MPATH} Testcase:${TESTCASE} Timeout:${TIMEOUT} Interval:${INTERVAL}"

MP_PATHS=$(sudo multipath -ll "${MPATH}" | grep -o 'sd[a-z]' | sort -u)

mp_cmd() {
  echo "$@"
  eval "$@"
  return $?

}
mp_wait_for() {
  local cmd="$1"                   # command
  local timeout="${2:-$TIMEOUT}"   # timeout(default 60S)
  local interval="${3:-$INTERVAL}" # interval (default 60S)
  local fail_msg="${4:-Timeout}"   # message
  local show_progress="${5:-true}" #

  local elapsed=0
  echo "Wait for $cmd"
  while ! eval "$cmd"; do
    sleep "$interval"
    elapsed=$((elapsed + interval))

    if [ "$show_progress" = "true" ]; then
      echo -n "."
    fi

    if [ "$elapsed" -ge "$timeout" ]; then
      if [ "$show_progress" = "true" ]; then
        echo " "

      fi
      echo "$fail_msg timeout after ${timeout}s"
      return 1
    fi
  done
  echo -e "\nWait end($(date "+%H:%M:%S")) time:$elapsed"

  return 0
}

mp_get_mpath_devices() {
  sudo multipath -ll "${MPATH}" | grep -o 'sd[a-z]' | sort -u
  # lsblk -s /dev/mapper/${MPATH}|grep "─"|grep -o 'sd[a-z]'|sort -u
}

mp_get_device_mpath() {
  lsblk /dev/"$1" | grep -o 'mpath[a-z]'
}

mp_online_paths() {

  paths=${1:-${MP_PATHS}}
  for path in $paths; do
    echo "online ====== ${path} $(date "+%H:%M:%S")"
    echo running >/sys/block/"${path}"/device/state
  done
}

mp_offline_paths() {
  paths=${1:-${MP_PATHS}}
  for path in $paths; do
    echo "offline ===== ${path} $(date "+%H:%M:%S")"
    echo offline >/sys/block/"${path}"/device/state
  done
}

get_iscsi_portal_by_device() {
  local dev=${1}
  local p=$(ls -l /dev/disk/by-path | grep iscsi | grep $dev | cut -f 2 -d "-")
  if [ -n "$p" ]; then
    # echo $p
    p=$(echo "$p" | cut -f 1 -d ":")
    echo $p
  else
    return 1
  fi

}

mp_iptables_disable_path() {
  dev=$1
  portal=${2:-$(get_iscsi_portal_by_device $dev)}
  echo "Disable ===== ${dev} $(date "+%H:%M:%S")"
  mp_cmd iptables -I INPUT -s ${portal} -j DROP
}

mp_iptables_enable_path() {
  dev=$1
  portal=${2:-$(get_iscsi_portal_by_device $dev)}
  echo "Enable ===== ${dev} $(date "+%H:%M:%S")"
  mp_cmd iptables -D INPUT -s ${portal} -j DROP
}

mp_iptables_enable_paths() {
  DEVS=$(mp_get_faulty_paths)

  if [ -n "${DEVS}" ]; then
    echo "Enable the faulty paths: ${DEVS}"
    for dev in $DEVS; do
      mp_iptables_enable_path "${dev}"
      mp_wait_for "mp_get_avaliable_paths $dev" $TIMEOUT 2 "Can not enable $dev"
    done
  fi
}

mp_get_active_paths() {
  sudo multipath -ll "${MPATH}" | grep "status=active" -A 1 | grep -E "active .* running" | grep -o 'sd[a-z]' | sort -u

}

mp_get_offline_paths() {
  sudo multipath -ll "${MPATH}" | grep -E "failed .* offline" | grep -o 'sd[a-z]' | sort -u

}

mp_get_faulty_paths() {
  sudo multipath -ll "${MPATH}" | grep -E "failed .* running" | grep -o 'sd[a-z]' | sort -u

}
mp_get_avaliable_paths() {
  dev=${1:-""}
  sudo multipath -ll "${MPATH}" | grep -E "active .* running" | grep -o 'sd[a-z]' | sort -u | grep $dev

}
mp_io_active_path(){
:
# Do not do io due to qemu will trigger avoid skip qemu test

# dd if=/dev/mapper/${MPATH} of=/dev/null bs=512 count=1 status=none iflag=direct >/dev/null
#   [ $? -ne 0 ] && {
#     echo "IO failed after offline, exit"
#     exit 1
#   }
}

mp_switch_by_offline() {
  echo -e "\n======== $(date "+%H:%M:%S")\n"
  multipath -ll "${MPATH}"
  DEVS=$(mp_get_offline_paths)
  if [ -n "${DEVS}" ]; then
    for dev in ${DEVS}; do

      echo "Online the inactive paths: ${dev}"
      mp_online_paths "${dev}"
      mp_wait_for "mp_get_avaliable_paths $dev" $TIMEOUT 3 "Can not online $dev"

    done
  fi
  echo "After online $DEVS"
  multipath -ll "${MPATH}"

  #mp_io_active_path
  mp_wait_for 'DEVS=$(mp_get_active_paths);[ -n "${DEVS}" ]'
  
  if [ -n "${DEVS}" ]; then
    echo "Offline active paths: ${DEVS}"
    mp_offline_paths "${DEVS}"
  fi

  #mp_io_active_path

  mp_wait_for 'NEW_DEVS=$(mp_get_active_paths);[ -n "${NEW_DEVS}" ] && [ "${NEW_DEVS}" != "${DEVS}" ]' $TIMEOUT $INTERVAL "Can not find new active path"
  
  
  [ $? -ne 0 ] && {
    echo "Switch failed, exit"
    exit 1
  }
  echo "Find new active path :${NEW_DEVS} from ${DEVS} $(date "+%H:%M:%S")"
  multipath -ll "${MPATH}"

  
}

mp_switch_by_iptables() {
  echo -e "\n======== $(date "+%H:%M:%S")\n"
  mp_iptables_enable_paths
  #mp_io_active_path
  DEVS=$(mp_get_active_paths)

  if [ -n "${DEVS}" ]; then
    echo "Disable active paths: ${DEVS}"
    mp_iptables_disable_path "${DEVS}"
  fi
  #mp_io_active_path
  # Wait time
  mp_wait_for 'NEW_DEVS=$(mp_get_active_paths);[ -n "${NEW_DEVS}" ] && [ "${NEW_DEVS}" != "${DEVS}" ]' $TIMEOUT $INTERVAL "Can not find new active path"
  [ $? -ne 0 ] && {
    echo "Switch failed, exit"
    exit 1
  }
  echo "Find new active path :${NEW_DEVS} from ${DEVS}  $(date "+%H:%M:%S")"
  multipath -ll "${MPATH}"

  # echo -e "\n========\n"
}

mp_periodic_switch_by_offline() {

  mp_online_paths
  sleep 10
  while true; do
    mp_switch_by_offline
    [ $? -ne 0 ] && {
      echo "Switch failed, exit"
      exit 1
    }

    sleep "${INTERVAL}"
  done
}

mp_periodic_switch_by_iptables() {

  mp_iptables_enable_paths
  sleep 5

  while true; do
    mp_switch_by_iptables
    [ $? -ne 0 ] && {
      echo "Switch failed, exit"
      exit 1
    }

    sleep "${INTERVAL}"
  done
}

mp_io_test() {
  m=${1:-mpatha}
  while true; do
    date
    #dd if=/dev/zero of=/dev/mapper/"$m" bs=4k count=1000000 oflag=direct
    time dd of=/dev/null if=/dev/mapper/"$m" bs=4k count=1000000 iflag=direct
    [[ $? -ne 0 ]] && { echo " IO ERROR";break; }
    sleep 0.5
  done
  
}

mp_service_restart() {
  echo " 
  systemctl stop multipathd.service multipathd.socket;  sleep 3;  systemctl start multipathd;  systemctl reload --now multipathd; 
  
  systemctl enable qemu-pr-helper;systemctl start qemu-pr-helper;  systemctl status qemu-pr-helper
  
  "

}

mp_version() {

  rpm -q device-mapper device-mapper-multipath
  uname -r
}

case ${TESTCASE} in
0)
  echo "Display multipath ..."
  multipath -ll "${MPATH}"
  ;;
1)
  echo "Switch active by offline..."
  mp_switch_by_offline
  ;;
2)
  echo "Online and Switch active..."
  mp_online_paths
  sleep 5
  mp_switch_by_offline
  ;;
3)
  echo "Online all paths ..."
  mp_online_paths
  sleep 5
  multipath -l "${MPATH}"
  ;;
4)
  echo "Offline all paths ..."
  mp_offline_paths
  sleep 5
  multipath -l "${MPATH}"
  ;;
6)
  echo "Enable paths by iptables..."
  mp_iptables_enable_paths
  multipath -l "${MPATH}"
  ;;
7)
  echo "Switch active by iptables.."
  mp_switch_by_iptables
  ;;
8)
  echo "Periodic switch by iptables..."
  mp_periodic_switch_by_iptables
  ;;
9)
  echo "Periodic switch by offline ..."
  mp_periodic_switch_by_offline
  ;;

? | *)
  echo "unknown TESTCASE: $TESTCASE"
  echo -e "$USAGE"
  exit 1 #
  ;;
esac
