#!/usr/bin/env bash
set -u

NS="${NS:-evict-test}"
ITERATIONS="${ITERATIONS:-200}"            # batches to run
PARALLEL="${PARALLEL:-8}"                  # pods per batch
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-360}"  # wait per pod for eviction
READY_TIMEOUT="${READY_TIMEOUT:-120}"      # initial ready wait (best-effort)
PAUSE_BETWEEN_BATCHES="${PAUSE_BETWEEN_BATCHES:-3}"

IMG="${IMG:-quay.io/openshift/community-e2e-images:e2e-1-registry-k8s-io-e2e-test-images-agnhost-2-56-_LlCpxAuzF2PkFYl}"
REQ_LIMIT="${REQ_LIMIT:-5Mi}"
DD_COUNT_MB="${DD_COUNT_MB:-10}"           # write this much to trigger eviction
LABEL_KEY="evict-batch"

LOG_FILE="${LOG_FILE:-evict-pod-times.log}"

echo "Ensuring namespace ${NS} exists..."
oc create ns "${NS}" 2>/dev/null || true

echo "# $(date --iso-8601=seconds) - starting run (ITERATIONS=${ITERATIONS}, PARALLEL=${PARALLEL})" >> "${LOG_FILE}"

create_pod() {
  local name="$1"
  cat <<EOF | oc apply -f - >/dev/null
apiVersion: v1
kind: Pod
metadata:
  name: ${name}
  namespace: ${NS}
  labels:
    ${LABEL_KEY}: "true"
spec:
  securityContext:
    runAsNonRoot: true
    runAsUser: 1000
    seccompProfile:
      type: RuntimeDefault
  restartPolicy: Never
  containers:
  - name: ${name}
    image: ${IMG}
    command: ["/bin/sh","-c"]
    args:
    - |
      echo "starting dd";
      sleep 10;
      dd if=/dev/zero of=/tmp/file bs=1M count=${DD_COUNT_MB};
      echo "dd done";
      sleep 10000
    resources:
      requests: { ephemeral-storage: ${REQ_LIMIT} }
      limits:   { ephemeral-storage: ${REQ_LIMIT} }
    securityContext:
      allowPrivilegeEscalation: false
      runAsNonRoot: true
      runAsUser: 1000
      capabilities: { drop: ["ALL"] }
      seccompProfile: { type: RuntimeDefault }
EOF
}

collect_debug_and_exit() {
  local name="$1" iter="$2"
  local dir="debug-parallel-${iter}-${name}"
  echo "[${iter}] === POSSIBLE FLAKE on ${name}: collecting debug to ${dir} ==="
  mkdir -p "${dir}"

  oc get pod "${name}" -n "${NS}" -o wide      > "${dir}/pod-get.txt" 2>&1 || true
  oc get pod "${name}" -n "${NS}" -o yaml      > "${dir}/pod.yaml"    2>&1 || true
  oc describe pod "${name}" -n "${NS}"         > "${dir}/pod-describe.txt" 2>&1 || true
  oc logs "${name}" -n "${NS}"                 > "${dir}/pod-logs.txt" 2>&1 || true
  oc logs "${name}" -n "${NS}" --previous      > "${dir}/pod-logs-previous.txt" 2>&1 || true
  oc get events -n "${NS}" --sort-by=.lastTimestamp > "${dir}/events.txt" 2>&1 || true

  local node
  node="$(oc get pod "${name}" -n "${NS}" -o jsonpath='{.spec.nodeName}' 2>/dev/null || echo '')"
  if [[ -n "${node}" ]]; then
    oc describe node "${node}"                 > "${dir}/node-describe.txt" 2>&1 || true
    oc adm node-logs "${node}" --path=kubelet.log > "${dir}/kubelet.log" 2>&1 || true
  fi

  echo "[${iter}] Debug captured. Exiting non-zero."
  exit 1
}

for iter in $(seq 1 "${ITERATIONS}"); do
  echo "=============================="
  echo "Batch iteration ${iter} (size=${PARALLEL})"
  echo "=============================="

  echo "[${iter}] Cleaning up any prior pods..."
  oc delete pod -n "${NS}" -l "${LABEL_KEY}=true" --ignore-not-found --wait=false >/dev/null 2>&1 || true

  tmpdir="$(mktemp -d)"
  names=()

  # Create batch
  for j in $(seq 1 "${PARALLEL}"); do
    name="evicted-pod-${iter}-${j}"
    names+=("${name}")
    create_pod "${name}"
    # record per-pod start time (approx “creation time” from our POV)
    date +%s > "${tmpdir}/${name}.start"
  done

  # Best-effort “ready” wait (doesn't fail the batch)
  for name in "${names[@]}"; do
    oc wait --for=condition=Ready pod/"${name}" -n "${NS}" --timeout="${READY_TIMEOUT}s" >/dev/null 2>&1 || true
  done

  start_ts=$(date +%s)

  while true; do
    all_done=true

    for name in "${names[@]}"; do
      # skip if already processed
      if [[ -f "${tmpdir}/${name}.done" ]]; then
        continue
      fi

      # robust get-pod check with retries to avoid transient oc failures
      pod_exists=false
      for attempt in 1 2 3; do
        if oc get pod "${name}" -n "${NS}" >/dev/null 2>&1; then
          pod_exists=true
          break
        fi
        echo "[${iter}] ${name}: oc get failed (attempt ${attempt}/3), retrying..."
        sleep 1
      done

      if [[ "${pod_exists}" == "false" ]]; then
        echo "[${iter}] ${name}: pod truly missing after retries → treating as flake."
        collect_debug_and_exit "${name}" "${iter}"
      fi

      phase="$(oc get pod "${name}" -n "${NS}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown")"
      reason="$(oc get pod "${name}" -n "${NS}" -o jsonpath='{.status.reason}' 2>/dev/null || echo "")"

      if [[ "${phase}" == "Failed" && "${reason}" == "Evicted" ]]; then
        # success for this pod; log total duration
        now=$(date +%s)
        start_val="$(cat "${tmpdir}/${name}.start" 2>/dev/null || echo "${now}")"
        duration=$(( now - start_val ))
        printf '%s iter=%s pod=%s phase=%s reason=%s duration_s=%s\n' \
          "$(date --iso-8601=seconds)" "${iter}" "${name}" "${phase}" "${reason}" "${duration}" >> "${LOG_FILE}"

        touch "${tmpdir}/${name}.done"
        # Optional cleanup to reduce pressure
        oc delete pod "${name}" -n "${NS}" --ignore-not-found --wait=false >/dev/null 2>&1 || true
      elif [[ "${phase}" == "Succeeded" ]] || \
           [[ "${phase}" == "Failed" && -n "${reason}" && "${reason}" != "Evicted" ]]; then
        echo "[${iter}] ${name}: Unexpected terminal state: phase=${phase} reason=${reason}"
        collect_debug_and_exit "${name}" "${iter}"
      else
        # still active; check timeout
        all_done=false
        now=$(date +%s)
        elapsed=$(( now - start_ts ))
        if (( elapsed > TIMEOUT_SECONDS )); then
          echo "[${iter}] ${name}: Timeout (${TIMEOUT_SECONDS}s) waiting for eviction."
          collect_debug_and_exit "${name}" "${iter}"
        fi
      fi
    done

    if "${all_done}"; then
      break
    fi
    sleep 5
  done

  echo "[${iter}] Batch evicted as expected. Tidying..."
  oc delete pod -n "${NS}" -l "${LABEL_KEY}=true" --ignore-not-found --wait=false >/dev/null 2>&1 || true
  rm -rf "${tmpdir}"
  sleep "${PAUSE_BETWEEN_BATCHES}"
done

echo "All ${ITERATIONS} batch iterations passed with ${PARALLEL} parallel pods each."
exit 0

