#!/bin/bash
#
# test_pr2105_config_error_local.sh - Test OCF_ERR_CONFIGURED for config errors (LOCAL)
#
# Purpose: Create a real configuration error and verify that:
#          1. The error returns OCF_ERR_CONFIGURED (exit code 6)
#          2. Pacemaker doesn't retry the start
#          3. The error appears in pacemaker logs with correct exit code
#
# Usage: Copy to 192.168.111.20 and run as root:
#        scp test_pr2105_config_error_local.sh core@192.168.111.20:
#        ssh core@192.168.111.20
#        sudo ./test_pr2105_config_error_local.sh
#
# JIRA: https://issues.redhat.com/browse/OCPEDGE-2231
# PR: https://github.com/ClusterLabs/resource-agents/pull/2105

set -euo pipefail

# Must run as root
if [ "$EUID" -ne 0 ]; then
    echo "ERROR: This script must be run as root (use sudo)"
    exit 1
fi

# Configuration - Get actual node names from cluster
NODE1=$(crm_node -l | grep -E '192\.168\.111\.20|master-0' | awk '{print $2}')
NODE2=$(crm_node -l | grep -E '192\.168\.111\.21|master-1' | awk '{print $2}')

# Fallback to common names if crm_node doesn't work
: ${NODE1:=master-0}
: ${NODE2:=master-1}

# OCF Return Codes
OCF_SUCCESS=0
OCF_ERR_GENERIC=1
OCF_ERR_CONFIGURED=6

# Locate pacemaker log file
PACEMAKER_LOG=""
for logpath in /var/log/pacemaker/pacemaker.log /var/log/cluster/pacemaker.log; do
    if [ -f "$logpath" ]; then
        PACEMAKER_LOG="$logpath"
        break
    fi
done

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Track overall pass/fail
TESTS_PASSED=0
TESTS_FAILED=0

echo "============================================================"
echo "PR #2105 Configuration Error Test (LOCAL)"
echo "============================================================"
echo "Testing: Config error returns OCF_ERR_CONFIGURED (exit 6)"
echo "Running on: $(hostname -I | awk '{print $1}')"
echo "Node1: $NODE1"
echo "Node2: $NODE2"
echo "Log file: ${PACEMAKER_LOG:-<not found, will use journalctl>}"
echo "Date: $(date)"
echo ""

# Get OCF resource config from pacemaker
AGENT_PATH="/usr/lib/ocf/resource.d/heartbeat/podman-etcd"
NODE_IP_MAP=$(pcs resource config etcd 2>/dev/null | grep node_ip_map | awk -F= '{print $2}' | xargs || echo "master-0:192.168.111.20;master-1:192.168.111.21")
DROP_IN=$(pcs resource config etcd 2>/dev/null | grep drop_in_dependency | awk -F= '{print $2}' | xargs || echo "true")

echo "Agent: $AGENT_PATH"
echo "node_ip_map: $NODE_IP_MAP"
echo ""

# Function to check cluster status
check_etcd_status() {
    local etcd_line
    etcd_line=$(pcs status 2>/dev/null | grep -A1 'Clone Set: etcd-clone' | grep 'Started:' || true)
    if [ -n "$etcd_line" ]; then
        echo "$etcd_line" | grep -o 'master-[0-9]' | wc -l
    else
        echo "0"
    fi
}

# Function to cleanup
cleanup() {
    echo ""
    echo "Cleaning up..."
    crm_attribute --lifetime reboot --node ${NODE1} --name force_new_cluster --delete 2>/dev/null || true
    crm_attribute --lifetime reboot --node ${NODE2} --name force_new_cluster --delete 2>/dev/null || true

    # Do NOT re-enable etcd-clone here. Re-enabling spawns an agent with a
    # 10-minute learner timeout that would interfere with subsequent test runs.
    # The operator should manually restore etcd after the test:
    #   sudo pcs resource cleanup etcd-clone && sudo pcs resource enable etcd-clone
    echo "Cleanup complete"
    echo ""
    echo "NOTE: etcd-clone is left disabled. To restore etcd:"
    echo "  sudo pcs resource cleanup etcd-clone"
    echo "  sudo pcs resource enable etcd-clone"
}

trap cleanup EXIT

# ============================================================
# PART A: Direct OCF Agent Test
# ============================================================
# Directly invoke the resource agent to verify exit code 6.
# This bypasses pacemaker orchestration for a reliable test.

echo "============================================================"
echo "PART A: Direct OCF Agent Invocation"
echo "============================================================"
echo ""

echo -e "${BLUE}A1: Verify detection logic (get_force_new_cluster)${NC}"
echo "--------------------------------------"

# Set force_new_cluster on BOTH nodes (invalid config)
crm_attribute --lifetime reboot --node ${NODE1} --name force_new_cluster --update ${NODE1} 2>/dev/null
crm_attribute --lifetime reboot --node ${NODE2} --name force_new_cluster --update ${NODE2} 2>/dev/null
sleep 2

# Replicate get_force_new_cluster logic exactly
holders=""
nodes=$(crm_node -l | awk '{print $2}')
for node in $nodes; do
    value=$(crm_attribute --query --lifetime reboot --name "force_new_cluster" --node "$node" 2>/dev/null | awk -F'value=' '{print $2}' | tr -d "'" || true)
    if [ -n "$value" ]; then
        holders="$holders$node "
    fi
done
holder_count=$(echo "$holders" | wc -w)

echo "force_new_cluster holders: [$holders]"
echo "holder_count: $holder_count"

if [ "$holder_count" -gt 1 ]; then
    echo -e "${GREEN}PASS${NC}: Detection logic finds $holder_count holders (>1 triggers OCF_ERR_CONFIGURED)"
    TESTS_PASSED=$((TESTS_PASSED + 1))
else
    echo -e "${RED}FAIL${NC}: Detection logic found $holder_count holders (expected >1)"
    TESTS_FAILED=$((TESTS_FAILED + 1))
fi

echo ""

echo -e "${BLUE}A2: Direct agent start (should return exit code 6)${NC}"
echo "--------------------------------------"

# Ensure etcd-clone is disabled so the direct invocation doesn't conflict
pcs resource disable etcd-clone 2>&1 || true
pcs resource cleanup etcd-clone 2>&1 || true
sleep 3

# Wait for any running etcd containers to stop
for i in $(seq 1 30); do
    if ! crictl ps 2>/dev/null | grep -q etcd && ! podman ps 2>/dev/null | grep -q etcd; then
        break
    fi
    sleep 1
done

echo "Invoking agent directly: $AGENT_PATH start"
echo "(With force_new_cluster set on both nodes)"
echo ""

# Set up OCF environment and invoke the agent directly.
# The agent will reach the force_new_cluster check and return 6.
set +e
OCF_ROOT=/usr/lib/ocf \
OCF_RESOURCE_INSTANCE=etcd \
OCF_RESKEY_node_ip_map="$NODE_IP_MAP" \
OCF_RESKEY_drop_in_dependency="$DROP_IN" \
OCF_RESKEY_CRM_meta_clone=0 \
OCF_RESKEY_CRM_meta_clone_max=2 \
OCF_RESKEY_CRM_meta_clone_node_max=1 \
OCF_RESKEY_CRM_meta_notify=true \
OCF_RESKEY_CRM_meta_notify_active_resource="" \
OCF_RESKEY_CRM_meta_notify_stop_resource="" \
OCF_RESKEY_CRM_meta_notify_start_resource="" \
timeout 120 "$AGENT_PATH" start 2>&1
AGENT_RC=$?
set -e

echo ""
echo "Agent exit code: $AGENT_RC"

if [ "$AGENT_RC" -eq 6 ]; then
    echo -e "${GREEN}PASS${NC}: Agent returned exit code 6 (OCF_ERR_CONFIGURED)"
    echo "  This confirms PR #2105 correctly handles config errors"
    TESTS_PASSED=$((TESTS_PASSED + 1))
elif [ "$AGENT_RC" -eq 1 ]; then
    echo -e "${RED}FAIL${NC}: Agent returned exit code 1 (OCF_ERR_GENERIC)"
    echo "  PR #2105 may not be present or the code path was not reached"
    TESTS_FAILED=$((TESTS_FAILED + 1))
elif [ "$AGENT_RC" -eq 124 ]; then
    echo -e "${RED}FAIL${NC}: Agent timed out (did not reach force_new_cluster check)"
    echo "  The agent may be stuck waiting for pod/manifest"
    TESTS_FAILED=$((TESTS_FAILED + 1))
else
    echo -e "${YELLOW}UNEXPECTED${NC}: Agent returned exit code $AGENT_RC"
    TESTS_FAILED=$((TESTS_FAILED + 1))
fi

echo ""

# Clean attributes before Part B
crm_attribute --lifetime reboot --node ${NODE1} --name force_new_cluster --delete 2>/dev/null || true
crm_attribute --lifetime reboot --node ${NODE2} --name force_new_cluster --delete 2>/dev/null || true

# ============================================================
# PART B: Pacemaker Integration Test
# ============================================================
# Verify pacemaker handles exit code 6 correctly (no retry).
# This requires the agent to actually fail through pacemaker.

echo "============================================================"
echo "PART B: Pacemaker Integration Test"
echo "============================================================"
echo ""

echo -e "${BLUE}B1: Prepare cluster state${NC}"
echo "--------------------------------------"

# Clear fail counts from Part A
pcs resource cleanup etcd-clone 2>&1 || true
sleep 2

# Set invalid config
echo "Setting force_new_cluster on both nodes..."
crm_attribute --lifetime reboot --node ${NODE1} --name force_new_cluster --update ${NODE1} 2>/dev/null
crm_attribute --lifetime reboot --node ${NODE2} --name force_new_cluster --update ${NODE2} 2>/dev/null
sleep 3

# Verify
FNC1=$(crm_attribute --lifetime reboot --query --node ${NODE1} --name force_new_cluster 2>/dev/null | awk -F'value=' '{print $2}' | tr -d "'" | xargs || echo "")
FNC2=$(crm_attribute --lifetime reboot --query --node ${NODE2} --name force_new_cluster 2>/dev/null | awk -F'value=' '{print $2}' | tr -d "'" | xargs || echo "")
echo "${NODE1}: ${FNC1:-<not set>}"
echo "${NODE2}: ${FNC2:-<not set>}"

if [ -z "$FNC1" ] || [ -z "$FNC2" ]; then
    echo -e "${RED}Failed to set attributes, skipping Part B${NC}"
else
    echo -e "${GREEN}Attributes confirmed${NC}"
    echo ""

    echo -e "${BLUE}B2: Enable etcd-clone and monitor pacemaker response${NC}"
    echo "--------------------------------------"

    # Record log position
    LOG_OFFSET=0
    if [ -n "$PACEMAKER_LOG" ]; then
        LOG_OFFSET=$(wc -c < "$PACEMAKER_LOG" 2>/dev/null || echo 0)
    fi
    LOG_TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')

    echo "Enabling etcd-clone..."
    pcs resource enable etcd-clone 2>&1 || true

    # The agent's start has several pre-checks (pod wait, cert check, manifest check)
    # before reaching the force_new_cluster validation. Wait long enough.
    echo "Waiting 60 seconds for agent to complete start attempt..."
    echo "(Agent pre-checks: pod wait, certs, manifest, then force_new_cluster check)"
    for i in $(seq 1 12); do
        sleep 5
        # Check if the agent has already finished (logged a result)
        if [ -n "$PACEMAKER_LOG" ]; then
            NEW_LOGS=$(tail -c +"$((LOG_OFFSET + 1))" "$PACEMAKER_LOG" 2>/dev/null | grep -E 'etcd.*start.*exited with status' || true)
            if [ -n "$NEW_LOGS" ]; then
                echo "Agent completed after ~$((i * 5))s"
                break
            fi
        fi
    done

    echo ""

    echo -e "${BLUE}B3: Check pacemaker logs${NC}"
    echo "--------------------------------------"

    # Search new log entries
    search_new() {
        if [ -n "$PACEMAKER_LOG" ]; then
            tail -c +"$((LOG_OFFSET + 1))" "$PACEMAKER_LOG" 2>/dev/null | grep -E "$1" || true
        else
            journalctl -u pacemaker --since "$LOG_TIMESTAMP" --no-pager 2>/dev/null | grep -E "$1" || true
        fi
    }

    # Show all etcd start results
    echo "Recent etcd start results:"
    echo "---"
    search_new 'etcd.*(start.*exited|Result of start)' | tail -10
    echo "---"
    echo ""

    # Check for exit code 6
    RC6_EXECD=$(search_new 'etcd.*start.*exited with status 6')
    RC6_CONTROLD=$(search_new 'etcd.*Not configured.*rc=6|Result of start.*etcd.*Not configured')
    RC6_REASON=$(search_new 'force_new_cluster.*multiple nodes')

    if [ -n "$RC6_EXECD" ] || [ -n "$RC6_CONTROLD" ] || [ -n "$RC6_REASON" ]; then
        echo -e "${GREEN}PASS${NC}: Pacemaker recorded exit code 6 (OCF_ERR_CONFIGURED)"
        [ -n "$RC6_EXECD" ] && echo "  execd: $(echo "$RC6_EXECD" | tail -1)"
        [ -n "$RC6_CONTROLD" ] && echo "  controld: $(echo "$RC6_CONTROLD" | tail -1)"
        [ -n "$RC6_REASON" ] && echo "  reason: $(echo "$RC6_REASON" | tail -1)"
        TESTS_PASSED=$((TESTS_PASSED + 1))

        # Check no-retry behavior
        echo ""
        RETRY_COUNT=$(search_new 'etcd.*start.*exited with status' | wc -l)
        echo "Start attempts: $RETRY_COUNT"
        if [ "$RETRY_COUNT" -le 2 ]; then
            echo -e "${GREEN}PASS${NC}: No excessive retries (pacemaker respects OCF_ERR_CONFIGURED)"
            TESTS_PASSED=$((TESTS_PASSED + 1))
        else
            echo -e "${RED}FAIL${NC}: $RETRY_COUNT retries detected (pacemaker should not retry)"
            TESTS_FAILED=$((TESTS_FAILED + 1))
        fi
    else
        # Check if agent is still running (pre-checks taking long)
        STILL_RUNNING=$(search_new 'etcd.*start.*exited' || true)
        if [ -z "$STILL_RUNNING" ]; then
            echo -e "${YELLOW}INCONCLUSIVE${NC}: Agent has not completed yet"
            echo "  The agent's pre-checks (pod wait, manifest) may still be running"
            echo "  Try waiting longer or check: sudo pcs status"
            echo ""
            echo "  Current new log entries:"
            echo "  ---"
            search_new 'etcd' | tail -10
            echo "  ---"
        else
            RC1_HIT=$(search_new 'etcd.*start.*exited with status 1|etcd.*rc=1')
            if [ -n "$RC1_HIT" ]; then
                echo -e "${RED}FAIL${NC}: Agent returned exit code 1 instead of 6"
                echo "  $(echo "$RC1_HIT" | tail -1)"
            else
                echo -e "${RED}FAIL${NC}: Unexpected agent result"
                echo "  $(echo "$STILL_RUNNING" | tail -1)"
            fi
            TESTS_FAILED=$((TESTS_FAILED + 1))
        fi
    fi
fi

echo ""
echo "============================================================"
echo "Test Summary"
echo "============================================================"
echo ""
echo "Passed: $TESTS_PASSED"
echo "Failed: $TESTS_FAILED"
echo ""

if [ "$TESTS_FAILED" -eq 0 ] && [ "$TESTS_PASSED" -gt 0 ]; then
    echo -e "${GREEN}OVERALL: PASS${NC}"
    echo "PR #2105 is correctly returning OCF_ERR_CONFIGURED (exit 6)"
    echo "for configuration errors."
else
    echo -e "${RED}OVERALL: FAIL${NC}"
    echo "See individual test results above for details."
fi

echo ""
echo "Post-test: Cleanup runs automatically."
echo "  If etcd doesn't recover: sudo pcs resource cleanup etcd-clone"
echo ""
