#!/bin/bash
# PR #2074 Fencing Test - Run from bastion/jump host

# Note: Don't use 'set -e' - we want to continue through errors and report them

# Configuration - ADJUST THESE
NODE1="192.168.111.20"      # Node to fence
NODE2="192.168.111.21"      # Surviving node
NODE_USER="core"            # SSH username
LOG_FILE="/tmp/pr2074-fence-test-$(date +%Y%m%d-%H%M%S).log"

echo "========================================"
echo "PR #2074 Fencing Test"
echo "========================================"
echo "Node to fence: $NODE1"
echo "Surviving node: $NODE2"
echo "SSH user: $NODE_USER"
echo "Log file: $LOG_FILE"
echo ""

# Helper function
ssh_node() {
    local node=$1
    shift
    ssh ${NODE_USER}@${node} "sudo $@"
}

# Step 1: Verify cluster is healthy
echo "=== Step 1: Verify Initial Cluster State ==="
echo "Checking cluster status..."
ssh_node $NODE1 "pcs status | head -20" | tee -a "$LOG_FILE"
echo ""

# Get node names
NODE1_NAME=$(ssh_node $NODE1 "crm_node -n")
NODE2_NAME=$(ssh_node $NODE2 "crm_node -n")
echo "Node names: $NODE1_NAME, $NODE2_NAME"
echo ""

# Check initial etcd member list (both should be full members)
echo "Initial etcd member list (both nodes should show isLearner: false):"
ssh_node $NODE2 "podman exec etcd etcdctl member list -w table" | tee -a "$LOG_FILE"
echo ""

# Step 2: Verify PR #2074 is installed
echo "=== Step 2: Verify PR #2074 Fix is Installed ==="
if ssh_node $NODE1 "grep -q 'crm_node -l' /usr/lib/ocf/resource.d/heartbeat/podman-etcd"; then
    echo "✓ PR #2074 detected on $NODE1_NAME"
else
    echo "✗ WARNING: PR #2074 may not be installed on $NODE1_NAME"
fi

if ssh_node $NODE2 "grep -q 'crm_node -l' /usr/lib/ocf/resource.d/heartbeat/podman-etcd"; then
    echo "✓ PR #2074 detected on $NODE2_NAME"
else
    echo "✗ WARNING: PR #2074 may not be installed on $NODE2_NAME"
fi
echo ""

# Step 3: Check initial force_new_cluster state (should be clean)
echo "=== Step 3: Check Initial force_new_cluster State ==="
for node in $NODE1 $NODE2; do
    node_name=$(ssh_node $node "crm_node -n")
    echo "Checking $node_name..."
    if ssh_node $node "crm_attribute --query --lifetime reboot --node $node_name --name 'force_new_cluster' 2>/dev/null | grep -q value="; then
        FNC=$(ssh_node $node "crm_attribute --query --lifetime reboot --node $node_name --name 'force_new_cluster' 2>/dev/null")
        echo "  ⚠ UNEXPECTED: $FNC"
    else
        echo "  ✓ Not set (good)"
    fi
done
echo ""

# Step 4: Start log monitoring (in background)
echo "=== Step 4: Starting Log Collection ==="
echo "Collecting logs from $NODE2_NAME (surviving node)..."
ssh_node $NODE2 "journalctl -u pacemaker -f" 2>/dev/null | grep --line-buffered -E "(force_new_cluster|marked to force|join as learner|get_force_new_cluster)" >> "$LOG_FILE" &
LOG_PID=$!
echo "Log monitoring started (PID: $LOG_PID)"
echo ""

# Step 5: Perform the fence
echo "=== Step 5: Fencing $NODE1_NAME ==="
echo "Press Enter to fence $NODE1_NAME, or Ctrl+C to abort..."
read

echo "Executing fence command at $(date)"
ssh_node $NODE2 "pcs stonith fence $NODE1_NAME" | tee -a "$LOG_FILE"
echo "Fence command completed at $(date)"
echo ""

# Wait for fence to complete
echo "Waiting 20 seconds for fence to complete..."
sleep 20

# Step 6: Check cluster status after fence
echo "=== Step 6: Cluster Status After Fence ==="
PCS_STATUS=$(ssh_node $NODE2 "pcs status" | tee -a "$LOG_FILE")
echo "$PCS_STATUS"
echo ""

# Check for ERROR states
if echo "$PCS_STATUS" | grep -i "error"; then
    echo "⚠ WARNING: Detected ERROR in cluster status after fence"
fi

# Check etcd member list after fence
echo "Etcd member list after fence (one node should be offline):"
ssh_node $NODE2 "podman exec etcd etcdctl member list -w table" 2>&1 | tee -a "$LOG_FILE"
echo ""

# Step 7: Monitor force_new_cluster attributes
echo "=== Step 7: Monitoring force_new_cluster Attributes ==="
echo "Checking every 10 seconds for up to 10 minutes..."
echo "Will exit early once surviving node sets force_new_cluster"
echo ""

NODE2_FNC_SET=false
for i in {1..60}; do
    echo "--- Check $i at $(date +%H:%M:%S) ---"
    
    # Check node 2 (surviving)
    if ssh_node $NODE2 "crm_attribute --query --lifetime reboot --node $NODE2_NAME --name 'force_new_cluster' 2>/dev/null | grep -q value="; then
        FNC=$(ssh_node $NODE2 "crm_attribute --query --lifetime reboot --node $NODE2_NAME --name 'force_new_cluster' 2>/dev/null | grep -oP 'value=\"\\K[^\"]+' || echo 'ERROR'")
        echo "  $NODE2_NAME: SET to '$FNC' ✓"
        NODE2_FNC_SET=true
    else
        echo "  $NODE2_NAME: Not set"
    fi
    
    # Check node 1 (fenced) - may not respond yet
    if ssh ${NODE_USER}@${NODE1} "sudo crm_attribute --query --lifetime reboot --node $NODE1_NAME --name 'force_new_cluster' 2>/dev/null | grep -q value=" 2>/dev/null; then
        FNC=$(ssh_node $NODE1 "crm_attribute --query --lifetime reboot --node $NODE1_NAME --name 'force_new_cluster' 2>/dev/null | grep -oP 'value=\"\\K[^\"]+' || echo 'ERROR'")
        echo "  $NODE1_NAME: SET to '$FNC'"
    else
        echo "  $NODE1_NAME: Not set or offline"
    fi
    
    # Exit early if surviving node has set force_new_cluster (expected state)
    if [ "$NODE2_FNC_SET" = true ]; then
        echo ""
        echo "✓ Expected state achieved: $NODE2_NAME has force_new_cluster set"
        echo "Proceeding to wait for $NODE1_NAME recovery..."
        break
    fi
    
    echo ""
    sleep 10
done

if [ "$NODE2_FNC_SET" = false ]; then
    echo "⚠ WARNING: Surviving node ($NODE2_NAME) did not set force_new_cluster within 10 minutes"
    echo "This may indicate the agent did not detect the failure or recovery is not triggering"
fi
echo ""

# Step 8: Wait for node 1 recovery
echo "=== Step 8: Waiting for $NODE1_NAME to Recover ==="
echo "This is the CRITICAL phase for PR #2074!"
echo ""
echo "When $NODE1_NAME comes back, it should:"
echo "  1. Query ALL nodes for force_new_cluster"
echo "  2. See that $NODE2_NAME has it set"
echo "  3. Log: '$NODE1_NAME shall join as learner because force_new_cluster is set on peer $NODE2_NAME'"
echo "  4. Join the cluster as a learner"
echo ""

# Monitor for up to 10 minutes
START_TIME=$(date +%s)
RECOVERED=false
while [ $(($(date +%s) - START_TIME)) -lt 600 ]; do
    # Check if node1 is back online
    PCS_OUTPUT=$(ssh_node $NODE2 "pcs status" 2>/dev/null)
    
    # Try multiple patterns to detect if node is online
    if echo "$PCS_OUTPUT" | grep -i "online:" | grep -q "$NODE1_NAME"; then
        if [ "$RECOVERED" = false ]; then
            echo "✓ $NODE1_NAME is back Online at $(date +%H:%M:%S)"
            RECOVERED=true
            
            # Give it a moment to start etcd
            sleep 10
        fi
        break  # Exit loop once node is detected online
    elif echo "$PCS_OUTPUT" | grep -q "$NODE1_NAME.*Online"; then
        if [ "$RECOVERED" = false ]; then
            echo "✓ $NODE1_NAME is back Online at $(date +%H:%M:%S)"
            RECOVERED=true
            
            # Give it a moment to start etcd
            sleep 10
        fi
        break  # Exit loop once node is detected online
    fi
    
    ELAPSED=$(( $(date +%s) - START_TIME ))
    echo "Waiting for $NODE1_NAME to come online... (${ELAPSED}s elapsed)"
    sleep 10
done

if [ "$RECOVERED" = true ]; then
    echo "✓ $NODE1_NAME recovered"
    
    # Check etcd member list right after node comes back
    echo ""
    echo "Etcd member list immediately after $NODE1_NAME recovery:"
    ssh_node $NODE2 "podman exec etcd etcdctl member list -w table" 2>&1 | tee -a "$LOG_FILE"
    echo ""
else
    echo "⚠ $NODE1_NAME did not come back online within 10 minutes"
    echo "Last pcs status output:"
    ssh_node $NODE2 "pcs status | head -20"
fi
echo ""

# Step 9: Check etcd member list (watch for isLearner status change)
echo "=== Step 9: Monitoring Etcd Member List ==="
echo "Looking for learner promotion (isLearner should change false -> true -> false)..."
for i in {1..5}; do
    echo "--- Check $i at $(date +%H:%M:%S) ---"
    MEMBER_OUTPUT=$(ssh_node $NODE2 "podman exec etcd etcdctl member list -w table" 2>/dev/null)
    echo "$MEMBER_OUTPUT"
    
    # Check if both members show isLearner: false (with proper error handling)
    LEARNER_COUNT=$(echo "$MEMBER_OUTPUT" | grep -c "true" 2>/dev/null)
    if [ -z "$LEARNER_COUNT" ]; then
        LEARNER_COUNT=0
    fi
    
    MEMBER_COUNT=$(echo "$MEMBER_OUTPUT" | grep -c "started" 2>/dev/null)
    if [ -z "$MEMBER_COUNT" ]; then
        MEMBER_COUNT=0
    fi
    
    echo "Status: $MEMBER_COUNT members started, $LEARNER_COUNT learners"
    
    # Only compare if we have valid integers
    if [ "$MEMBER_COUNT" -eq 2 ] 2>/dev/null && [ "$LEARNER_COUNT" -eq 0 ] 2>/dev/null; then
        echo "✓ Both members are full members (isLearner: false) - Recovery complete!"
        break
    fi
    
    echo ""
    sleep 5
done
echo "" | tee -a "$LOG_FILE"

# Step 10: Stop log monitoring
echo "=== Step 10: Stopping Log Collection ==="
kill $LOG_PID 2>/dev/null || true
echo "Log monitoring stopped"
echo ""

# Step 11: Analyze logs for PR #2074 behavior
echo "=== Step 11: PR #2074 Behavior Analysis ==="
echo ""

if grep -q "shall join as learner because force_new_cluster is set on peer" "$LOG_FILE"; then
    echo "✓✓✓ SUCCESS! PR #2074 WORKING CORRECTLY ✓✓✓"
    echo ""
    echo "Found evidence that $NODE1_NAME checked cluster-wide attributes:"
    grep "shall join as learner" "$LOG_FILE" | tail -3
    echo ""
    echo "This proves:"
    echo "  ✓ Node 1 queried ALL nodes (not just local)"
    echo "  ✓ Node 1 detected Node 2 had force_new_cluster set"
    echo "  ✓ Node 1 made correct decision to join as learner"
    echo "  ✓ Split-brain was prevented!"
elif grep -q "marked to force-new-cluster" "$LOG_FILE"; then
    echo "⚠ Found 'marked to force-new-cluster' messages:"
    grep "marked to force-new-cluster" "$LOG_FILE" | tail -5
    echo ""
    echo "This shows which node set force_new_cluster"
elif grep -q "force_new_cluster attribute is set on multiple nodes" "$LOG_FILE"; then
    echo "✓ SPLIT-BRAIN PREVENTION WORKED!"
    echo "PR #2074 detected multiple nodes with attribute and prevented start:"
    grep "multiple nodes" "$LOG_FILE"
else
    echo "⚠ No force_new_cluster log messages found"
    echo "Check if recovery completed or if logs need manual inspection"
fi
echo ""

# Step 12: Final cluster state
echo "=== Step 12: Final Cluster State ==="
FINAL_PCS_STATUS=$(ssh_node $NODE2 "pcs status")
echo "$FINAL_PCS_STATUS" | tee -a "$LOG_FILE"
echo ""

# Check for ERROR or Failed states in final status
RECOVERY_SUCCESS=true
if echo "$FINAL_PCS_STATUS" | grep -iE "(error|failed|stopped)"; then
    echo "✗✗✗ CLUSTER RECOVERY FAILED ✗✗✗"
    echo ""
    echo "Detected issues in final cluster state:"
    echo "$FINAL_PCS_STATUS" | grep -iE "(error|failed|stopped)" | sed 's/^/  /'
    echo ""
    RECOVERY_SUCCESS=false
fi

# Check if both nodes are online
NODE_ONLINE_COUNT=$(echo "$FINAL_PCS_STATUS" | grep -c "Online:" || echo "0")
if [ "$NODE_ONLINE_COUNT" -eq 0 ] || ! echo "$FINAL_PCS_STATUS" | grep "Online:" | grep -q "$NODE1_NAME"; then
    echo "✗ WARNING: $NODE1_NAME is not showing as Online"
    RECOVERY_SUCCESS=false
fi

# Check resource status
if ! echo "$FINAL_PCS_STATUS" | grep -q "etcd.*Started"; then
    echo "✗ WARNING: etcd resources not showing as Started on both nodes"
    RECOVERY_SUCCESS=false
fi

# Final etcd member check
echo "=== Final Etcd Member List (both should be isLearner: false) ==="
FINAL_MEMBER_OUTPUT=$(ssh_node $NODE2 "podman exec etcd etcdctl member list -w table" 2>&1)
echo "$FINAL_MEMBER_OUTPUT" | tee -a "$LOG_FILE"
echo ""

# Verify etcd health
FINAL_MEMBER_COUNT=$(echo "$FINAL_MEMBER_OUTPUT" | grep -c "started" 2>/dev/null)
if [ -z "$FINAL_MEMBER_COUNT" ]; then
    FINAL_MEMBER_COUNT=0
fi

FINAL_LEARNER_COUNT=$(echo "$FINAL_MEMBER_OUTPUT" | grep -c "true" 2>/dev/null)
if [ -z "$FINAL_LEARNER_COUNT" ]; then
    FINAL_LEARNER_COUNT=0
fi

if [ "$FINAL_MEMBER_COUNT" -ne 2 ]; then
    echo "✗ WARNING: Expected 2 etcd members started, found $FINAL_MEMBER_COUNT"
    RECOVERY_SUCCESS=false
fi

if [ "$FINAL_LEARNER_COUNT" -ne 0 ]; then
    echo "✗ WARNING: Still have $FINAL_LEARNER_COUNT learner(s), expected 0"
    RECOVERY_SUCCESS=false
fi

echo "=== Step 13: Final force_new_cluster Check ==="
for node in $NODE1 $NODE2; do
    node_name=$(ssh_node $node "crm_node -n" 2>/dev/null) || node_name="$node"
    echo "Checking $node_name..."
    if ssh_node $node "crm_attribute --query --lifetime reboot --node $node_name --name 'force_new_cluster' 2>/dev/null | grep -q value="; then
        FNC=$(ssh_node $node "crm_attribute --query --lifetime reboot --node $node_name --name 'force_new_cluster' 2>/dev/null")
        echo "  ⚠ Still set: $FNC (should be cleared after recovery)"
    else
        echo "  ✓ Not set (good - attribute was cleared)"
    fi
done
echo ""

echo "========================================"
if [ "$RECOVERY_SUCCESS" = true ]; then
    echo "✓✓✓ Test Complete - RECOVERY SUCCESSFUL ✓✓✓"
else
    echo "✗✗✗ Test Complete - RECOVERY FAILED ✗✗✗"
fi
echo "========================================"
echo ""
echo "Results Summary:"
echo "  Log file: $LOG_FILE"
echo "  Recovery Status: $([ "$RECOVERY_SUCCESS" = true ] && echo "SUCCESS" || echo "FAILED")"
echo ""

if [ "$RECOVERY_SUCCESS" = false ]; then
    echo "⚠⚠⚠ TROUBLESHOOTING REQUIRED ⚠⚠⚠"
    echo ""
    echo "The cluster did not recover successfully. Common issues:"
    echo ""
    echo "1. Check resource failures:"
    echo "   ssh $NODE_USER@$NODE2 'sudo pcs resource failcount show'"
    echo "   ssh $NODE_USER@$NODE2 'sudo pcs resource debug-start etcd-clone'"
    echo ""
    echo "2. Check Pacemaker logs for errors:"
    echo "   ssh $NODE_USER@$NODE2 'sudo journalctl -u pacemaker --since \"15 minutes ago\" | grep -i error'"
    echo ""
    echo "3. Check etcd container status:"
    echo "   ssh $NODE_USER@$NODE1 'sudo podman ps -a | grep etcd'"
    echo "   ssh $NODE_USER@$NODE2 'sudo podman ps -a | grep etcd'"
    echo ""
    echo "4. Check etcd logs:"
    echo "   ssh $NODE_USER@$NODE2 'sudo podman logs etcd'"
    echo ""
    echo "5. Try cleanup and restart:"
    echo "   ssh $NODE_USER@$NODE2 'sudo pcs resource cleanup etcd-clone'"
    echo "   ssh $NODE_USER@$NODE2 'sudo pcs resource restart etcd-clone'"
    echo ""
fi
echo "PR #2074 Verification Checklist:"
echo "  [ ] $NODE1_NAME checked ALL nodes (not just local)"
echo "  [ ] $NODE1_NAME detected $NODE2_NAME had force_new_cluster"
echo "  [ ] $NODE1_NAME joined as learner (not force new cluster)"
echo "  [ ] No split-brain occurred"
echo "  [ ] Cluster recovered successfully"
echo "  [ ] force_new_cluster attributes cleared"
echo "  [ ] Both etcd members show isLearner: false (full members)"
echo ""
echo "Etcd Member Status Expected:"
echo "  Initial:  Both nodes - isLearner: false (full members)"
echo "  After fence: One node offline, one online"
echo "  Recovery: $NODE1_NAME briefly shows isLearner: true (joining)"
echo "  Final:    Both nodes - isLearner: false (full members restored)"
echo ""
echo "Review full logs:"
echo "  cat $LOG_FILE"
echo ""
echo "Review Pacemaker logs on nodes:"
echo "  ssh $NODE_USER@$NODE2 'sudo journalctl -u pacemaker --since \"10 minutes ago\" | grep force_new_cluster'"

# Exit with appropriate code
if [ "$RECOVERY_SUCCESS" = true ]; then
    exit 0
else
    exit 1
fi
