#!/bin/bash
# This script is provided by Habana Taiwan Software Team
# To enable Dell I2C access, need to enable I2C acess from iDRAC, command: "racadm set iDRAC.Security.AllowIpmiI2cCommands 1"
# This will not works on official release

LOGPATH="$HOME/habana-diag-log"
DATESTR=`date +%Y%m%d%H%M`

check_hl_packages()
{
    echo -e "Kernel Version:"
    uname -a
    echo -e "\nOS Version:"
    cat /etc/os-release
    echo -e "\nHabana packages:"
    dpkg -l | grep -i habana
    echo -e "\nOpenMPI packages:"
    dpkg -l | grep -i openmpi
    echo -e "\nMemory info:"
    cat /proc/meminfo
}

check_hl_smi()
{
    echo "Driver status:"
    for i in {0..7}; do echo  "accel$i: " `cat /sys/class/accel/accel$i/device/status` ; done
    echo "OAM map:"
    hl-smi -Q bus_id,module_id,serial -f csv
    echo "FW versions:"
    hl-smi --fw-version
    echo "Detail:"
    hl-smi -q
    echo "Internal port stats:"
    OAM_ADDR=`lspci | grep accel | awk '{print $1}'`
    for i in $OAM_ADDR; do
		echo "OAM $i"
		hl-smi -n link -i $i
    done

    echo "External port stats:"
    /opt/habanalabs/qual/gaudi3/bin/manage_network_ifs.sh --status
}


dell_dump_cpld()
{
    echo "UBB Pri CPLD dump"
    echo "     0   1   2   3   4   5   6   7   8   9   a   b   c   d   e   f"
    for i in {0..15}; do
	ROW=$(echo "obase=16;$i" | bc)
	echo -n "$ROW: "
	for j in {0..15}; do
	    ADDR=$(echo "obase=16;$((i*16+j))" | bc)
	    REG=`ipmitool raw 0x06 0x52 0x3c 0x9e 0x01 0x$ADDR`
	    echo -n "$REG "
	done
	echo ""
    done

    for k in {0..7}; do
	echo "OAM $k CPLD dump"
	echo "     0   1   2   3   4   5   6   7   8   9   a   b   c   d   e   f"
	for i in {0..15}; do
	    ROW=$(echo "obase=16;$i" | bc)
	    echo -n "$ROW: "
	    for j in {0..15}; do
		ADDR=$(echo "obase=16;$((i*16+j))" | bc)
		BUS=$((k+44))
		REG=`ipmitool raw 0x06 0x52 $BUS 0x4a 0x01 0x$ADDR`
		echo -n "$REG "
	    done
	    echo ""
	done
    done
}

dump_cpld()
{
    # Need to customized for customers
    echo "Dump UBB and OAM CPLD"
    dell_dump_cpld
}

check_pci()
{
    PCI_STR=`lspci -vvv`
    NUM_OAMS=`echo "$PCI_STR" | grep accel | wc | awk '{print $1}'`

    if [ $NUM_OAMS -eq 0 ]; then
	echo "No OAM found!!"
	dump_cpld | tee $LOGPATH/cpld-$DATESTR.log
	exit
    fi

    echo "There are $NUM_OAMS found in the system"
    echo  "$PCI_STR" | grep accel
    
    echo "PCI detail:"
    echo "$PCI_STR"
}

capture_oam_uart()
{
    OAM_ADDR=`lspci | grep accel | awk '{print $1}'`
    for addr in $OAM_ADDR; do
        echo "Enable virtual console for $addr"
        hl-smi dmon -i 0000:$addr > $LOGPATH/hl-smi-tmp.log
        cat $LOGPATH/hl-smi-tmp.log
        oam=`hl-smi -Q bus_id,module_id -f csv | grep $addr | awk '{print $2}'`
        PTS=`cat $LOGPATH/hl-smi-tmp.log | grep "ARC1" | awk -F: '{print $2}'`
        case $oam in
            ''|*[!0-9]*)
                echo "Log $addr virtual console from $PTS"
                cat $PTS > $LOGPATH/OAM-$addr-UART-$DATESTR.log&
                ;;
            *)
                echo "Log OAM $oam virtual console from $PTS"
                cat $PTS > $LOGPATH/OAM$oam-UART-$DATESTR.log&
                ;;
        esac
    done
    rm $LOGPATH/hl-smi-tmp.log
    echo "Unload driver to capture OAM UART LOG"
    rmmod habanalabs
    rmmod habanalabs_en
    rmmod habanalabs_cn
    rmmod habanalabs_ib
    echo "Reset OAM to capture OAM UART LOG"
    for addr in $OAM_ADDR; do
	hl-fw-loader -R -y -d $addr
    done
    echo "Load driver to capture OAM UART LOG"
    modprobe habanalabs_en
    modprobe habanalabs_cn
    modprobe habanalabs_ib
    modprobe habanalabs

    echo "Stop capturing the OAM UART log"
    CATS=`ps auxw |grep cat | grep "dev/pts" | awk '{print $2}'`
    for i in $CATS; do
	kill -9 $i
    done
    
    killall -9 hl-smi
}

run()
{
    check_hl_packages | tee $LOGPATH/hl-packages-$DATESTR.log
    check_pci | tee $LOGPATH/pci-$DATESTR.log
    dump_cpld | tee $LOGPATH/cpld-$DATESTR.log
    echo "Load drivers..."
    modprobe habanalabs_en
    modprobe habanalabs_cn
    modprobe habanalabs_ib
    modprobe habanalabs
    sleep 30
    check_hl_smi | tee $LOGPATH/hl-smi-$DATESTR.log
    capture_oam_uart | tee $LOGPATH/capture-uart-$DATESTR.log
    dmesg > $LOGPATH/dmesg-$DATESTR.log
}

rm -rf $LOGPATH
mkdir -p $LOGPATH

if [ "$EUID" -ne 0 ]; then
    echo "Please run as root"
    exit
fi

#run
dump_cpld

#tar jcvf $HOME/hl-diag-log-$DATESTR.tar.bz2 $LOGPATH
rm -rf $LOGPATH
echo -e "\n********************************************"
echo "* Finished"
echo -e "\n********************************************\n"
echo -e "Please send $HOME/hl-diag-log-$DATESTR.tar.bz2 to Habana team for analysis\n\n"
