#!/bin/bash
#########################################################################
# Script for check HDD & RAID state #
# Created by Pavol Macko - DoIT #
# ver 0.7 #
#########################################################################
#---------------------------- VARIABLES --------------------------------#
REP=/tmp/hdd.rep
WEB=/var/www/hdd
TMP=/tmp/hdd.tmp
LOG=/tmp/hdd.log
STATE=/tmp/hdd.state
MAIL_TO="info@doit.sk"
EMAIL_ALERT=0
EMAIL_BODY=""
mkdir -p $WEB
rm -f $TMP $LOG $REP
#----------------------------- WEB-START -------------------------------#
webstart(){
cat > $WEB/hdd.html <<\EOF
SITE - Storage
RAID Storage Status on [SITE]
Number of disks: [NUM] | Last update: DATE
HW RAID Controller
CTRL_INFO
BBU / CacheVault
BBU_INFO
Background Tasks
TASK_INFO
Logical Arrays (RAID)
RAID_INFO
| Slot |
HDD-Model |
Serial |
Size |
Type |
Temp |
Media Err |
Pred Fail |
EOF
}
#--------------------------- CHECK HDD INFO ----------------------------#
checkhdd(){
echo -e "- \e[1;32mExtracting\e[0m HDD info from PVE hypervisor"
CON=$(megacli -ShowSummary -aALL 2>/dev/null | grep "ProductName" | head -n1 | awk -F':' '{print $2}' | xargs)
BBU_RAW=$(megacli -AdpBbuCmd -aALL 2>/dev/null)
if echo "$BBU_RAW" | grep -qi "Get BBU Status Failed\|Adapter.*Has No BBU"; then
BBU_HTML="N/A (No BBU)"
else
BBU_STATE=$(echo "$BBU_RAW" | grep "Battery State" | head -n1 | awk -F':' '{print $2}' | xargs)
BBU_TEMP=$(echo "$BBU_RAW" | grep "Temperature" | head -n1 | awk -F':' '{print $2}' | xargs)
[ -z "$BBU_STATE" ] && BBU_STATE="Optimal"
if [[ "$BBU_STATE" == "Optimal" || "$BBU_STATE" == "Operational" ]]; then
BBU_HTML="$BBU_STATE (${BBU_TEMP:-N/A})"
else
BBU_HTML="${BBU_STATE} (${BBU_TEMP:-N/A})"
EMAIL_ALERT=1
EMAIL_BODY="${EMAIL_BODY}\n- BBU/CacheVault error: State=$BBU_STATE, Temp=$BBU_TEMP"
fi
fi
REBUILD_INFO=""
megacli -PDList -aALL 2>/dev/null > /tmp/pd_task.tmp
csplit -s -f /tmp/pd_tblock_ /tmp/pd_task.tmp '/Device Id:/' '{*}' 2>/dev/null
for block in /tmp/pd_tblock_*; do
[ -f "$block" ] || continue
if grep -q "Firmware state" "$block"; then
F_STATE=$(grep "Firmware state" "$block" | awk -F':' '{print $2}' | xargs)
if echo "$F_STATE" | grep -qi "Rebuild"; then
DEV_ID=$(grep "Device Id" "$block" | awk -F':' '{print $2}' | xargs)
ENC_ID=$(grep "Enclosure Device ID" "$block" | awk -F':' '{print $2}' | xargs)
PROG=$(megacli -PDRbld -ShowProg -PhysDrv "[$ENC_ID:$DEV_ID]" -aALL 2>/dev/null | grep -i "Completed" | awk -F'=' '{print $2}' | xargs)
REBUILD_INFO="Rebuild Dev $DEV_ID: ${PROG:-In Progress}"
break
fi
fi
rm -f "$block"
done
rm -f /tmp/pd_task.tmp /tmp/pd_tblock_*
if [ -n "$REBUILD_INFO" ]; then
TASK_HTML="$REBUILD_INFO"
else
TASK_HTML="Idle"
fi
echo "--------------------------------------" > /tmp/hdd.rac
echo -e "\e[1;37mHW-RAID-CONTROLLER\e[0m" >> /tmp/hdd.rac
echo "--------------------------------------" >> /tmp/hdd.rac
echo -e "\e[1;36m${CON:-N/A}\e[0m" >> /tmp/hdd.rac
echo "------ ---------- --------" > /tmp/hdd.rai
echo -e "\e[1;37mTYPE SIZE STATUS\e[0m" >> /tmp/hdd.rai
echo "------ ---------- --------" >> /tmp/hdd.rai
RAID_HTML_ACC=""
megacli -LDInfo -Lall -aALL 2>/dev/null > /tmp/ld.tmp
csplit -s -f /tmp/ld_block_ /tmp/ld.tmp '/Virtual Drive:/' '{*}' 2>/dev/null
for block in /tmp/ld_block_*; do
[ -f "$block" ] || continue
if grep -q "RAID Level" "$block"; then
RAI=$(grep -E "Name|RAID Level" "$block" | grep -v "User Name" | head -n1 | awk -F':' '{print $2}' | xargs)
SIZ=$(grep "Size" "$block" | grep -vE "Sector|Strip" | awk -F':' '{print $2}' | xargs | tr -d ' ')
STA=$(grep "State" "$block" | awk -F':' '{print $2}' | xargs)
echo -e "\e[1;36m${RAI:-LD} ${SIZ:-N/A} ${STA:-N/A}\e[0m" >> /tmp/hdd.rai
if [[ "$STA" == "Optimal" ]]; then
STA_HTML="$STA"
else
STA_HTML="$STA"
EMAIL_ALERT=1
EMAIL_BODY="${EMAIL_BODY}\n- RAID array ${RAI:-LD} bad state: $STA"
fi
RAID_HTML_ACC="${RAID_HTML_ACC}${RAI:-LD} (${SIZ:-N/A}) ${STA_HTML}
"
fi
rm -f "$block"
done
rm -f /tmp/ld.tmp
echo "------ ---------- --------" >> /tmp/hdd.rai
echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" > $REP
echo -e "\e[1;37mSLOT HDD-MODEL SERIAL SIZE TYPE TEMP MEDIA-ERR FAIL\e[0m" >> $REP
echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
megacli -PDList -aALL 2>/dev/null > /tmp/hdd.tmp
csplit -s -f /tmp/pd_block_ /tmp/hdd.tmp '/Device Id:/' '{*}' 2>/dev/null
for block in /tmp/pd_block_*; do
[ -f "$block" ] || continue
SN=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $NF}')
if [ -n "$SN" ]; then
MODL=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $1"-"$2}')
SIZE=$(grep "Raw Size" "$block" | awk -F':' '{print $2}' | awk '{print $1$2}')
SLOT=$(grep "Slot Number" "$block" | awk -F':' '{print $2}' | xargs)
FAIL=$(grep "Predictive Failure Count" "$block" | awk -F':' '{print $2}' | xargs)
MEDIA_ERR=$(grep "Media Error Count" "$block" | awk -F':' '{print $2}' | xargs)
TEMP=$(grep "Drive Temperature" "$block" | awk -F':' '{print $2}' | awk '{print $1}')
M_TYPE_RAW=$(grep "Media Type" "$block" | awk -F':' '{print $2}' | xargs)
I_TYPE_RAW=$(grep "PD Type" "$block" | awk -F':' '{print $2}' | xargs)
if echo "$M_TYPE_RAW" | grep -qi "Solid State"; then
M_TYPE="SSD"
else
M_TYPE="HDD"
fi
if echo "$I_TYPE_RAW" | grep -qi "SATA"; then
I_TYPE="SATA"
else
I_TYPE="SAS"
fi
BUS_TYPE="${I_TYPE}/${M_TYPE}"
[ -z "$SLOT" ] && SLOT="N/A"
[ -z "$FAIL" ] && FAIL="0"
[ -z "$MEDIA_ERR" ] && MEDIA_ERR="0"
[ -z "$TEMP" ] && TEMP="N/A"
if [[ "$FAIL" == "0" ]]; then
FAIL_HTML="0"
else
FAIL_HTML="$FAIL"
fi
if [[ "$MEDIA_ERR" == "0" ]]; then
MEDIA_HTML="0"
else
MEDIA_HTML="$MEDIA_ERR"
fi
# Zachytenie chyby disku pre e-mail
if [[ "$FAIL" != "0" || "$MEDIA_ERR" != "0" ]]; then
EMAIL_ALERT=1
EMAIL_BODY="${EMAIL_BODY}\n- Disk Slot $SLOT ($MODL, SN: $SN): Media Errors = $MEDIA_ERR, Predictive Failures = $FAIL"
fi
cat >> $WEB/hdd.html <
$SLOT |
$MODL |
$SN |
$SIZE |
$BUS_TYPE |
$TEMP |
$MEDIA_HTML |
$FAIL_HTML |
EOF
echo "$SLOT $MODL $SN $SIZE $BUS_TYPE $TEMP $MEDIA_ERR $FAIL" >> $REP
fi
rm -f "$block"
done
echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
rm -f /tmp/hdd.tmp
}
#------------------------------ WEB-STOP -------------------------------#
webend(){
cat >> $WEB/hdd.html <
EOF
NUM=$(grep -vE "SERIAL|---" $REP | wc -l)
SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}')
sed -i "s/\[NUM\]/$NUM/g" $WEB/hdd.html
sed -i "s/DATE/$(date '+%Y-%m-%d %H:%M:%S')/g" $WEB/hdd.html
sed -i "s/SITE/$SITE/g" $WEB/hdd.html
sed -i "s|CTRL_INFO|${CON:-N/A}|g" $WEB/hdd.html
SAFE_BBU=$(echo "$BBU_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
SAFE_TASK=$(echo "$TASK_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
SAFE_RAID=$(echo "$RAID_HTML_ACC" | sed -e 's/[]\/$*.^[]/\\&/g')
sed -i "s|BBU_INFO|$SAFE_BBU|g" $WEB/hdd.html
sed -i "s|TASK_INFO|$SAFE_TASK|g" $WEB/hdd.html
sed -i "s|RAID_INFO|$SAFE_RAID|g" $WEB/hdd.html
}
#------------------------------- CORE ----------------------------------#
webstart
checkhdd
webend
IP=$(ip route get 1.1.1.1 2>/dev/null | awk '{print $7}')
echo -e "- Final HTML report (\e[1;35mhttp://${IP:-localhost}/hdd/hdd.html\e[0m)"
cat /tmp/hdd.rac
cat /tmp/hdd.rai | column -t
cat $REP | column -t
#------------------------------ NOTIFICATION ---------------------------#
if [ "$EMAIL_ALERT" -eq 1 ]; then
if [ ! -f "$STATE" ]; then
SUBJECT="[CRITICAL] Storage Error on $SITE ($IP)"
echo -e "On $SITE was detected errors:\n$EMAIL_BODY\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
touch "$STATE"
fi
else
if [ -f "$STATE" ]; then
SUBJECT="[RECOVERY] Storage OK on $SITE ($IP)"
echo -e "All drives and RAID array on $SITE are optimal state.\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
rm -f "$STATE"
fi
fi
EOGTR