#!/bin/bash ######################################################################### # Script for check HDD & RAID state # # Created by Pavol Macko - DoIT # # ver 0.7 # ######################################################################### #---------------------------- VARIABLES --------------------------------# REP=/tmp/hdd.rep WEB=/var/www/hdd TMP=/tmp/hdd.tmp LOG=/tmp/hdd.log STATE=/tmp/hdd.state MAIL_TO="info@doit.sk" EMAIL_ALERT=0 EMAIL_BODY="" mkdir -p $WEB rm -f $TMP $LOG $REP #----------------------------- WEB-START -------------------------------# webstart(){ cat > $WEB/hdd.html <<\EOF SITE - Storage

RAID Storage Status on [SITE]

Number of disks: [NUM] | Last update: DATE
HW RAID Controller
CTRL_INFO
BBU / CacheVault
BBU_INFO
Background Tasks
TASK_INFO
Logical Arrays (RAID)
RAID_INFO
EOF } #--------------------------- CHECK HDD INFO ----------------------------# checkhdd(){ echo -e "- \e[1;32mExtracting\e[0m HDD info from PVE hypervisor" CON=$(megacli -ShowSummary -aALL 2>/dev/null | grep "ProductName" | head -n1 | awk -F':' '{print $2}' | xargs) BBU_RAW=$(megacli -AdpBbuCmd -aALL 2>/dev/null) if echo "$BBU_RAW" | grep -qi "Get BBU Status Failed\|Adapter.*Has No BBU"; then BBU_HTML="N/A (No BBU)" else BBU_STATE=$(echo "$BBU_RAW" | grep "Battery State" | head -n1 | awk -F':' '{print $2}' | xargs) BBU_TEMP=$(echo "$BBU_RAW" | grep "Temperature" | head -n1 | awk -F':' '{print $2}' | xargs) [ -z "$BBU_STATE" ] && BBU_STATE="Optimal" if [[ "$BBU_STATE" == "Optimal" || "$BBU_STATE" == "Operational" ]]; then BBU_HTML="$BBU_STATE(${BBU_TEMP:-N/A})" else BBU_HTML="${BBU_STATE}(${BBU_TEMP:-N/A})" EMAIL_ALERT=1 EMAIL_BODY="${EMAIL_BODY}\n- BBU/CacheVault error: State=$BBU_STATE, Temp=$BBU_TEMP" fi fi REBUILD_INFO="" megacli -PDList -aALL 2>/dev/null > /tmp/pd_task.tmp csplit -s -f /tmp/pd_tblock_ /tmp/pd_task.tmp '/Device Id:/' '{*}' 2>/dev/null for block in /tmp/pd_tblock_*; do [ -f "$block" ] || continue if grep -q "Firmware state" "$block"; then F_STATE=$(grep "Firmware state" "$block" | awk -F':' '{print $2}' | xargs) if echo "$F_STATE" | grep -qi "Rebuild"; then DEV_ID=$(grep "Device Id" "$block" | awk -F':' '{print $2}' | xargs) ENC_ID=$(grep "Enclosure Device ID" "$block" | awk -F':' '{print $2}' | xargs) PROG=$(megacli -PDRbld -ShowProg -PhysDrv "[$ENC_ID:$DEV_ID]" -aALL 2>/dev/null | grep -i "Completed" | awk -F'=' '{print $2}' | xargs) REBUILD_INFO="Rebuild Dev $DEV_ID: ${PROG:-In Progress}" break fi fi rm -f "$block" done rm -f /tmp/pd_task.tmp /tmp/pd_tblock_* if [ -n "$REBUILD_INFO" ]; then TASK_HTML="$REBUILD_INFO" else TASK_HTML="Idle" fi echo "--------------------------------------" > /tmp/hdd.rac echo -e "\e[1;37mHW-RAID-CONTROLLER\e[0m" >> /tmp/hdd.rac echo "--------------------------------------" >> /tmp/hdd.rac echo -e "\e[1;36m${CON:-N/A}\e[0m" >> /tmp/hdd.rac echo "------ ---------- --------" > /tmp/hdd.rai echo -e "\e[1;37mTYPE SIZE STATUS\e[0m" >> /tmp/hdd.rai echo "------ ---------- --------" >> /tmp/hdd.rai RAID_HTML_ACC="" megacli -LDInfo -Lall -aALL 2>/dev/null > /tmp/ld.tmp csplit -s -f /tmp/ld_block_ /tmp/ld.tmp '/Virtual Drive:/' '{*}' 2>/dev/null for block in /tmp/ld_block_*; do [ -f "$block" ] || continue if grep -q "RAID Level" "$block"; then RAI=$(grep -E "Name|RAID Level" "$block" | grep -v "User Name" | head -n1 | awk -F':' '{print $2}' | xargs) SIZ=$(grep "Size" "$block" | grep -vE "Sector|Strip" | awk -F':' '{print $2}' | xargs | tr -d ' ') STA=$(grep "State" "$block" | awk -F':' '{print $2}' | xargs) echo -e "\e[1;36m${RAI:-LD} ${SIZ:-N/A} ${STA:-N/A}\e[0m" >> /tmp/hdd.rai if [[ "$STA" == "Optimal" ]]; then STA_HTML="$STA" else STA_HTML="$STA" EMAIL_ALERT=1 EMAIL_BODY="${EMAIL_BODY}\n- RAID array ${RAI:-LD} bad state: $STA" fi RAID_HTML_ACC="${RAID_HTML_ACC}
${RAI:-LD} (${SIZ:-N/A}) ${STA_HTML}
" fi rm -f "$block" done rm -f /tmp/ld.tmp echo "------ ---------- --------" >> /tmp/hdd.rai echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" > $REP echo -e "\e[1;37mSLOT HDD-MODEL SERIAL SIZE TYPE TEMP MEDIA-ERR FAIL\e[0m" >> $REP echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP megacli -PDList -aALL 2>/dev/null > /tmp/hdd.tmp csplit -s -f /tmp/pd_block_ /tmp/hdd.tmp '/Device Id:/' '{*}' 2>/dev/null for block in /tmp/pd_block_*; do [ -f "$block" ] || continue SN=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $NF}') if [ -n "$SN" ]; then MODL=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $1"-"$2}') SIZE=$(grep "Raw Size" "$block" | awk -F':' '{print $2}' | awk '{print $1$2}') SLOT=$(grep "Slot Number" "$block" | awk -F':' '{print $2}' | xargs) FAIL=$(grep "Predictive Failure Count" "$block" | awk -F':' '{print $2}' | xargs) MEDIA_ERR=$(grep "Media Error Count" "$block" | awk -F':' '{print $2}' | xargs) TEMP=$(grep "Drive Temperature" "$block" | awk -F':' '{print $2}' | awk '{print $1}') M_TYPE_RAW=$(grep "Media Type" "$block" | awk -F':' '{print $2}' | xargs) I_TYPE_RAW=$(grep "PD Type" "$block" | awk -F':' '{print $2}' | xargs) if echo "$M_TYPE_RAW" | grep -qi "Solid State"; then M_TYPE="SSD" else M_TYPE="HDD" fi if echo "$I_TYPE_RAW" | grep -qi "SATA"; then I_TYPE="SATA" else I_TYPE="SAS" fi BUS_TYPE="${I_TYPE}/${M_TYPE}" [ -z "$SLOT" ] && SLOT="N/A" [ -z "$FAIL" ] && FAIL="0" [ -z "$MEDIA_ERR" ] && MEDIA_ERR="0" [ -z "$TEMP" ] && TEMP="N/A" if [[ "$FAIL" == "0" ]]; then FAIL_HTML="0" else FAIL_HTML="$FAIL" fi if [[ "$MEDIA_ERR" == "0" ]]; then MEDIA_HTML="0" else MEDIA_HTML="$MEDIA_ERR" fi # Zachytenie chyby disku pre e-mail if [[ "$FAIL" != "0" || "$MEDIA_ERR" != "0" ]]; then EMAIL_ALERT=1 EMAIL_BODY="${EMAIL_BODY}\n- Disk Slot $SLOT ($MODL, SN: $SN): Media Errors = $MEDIA_ERR, Predictive Failures = $FAIL" fi cat >> $WEB/hdd.html < EOF echo "$SLOT $MODL $SN $SIZE $BUS_TYPE $TEMP $MEDIA_ERR $FAIL" >> $REP fi rm -f "$block" done echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP rm -f /tmp/hdd.tmp } #------------------------------ WEB-STOP -------------------------------# webend(){ cat >> $WEB/hdd.html <
Slot HDD-Model Serial Size Type Temp Media Err Pred Fail
$SLOT $MODL $SN $SIZE $BUS_TYPE $TEMP $MEDIA_HTML $FAIL_HTML
EOF NUM=$(grep -vE "SERIAL|---" $REP | wc -l) SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}') sed -i "s/\[NUM\]/$NUM/g" $WEB/hdd.html sed -i "s/DATE/$(date '+%Y-%m-%d %H:%M:%S')/g" $WEB/hdd.html sed -i "s/SITE/$SITE/g" $WEB/hdd.html sed -i "s|CTRL_INFO|${CON:-N/A}|g" $WEB/hdd.html SAFE_BBU=$(echo "$BBU_HTML" | sed -e 's/[]\/$*.^[]/\\&/g') SAFE_TASK=$(echo "$TASK_HTML" | sed -e 's/[]\/$*.^[]/\\&/g') SAFE_RAID=$(echo "$RAID_HTML_ACC" | sed -e 's/[]\/$*.^[]/\\&/g') sed -i "s|BBU_INFO|$SAFE_BBU|g" $WEB/hdd.html sed -i "s|TASK_INFO|$SAFE_TASK|g" $WEB/hdd.html sed -i "s|RAID_INFO|$SAFE_RAID|g" $WEB/hdd.html } #------------------------------- CORE ----------------------------------# webstart checkhdd webend IP=$(ip route get 1.1.1.1 2>/dev/null | awk '{print $7}') echo -e "- Final HTML report (\e[1;35mhttp://${IP:-localhost}/hdd/hdd.html\e[0m)" cat /tmp/hdd.rac cat /tmp/hdd.rai | column -t cat $REP | column -t #------------------------------ NOTIFICATION ---------------------------# if [ "$EMAIL_ALERT" -eq 1 ]; then if [ ! -f "$STATE" ]; then SUBJECT="[CRITICAL] Storage Error on $SITE ($IP)" echo -e "On $SITE was detected errors:\n$EMAIL_BODY\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO" touch "$STATE" fi else if [ -f "$STATE" ]; then SUBJECT="[RECOVERY] Storage OK on $SITE ($IP)" echo -e "All drives and RAID array on $SITE are optimal state.\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO" rm -f "$STATE" fi fi EOGTR