UNI/raid.sh

#!/bin/bash
#########################################################################
#                    Script for check HDD & RAID state                  #
#                    Created by Pavol Macko - DoIT                      #
#                               ver 0.7                                 #
#########################################################################

#---------------------------- VARIABLES --------------------------------#
REP=/tmp/hdd.rep
WEB=/var/www/hdd
TMP=/tmp/hdd.tmp
LOG=/tmp/hdd.log
STATE=/tmp/hdd.state
MAIL_TO="info@doit.sk"
EMAIL_ALERT=0
EMAIL_BODY=""
mkdir -p $WEB
rm -f $TMP $LOG $REP

#----------------------------- WEB-START -------------------------------#
webstart(){
cat > $WEB/hdd.html <<\EOF
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>SITE - Storage</title>
<link rel="icon" href="logo.ico">
<style>
    :root {
        --bg-main: #0f172a;
        --card-bg: #1e293b;
        --border-color: #334155;
        --text-muted: #94a3b8;
        --accent-blue: #38bdf8;
    }
    body {
        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
        background-color: var(--bg-main);
        color: #f8fafc;
        margin: 0;
        padding: 40px 20px;
        line-height: 1.5;
    }
    .container {
        max-width: 1150px;
        margin: 0 auto;
        background: var(--card-bg);
        border-radius: 12px;
        box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3);
        padding: 32px;
        border: 1px solid var(--border-color);
    }
    h2 {
        text-align: center;
        color: #f8fafc;
        margin: 0 0 6px 0;
        font-weight: 700;
        font-size: 1.6em;
        letter-spacing: -0.02em;
    }
    .subtitle {
        text-align: center;
        color: var(--text-muted);
        font-size: 0.88em;
        margin-bottom: 28px;
    }
    .cards-grid {
        display: grid;
        grid-template-columns: repeat(auto-fit, minmax(230px, 1fr));
        gap: 16px;
        margin-bottom: 30px;
    }
    .card {
        background: #0f172a;
        border: 1px solid var(--border-color);
        border-radius: 8px;
        padding: 16px;
        transition: transform 0.2s ease, border-color 0.2s ease;
    }
    .card:hover {
        border-color: #475569;
        transform: translateY(-2px);
    }
    .card-title {
        font-size: 0.72em;
        text-transform: uppercase;
        letter-spacing: 0.08em;
        color: var(--text-muted);
        font-weight: 700;
        margin-bottom: 8px;
    }
    .card-content {
        font-size: 0.98em;
        font-weight: 600;
        color: var(--accent-blue);
    }
    .raid-item {
        display: flex;
        justify-content: space-between;
        align-items: center;
        font-size: 0.85em;
        padding: 4px 0;
        border-bottom: 1px dashed var(--border-color);
    }
    .raid-item:last-child { border-bottom: none; }
    table {
        width: 100%;
        border-collapse: separate;
        border-spacing: 0;
        border-radius: 8px;
        overflow: hidden;
        border: 1px solid var(--border-color);
    }
    th {
        background-color: #0f172a;
        color: #cbd5e1;
        font-size: 0.75em;
        text-transform: uppercase;
        letter-spacing: 0.08em;
        font-weight: 700;
        padding: 14px;
        border-bottom: 1px solid var(--border-color);
        text-align: center;
    }
    td {
        padding: 12px 14px;
        text-align: center;
        border-bottom: 1px solid var(--border-color);
        font-size: 0.88em;
        color: #e2e8f0;
    }
    tr:last-child td {
        border-bottom: none;
    }
    tr:nth-child(even) {
        background-color: rgba(255, 255, 255, 0.015);
    }
    tr:hover {
        background-color: rgba(51, 65, 85, 0.5);
    }
    .status-ok {
        background-color: rgba(16, 185, 129, 0.12);
        color: #34d399;
        border: 1px solid rgba(52, 211, 153, 0.25);
        padding: 3px 10px;
        border-radius: 20px;
        font-size: 0.78em;
        font-weight: 600;
        display: inline-block;
    }
    .status-fail {
        background-color: rgba(239, 68, 68, 0.15);
        color: #fca5a5;
        border: 1px solid rgba(252, 165, 165, 0.3);
        padding: 3px 10px;
        border-radius: 20px;
        font-size: 0.78em;
        font-weight: 600;
        display: inline-block;
    }
    .status-warn {
        background-color: rgba(245, 158, 11, 0.15);
        color: #fbbf24;
        border: 1px solid rgba(251, 191, 36, 0.3);
        padding: 3px 10px;
        border-radius: 20px;
        font-size: 0.78em;
        font-weight: 600;
        display: inline-block;
    }
    .type-tag {
        background-color: rgba(56, 189, 248, 0.08);
        color: #38bdf8;
        border: 1px solid rgba(56, 189, 248, 0.25);
        padding: 2px 8px;
        border-radius: 4px;
        font-size: 0.78em;
        font-weight: 600;
        display: inline-block;
    }
</style>

</head>
<body>
<div class="container">
    <h2>RAID Storage Status on [SITE]</h2>
    <div class="subtitle">Number of disks: <b style="color: #f8fafc;">[NUM]</b> | Last update: DATE</div>
    <!-- Info Cards -->
    <div class="cards-grid">
        <div class="card">
            <div class="card-title">HW RAID Controller</div>
            <div class="card-content">CTRL_INFO</div>
        </div>
        <div class="card">
            <div class="card-title">BBU / CacheVault</div>
            <div class="card-content">BBU_INFO</div>
        </div>
        <div class="card">
            <div class="card-title">Background Tasks</div>
            <div class="card-content">TASK_INFO</div>
        </div>
        <div class="card">
            <div class="card-title">Logical Arrays (RAID)</div>
            RAID_INFO
        </div>
    </div>

    <table>
        <thead>
            <tr>
                <th>Slot</th>
                <th>HDD-Model</th>
                <th>Serial</th>
                <th>Size</th>
                <th>Type</th>
                <th>Temp</th>
                <th>Media Err</th>
                <th>Pred Fail</th>
            </tr>
        </thead>
        <tbody>
EOF
}

#--------------------------- CHECK HDD INFO ----------------------------#
checkhdd(){
echo -e "- \e[1;32mExtracting\e[0m HDD info from PVE hypervisor"
CON=$(megacli -ShowSummary -aALL 2>/dev/null | grep "ProductName" | head -n1 | awk -F':' '{print $2}' | xargs)
BBU_RAW=$(megacli -AdpBbuCmd -aALL 2>/dev/null)
if echo "$BBU_RAW" | grep -qi "Get BBU Status Failed\|Adapter.*Has No BBU"; then
    BBU_HTML="<span style=\"color:#94a3b8; font-size:0.9em;\">N/A (No BBU)</span>"
else
    BBU_STATE=$(echo "$BBU_RAW" | grep "Battery State" | head -n1 | awk -F':' '{print $2}' | xargs)
    BBU_TEMP=$(echo "$BBU_RAW" | grep "Temperature" | head -n1 | awk -F':' '{print $2}' | xargs)
    [ -z "$BBU_STATE" ] && BBU_STATE="Optimal"
    if [[ "$BBU_STATE" == "Optimal" || "$BBU_STATE" == "Operational" ]]; then
        BBU_HTML="<span class=\"status-ok\">$BBU_STATE</span> <span style=\"font-size:0.85em; color:#94a3b8;\">(${BBU_TEMP:-N/A})</span>"
    else
        BBU_HTML="<span class=\"status-fail\">${BBU_STATE}</span> <span style=\"font-size:0.85em; color:#94a3b8;\">(${BBU_TEMP:-N/A})</span>"
        EMAIL_ALERT=1
        EMAIL_BODY="${EMAIL_BODY}\n- BBU/CacheVault error: State=$BBU_STATE, Temp=$BBU_TEMP"
    fi
fi

REBUILD_INFO=""
megacli -PDList -aALL 2>/dev/null > /tmp/pd_task.tmp
csplit -s -f /tmp/pd_tblock_ /tmp/pd_task.tmp '/Device Id:/' '{*}' 2>/dev/null
for block in /tmp/pd_tblock_*; do
    [ -f "$block" ] || continue
    if grep -q "Firmware state" "$block"; then
        F_STATE=$(grep "Firmware state" "$block" | awk -F':' '{print $2}' | xargs)
        if echo "$F_STATE" | grep -qi "Rebuild"; then
            DEV_ID=$(grep "Device Id" "$block" | awk -F':' '{print $2}' | xargs)
            ENC_ID=$(grep "Enclosure Device ID" "$block" | awk -F':' '{print $2}' | xargs)
            PROG=$(megacli -PDRbld -ShowProg -PhysDrv "[$ENC_ID:$DEV_ID]" -aALL 2>/dev/null | grep -i "Completed" | awk -F'=' '{print $2}' | xargs)
            REBUILD_INFO="Rebuild Dev $DEV_ID: ${PROG:-In Progress}"
            break
        fi
    fi
    rm -f "$block"
done
rm -f /tmp/pd_task.tmp /tmp/pd_tblock_*

if [ -n "$REBUILD_INFO" ]; then
    TASK_HTML="<span class=\"status-warn\">$REBUILD_INFO</span>"
else
    TASK_HTML="<span class=\"status-ok\">Idle</span>"
fi

echo "--------------------------------------" > /tmp/hdd.rac
echo -e "\e[1;37mHW-RAID-CONTROLLER\e[0m" >> /tmp/hdd.rac
echo "--------------------------------------" >> /tmp/hdd.rac
echo -e "\e[1;36m${CON:-N/A}\e[0m" >> /tmp/hdd.rac
echo "------ ---------- --------" > /tmp/hdd.rai
echo -e "\e[1;37mTYPE SIZE STATUS\e[0m" >> /tmp/hdd.rai
echo "------ ---------- --------" >> /tmp/hdd.rai

RAID_HTML_ACC=""
megacli -LDInfo -Lall -aALL 2>/dev/null > /tmp/ld.tmp
csplit -s -f /tmp/ld_block_ /tmp/ld.tmp '/Virtual Drive:/' '{*}' 2>/dev/null
for block in /tmp/ld_block_*; do
    [ -f "$block" ] || continue
    if grep -q "RAID Level" "$block"; then
        RAI=$(grep -E "Name|RAID Level" "$block" | grep -v "User Name" | head -n1 | awk -F':' '{print $2}' | xargs)
        SIZ=$(grep "Size" "$block" | grep -vE "Sector|Strip" | awk -F':' '{print $2}' | xargs | tr -d ' ')
        STA=$(grep "State" "$block" | awk -F':' '{print $2}' | xargs)
        echo -e "\e[1;36m${RAI:-LD} ${SIZ:-N/A} ${STA:-N/A}\e[0m" >> /tmp/hdd.rai
        if [[ "$STA" == "Optimal" ]]; then
            STA_HTML="<span class=\"status-ok\">$STA</span>"
        else
            STA_HTML="<span class=\"status-fail\">$STA</span>"
            EMAIL_ALERT=1
            EMAIL_BODY="${EMAIL_BODY}\n- RAID array ${RAI:-LD} bad state: $STA"
        fi
        RAID_HTML_ACC="${RAID_HTML_ACC}<div class=\"raid-item\"><span><b>${RAI:-LD}</b> (${SIZ:-N/A})</span> ${STA_HTML}</div>"
    fi
    rm -f "$block"
done
rm -f /tmp/ld.tmp

echo "------ ---------- --------" >> /tmp/hdd.rai
echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" > $REP
echo -e "\e[1;37mSLOT HDD-MODEL SERIAL SIZE TYPE TEMP MEDIA-ERR FAIL\e[0m" >> $REP
echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP

megacli -PDList -aALL 2>/dev/null > /tmp/hdd.tmp
csplit -s -f /tmp/pd_block_ /tmp/hdd.tmp '/Device Id:/' '{*}' 2>/dev/null
for block in /tmp/pd_block_*; do
    [ -f "$block" ] || continue
    SN=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $NF}')
    if [ -n "$SN" ]; then
        MODL=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $1"-"$2}')
        SIZE=$(grep "Raw Size" "$block" | awk -F':' '{print $2}' | awk '{print $1$2}')
        SLOT=$(grep "Slot Number" "$block" | awk -F':' '{print $2}' | xargs)
        FAIL=$(grep "Predictive Failure Count" "$block" | awk -F':' '{print $2}' | xargs)
        MEDIA_ERR=$(grep "Media Error Count" "$block" | awk -F':' '{print $2}' | xargs)
        TEMP=$(grep "Drive Temperature" "$block" | awk -F':' '{print $2}' | awk '{print $1}')
        M_TYPE_RAW=$(grep "Media Type" "$block" | awk -F':' '{print $2}' | xargs)
        I_TYPE_RAW=$(grep "PD Type" "$block" | awk -F':' '{print $2}' | xargs)
        if echo "$M_TYPE_RAW" | grep -qi "Solid State"; then
            M_TYPE="SSD"
        else
            M_TYPE="HDD"
        fi
        if echo "$I_TYPE_RAW" | grep -qi "SATA"; then
            I_TYPE="SATA"
        else
            I_TYPE="SAS"
        fi
        BUS_TYPE="${I_TYPE}/${M_TYPE}"

        [ -z "$SLOT" ] && SLOT="N/A"
        [ -z "$FAIL" ] && FAIL="0"
        [ -z "$MEDIA_ERR" ] && MEDIA_ERR="0"
        [ -z "$TEMP" ] && TEMP="N/A"
        
        if [[ "$FAIL" == "0" ]]; then
            FAIL_HTML="<span class=\"status-ok\">0</span>"
        else
            FAIL_HTML="<span class=\"status-fail\">$FAIL</span>"
        fi

        if [[ "$MEDIA_ERR" == "0" ]]; then
            MEDIA_HTML="<span class=\"status-ok\">0</span>"
        else
            MEDIA_HTML="<span class=\"status-fail\">$MEDIA_ERR</span>"
        fi

        # Zachytenie chyby disku pre e-mail
        if [[ "$FAIL" != "0" || "$MEDIA_ERR" != "0" ]]; then
            EMAIL_ALERT=1
            EMAIL_BODY="${EMAIL_BODY}\n- Disk Slot $SLOT ($MODL, SN: $SN): Media Errors = $MEDIA_ERR, Predictive Failures = $FAIL"
        fi

        cat >> $WEB/hdd.html <<EOF
        <tr>
            <td><b>$SLOT</b></td>
            <td>$MODL</td>
            <td>$SN</td>
            <td>$SIZE</td>
            <td><span class="type-tag">$BUS_TYPE</span></td>
            <td>$TEMP</td>
            <td>$MEDIA_HTML</td>
            <td>$FAIL_HTML</td>
        </tr>
EOF
        echo "$SLOT $MODL $SN $SIZE $BUS_TYPE $TEMP $MEDIA_ERR $FAIL" >> $REP
    fi
    rm -f "$block"
done

echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
rm -f /tmp/hdd.tmp
}

#------------------------------ WEB-STOP -------------------------------#
webend(){
cat >> $WEB/hdd.html <<EOF
        </tbody>
    </table>
</div>
</body>
</html>
EOF
NUM=$(grep -vE "SERIAL|---" $REP | wc -l)
SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}')
sed -i "s/\[NUM\]/$NUM/g" $WEB/hdd.html
sed -i "s/DATE/$(date '+%Y-%m-%d %H:%M:%S')/g" $WEB/hdd.html
sed -i "s/SITE/$SITE/g" $WEB/hdd.html
sed -i "s|CTRL_INFO|${CON:-N/A}|g" $WEB/hdd.html
SAFE_BBU=$(echo "$BBU_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
SAFE_TASK=$(echo "$TASK_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
SAFE_RAID=$(echo "$RAID_HTML_ACC" | sed -e 's/[]\/$*.^[]/\\&/g')
sed -i "s|BBU_INFO|$SAFE_BBU|g" $WEB/hdd.html
sed -i "s|TASK_INFO|$SAFE_TASK|g" $WEB/hdd.html
sed -i "s|RAID_INFO|$SAFE_RAID|g" $WEB/hdd.html
}

#------------------------------- CORE ----------------------------------#
webstart
checkhdd
webend
IP=$(ip route get 1.1.1.1 2>/dev/null | awk '{print $7}')
echo -e "- Final HTML report (\e[1;35mhttp://${IP:-localhost}/hdd/hdd.html\e[0m)"
cat /tmp/hdd.rac
cat /tmp/hdd.rai | column -t
cat $REP | column -t

#------------------------------ NOTIFICATION ---------------------------#
if [ "$EMAIL_ALERT" -eq 1 ]; then
    if [ ! -f "$STATE" ]; then
        SUBJECT="[CRITICAL] Storage Error on $SITE ($IP)"
        echo -e "On $SITE was detected errors:\n$EMAIL_BODY\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
        touch "$STATE"
    fi
else
    if [ -f "$STATE" ]; then
        SUBJECT="[RECOVERY] Storage OK on $SITE ($IP)"
        echo -e "All drives and RAID array on $SITE are optimal state.\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
        rm -f "$STATE"
    fi
fi
EOGTR