Переглянути джерело

Extended collection, updated modern HTML5 reporting and added mail notifications

pmacko 1 місяць тому
батько
коміт
ccaccbf94c
1 змінених файлів з 366 додано та 75 видалено
  1. 366 75
      raid.sh

+ 366 - 75
raid.sh

@@ -2,7 +2,7 @@
 #########################################################################
 #                    Script for check HDD & RAID state                  #
 #                    Created by Pavol Macko - DoIT                      #
-#                                  ver 0.4                              #
+#                               ver 0.7                                 #
 #########################################################################
 
 #---------------------------- VARIABLES --------------------------------#
@@ -10,9 +10,12 @@ REP=/tmp/hdd.rep
 WEB=/var/www/hdd
 TMP=/tmp/hdd.tmp
 LOG=/tmp/hdd.log
-test -d $WEB || mkdir -p $WEB
-test -f $TMP && rm -rf $TMP
-test -f $LOG && rm -rf $LOG
+STATE=/tmp/hdd.state
+MAIL_TO="info@doit.sk"
+EMAIL_ALERT=0
+EMAIL_BODY=""
+mkdir -p $WEB
+rm -f $TMP $LOG $REP
 
 #----------------------------- WEB-START -------------------------------#
 webstart(){
@@ -22,113 +25,401 @@ cat > $WEB/hdd.html <<\EOF
 <head>
 <meta charset="UTF-8">
 <meta name="viewport" content="width=device-width, initial-scale=1.0">
-<title>SITE HDD</title>
+<title>SITE - Storage</title>
 <link rel="icon" href="logo.ico">
 <style>
-    body { font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; background-color: #0f172a; color: #f8fafc; margin: 0; padding: 40px 20px; }
-    .container { max-width: 1000px; margin: 0 auto; background: #1e293b; border-radius: 10px; box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.5); padding: 30px; border: 1px solid #334155; }
-    h2 { text-align: center; color: #f1f5f9; margin-top: 0; margin-bottom: 6px; font-weight: 600; letter-spacing: -0.025em; }
-    .subtitle { text-align: center; color: #94a3b8; font-size: 0.9em; margin-bottom: 30px; }
-    table { width: 100%; border-collapse: collapse; }
-    th, td { padding: 14px 16px; text-align: center; border-bottom: 1px solid #334155; }
-    th { background-color: #0f172a; color: #cbd5e1; font-size: 0.8em; text-transform: uppercase; letter-spacing: 0.08em; font-weight: 700; }
-    tr:hover { background-color: #334155; transition: background-color 0.15s ease; }
-    .status-ok { background-color: rgba(16, 185, 129, 0.15); color: #34d399; border: 1px solid rgba(52, 211, 153, 0.3); padding: 4px 12px; border-radius: 20px; font-size: 0.85em; font-weight: 600; display: inline-block; }
-    .status-fail { background-color: rgba(239, 68, 68, 0.15); color: #fca5a5; border: 1px solid rgba(252, 165, 165, 0.3); padding: 4px 12px; border-radius: 20px; font-size: 0.85em; font-weight: 600; display: inline-block; }
+    :root {
+        --bg-main: #0f172a;
+        --card-bg: #1e293b;
+        --border-color: #334155;
+        --text-muted: #94a3b8;
+        --accent-blue: #38bdf8;
+    }
+    body {
+        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+        background-color: var(--bg-main);
+        color: #f8fafc;
+        margin: 0;
+        padding: 40px 20px;
+        line-height: 1.5;
+    }
+    .container {
+        max-width: 1150px;
+        margin: 0 auto;
+        background: var(--card-bg);
+        border-radius: 12px;
+        box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3);
+        padding: 32px;
+        border: 1px solid var(--border-color);
+    }
+    h2 {
+        text-align: center;
+        color: #f8fafc;
+        margin: 0 0 6px 0;
+        font-weight: 700;
+        font-size: 1.6em;
+        letter-spacing: -0.02em;
+    }
+    .subtitle {
+        text-align: center;
+        color: var(--text-muted);
+        font-size: 0.88em;
+        margin-bottom: 28px;
+    }
+    .cards-grid {
+        display: grid;
+        grid-template-columns: repeat(auto-fit, minmax(230px, 1fr));
+        gap: 16px;
+        margin-bottom: 30px;
+    }
+    .card {
+        background: #0f172a;
+        border: 1px solid var(--border-color);
+        border-radius: 8px;
+        padding: 16px;
+        transition: transform 0.2s ease, border-color 0.2s ease;
+    }
+    .card:hover {
+        border-color: #475569;
+        transform: translateY(-2px);
+    }
+    .card-title {
+        font-size: 0.72em;
+        text-transform: uppercase;
+        letter-spacing: 0.08em;
+        color: var(--text-muted);
+        font-weight: 700;
+        margin-bottom: 8px;
+    }
+    .card-content {
+        font-size: 0.98em;
+        font-weight: 600;
+        color: var(--accent-blue);
+    }
+    .raid-item {
+        display: flex;
+        justify-content: space-between;
+        align-items: center;
+        font-size: 0.85em;
+        padding: 4px 0;
+        border-bottom: 1px dashed var(--border-color);
+    }
+    .raid-item:last-child { border-bottom: none; }
+    table {
+        width: 100%;
+        border-collapse: separate;
+        border-spacing: 0;
+        border-radius: 8px;
+        overflow: hidden;
+        border: 1px solid var(--border-color);
+    }
+    th {
+        background-color: #0f172a;
+        color: #cbd5e1;
+        font-size: 0.75em;
+        text-transform: uppercase;
+        letter-spacing: 0.08em;
+        font-weight: 700;
+        padding: 14px;
+        border-bottom: 1px solid var(--border-color);
+        text-align: center;
+    }
+    td {
+        padding: 12px 14px;
+        text-align: center;
+        border-bottom: 1px solid var(--border-color);
+        font-size: 0.88em;
+        color: #e2e8f0;
+    }
+    tr:last-child td {
+        border-bottom: none;
+    }
+    tr:nth-child(even) {
+        background-color: rgba(255, 255, 255, 0.015);
+    }
+    tr:hover {
+        background-color: rgba(51, 65, 85, 0.5);
+    }
+    .status-ok {
+        background-color: rgba(16, 185, 129, 0.12);
+        color: #34d399;
+        border: 1px solid rgba(52, 211, 153, 0.25);
+        padding: 3px 10px;
+        border-radius: 20px;
+        font-size: 0.78em;
+        font-weight: 600;
+        display: inline-block;
+    }
+    .status-fail {
+        background-color: rgba(239, 68, 68, 0.15);
+        color: #fca5a5;
+        border: 1px solid rgba(252, 165, 165, 0.3);
+        padding: 3px 10px;
+        border-radius: 20px;
+        font-size: 0.78em;
+        font-weight: 600;
+        display: inline-block;
+    }
+    .status-warn {
+        background-color: rgba(245, 158, 11, 0.15);
+        color: #fbbf24;
+        border: 1px solid rgba(251, 191, 36, 0.3);
+        padding: 3px 10px;
+        border-radius: 20px;
+        font-size: 0.78em;
+        font-weight: 600;
+        display: inline-block;
+    }
+    .type-tag {
+        background-color: rgba(56, 189, 248, 0.08);
+        color: #38bdf8;
+        border: 1px solid rgba(56, 189, 248, 0.25);
+        padding: 2px 8px;
+        border-radius: 4px;
+        font-size: 0.78em;
+        font-weight: 600;
+        display: inline-block;
+    }
 </style>
+
 </head>
 <body>
 <div class="container">
-    <h2>Disk overview on server [SITE]</h2>
+    <h2>RAID Storage Status on [SITE]</h2>
     <div class="subtitle">Number of disks: <b style="color: #f8fafc;">[NUM]</b> | Last update: DATE</div>
+    <!-- Info Cards -->
+    <div class="cards-grid">
+        <div class="card">
+            <div class="card-title">HW RAID Controller</div>
+            <div class="card-content">CTRL_INFO</div>
+        </div>
+        <div class="card">
+            <div class="card-title">BBU / CacheVault</div>
+            <div class="card-content">BBU_INFO</div>
+        </div>
+        <div class="card">
+            <div class="card-title">Background Tasks</div>
+            <div class="card-content">TASK_INFO</div>
+        </div>
+        <div class="card">
+            <div class="card-title">Logical Arrays (RAID)</div>
+            RAID_INFO
+        </div>
+    </div>
+
     <table>
         <thead>
             <tr>
+                <th>Slot</th>
                 <th>HDD-Model</th>
                 <th>Serial</th>
                 <th>Size</th>
-                <th>Path</th>
+                <th>Type</th>
                 <th>Temp</th>
-                <th>Fail</th>
+                <th>Media Err</th>
+                <th>Pred Fail</th>
             </tr>
         </thead>
         <tbody>
 EOF
 }
 
-#------------------------------ WEB-STOP -------------------------------#
-webend(){
-cat >> $WEB/hdd.html <<\EOF
-        </tbody>
-    </table>
-</div>
-</body>
-</html>
-EOF
-NUM=$(cat $REP | egrep -v "SERIAL|---" | wc -l)
-SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}')
-sed -i "s/NUM/$NUM/g" $WEB/hdd.html
-sed -i "s/DATE/`date`/g" $WEB/hdd.html
-sed -i "s/SITE/$SITE/g" $WEB/hdd.html
-}
-
 #--------------------------- CHECK HDD INFO ----------------------------#
 checkhdd(){
 echo -e "- \e[1;32mExtracting\e[0m HDD info from PVE hypervisor"
+CON=$(megacli -ShowSummary -aALL 2>/dev/null | grep "ProductName" | head -n1 | awk -F':' '{print $2}' | xargs)
+BBU_RAW=$(megacli -AdpBbuCmd -aALL 2>/dev/null)
+if echo "$BBU_RAW" | grep -qi "Get BBU Status Failed\|Adapter.*Has No BBU"; then
+    BBU_HTML="<span style=\"color:#94a3b8; font-size:0.9em;\">N/A (No BBU)</span>"
+else
+    BBU_STATE=$(echo "$BBU_RAW" | grep "Battery State" | head -n1 | awk -F':' '{print $2}' | xargs)
+    BBU_TEMP=$(echo "$BBU_RAW" | grep "Temperature" | head -n1 | awk -F':' '{print $2}' | xargs)
+    [ -z "$BBU_STATE" ] && BBU_STATE="Optimal"
+    if [[ "$BBU_STATE" == "Optimal" || "$BBU_STATE" == "Operational" ]]; then
+        BBU_HTML="<span class=\"status-ok\">$BBU_STATE</span> <span style=\"font-size:0.85em; color:#94a3b8;\">(${BBU_TEMP:-N/A})</span>"
+    else
+        BBU_HTML="<span class=\"status-fail\">${BBU_STATE}</span> <span style=\"font-size:0.85em; color:#94a3b8;\">(${BBU_TEMP:-N/A})</span>"
+        EMAIL_ALERT=1
+        EMAIL_BODY="${EMAIL_BODY}\n- BBU/CacheVault error: State=$BBU_STATE, Temp=$BBU_TEMP"
+    fi
+fi
+
+REBUILD_INFO=""
+megacli -PDList -aALL 2>/dev/null > /tmp/pd_task.tmp
+csplit -s -f /tmp/pd_tblock_ /tmp/pd_task.tmp '/Device Id:/' '{*}' 2>/dev/null
+for block in /tmp/pd_tblock_*; do
+    [ -f "$block" ] || continue
+    if grep -q "Firmware state" "$block"; then
+        F_STATE=$(grep "Firmware state" "$block" | awk -F':' '{print $2}' | xargs)
+        if echo "$F_STATE" | grep -qi "Rebuild"; then
+            DEV_ID=$(grep "Device Id" "$block" | awk -F':' '{print $2}' | xargs)
+            ENC_ID=$(grep "Enclosure Device ID" "$block" | awk -F':' '{print $2}' | xargs)
+            PROG=$(megacli -PDRbld -ShowProg -PhysDrv "[$ENC_ID:$DEV_ID]" -aALL 2>/dev/null | grep -i "Completed" | awk -F'=' '{print $2}' | xargs)
+            REBUILD_INFO="Rebuild Dev $DEV_ID: ${PROG:-In Progress}"
+            break
+        fi
+    fi
+    rm -f "$block"
+done
+rm -f /tmp/pd_task.tmp /tmp/pd_tblock_*
+
+if [ -n "$REBUILD_INFO" ]; then
+    TASK_HTML="<span class=\"status-warn\">$REBUILD_INFO</span>"
+else
+    TASK_HTML="<span class=\"status-ok\">Idle</span>"
+fi
+
 echo "--------------------------------------" > /tmp/hdd.rac
-echo -e "\e[1;37mHW-RAID-CONTROLER\e[0m" >> /tmp/hdd.rac
+echo -e "\e[1;37mHW-RAID-CONTROLLER\e[0m" >> /tmp/hdd.rac
 echo "--------------------------------------" >> /tmp/hdd.rac
-CON=$(megacli -ShowSummary -aALL | grep ProductName | awk '{$1=$2="";print $0}' | sed 's/  //g')
-echo -e "\e[1;36m$CON\e[0m" >>/tmp/hdd.rac
+echo -e "\e[1;36m${CON:-N/A}\e[0m" >> /tmp/hdd.rac
 echo "------ ---------- --------" > /tmp/hdd.rai
 echo -e "\e[1;37mTYPE SIZE STATUS\e[0m" >> /tmp/hdd.rai
 echo "------ ---------- --------" >> /tmp/hdd.rai
-megacli -LDInfo -L0 -a0  > /tmp/hdd.r0
-RAI=$(cat /tmp/hdd.r0 | grep Name | sed 's/://g' | awk '{print $2}')
-SIZ=$(cat /tmp/hdd.r0 | grep "Size" | egrep -v "Sector|Strip" | awk '{print $3""$4}')
-STA=$(cat /tmp/hdd.r0 | grep State | awk '{print $3}')
-echo -e "\e[1;36m$RAI $SIZ $STA\e[0m" >> /tmp/hdd.rai
-megacli -LDInfo -L1 -a0 > /tmp/hdd.r1
-RAI=$(cat /tmp/hdd.r1 | grep Name | sed 's/://g' | awk '{print $2}')
-SIZ=$(cat /tmp/hdd.r1 | grep "Size" | egrep -v "Sector|Strip" | awk '{print $3""$4}')
-STA=$(cat /tmp/hdd.r1 | grep State | awk '{print $3}')
-echo -e "\e[1;36m$RAI $SIZ $STA\e[0m" >> /tmp/hdd.rai
+
+RAID_HTML_ACC=""
+megacli -LDInfo -Lall -aALL 2>/dev/null > /tmp/ld.tmp
+csplit -s -f /tmp/ld_block_ /tmp/ld.tmp '/Virtual Drive:/' '{*}' 2>/dev/null
+for block in /tmp/ld_block_*; do
+    [ -f "$block" ] || continue
+    if grep -q "RAID Level" "$block"; then
+        RAI=$(grep -E "Name|RAID Level" "$block" | grep -v "User Name" | head -n1 | awk -F':' '{print $2}' | xargs)
+        SIZ=$(grep "Size" "$block" | grep -vE "Sector|Strip" | awk -F':' '{print $2}' | xargs | tr -d ' ')
+        STA=$(grep "State" "$block" | awk -F':' '{print $2}' | xargs)
+        echo -e "\e[1;36m${RAI:-LD} ${SIZ:-N/A} ${STA:-N/A}\e[0m" >> /tmp/hdd.rai
+        if [[ "$STA" == "Optimal" ]]; then
+            STA_HTML="<span class=\"status-ok\">$STA</span>"
+        else
+            STA_HTML="<span class=\"status-fail\">$STA</span>"
+            EMAIL_ALERT=1
+            EMAIL_BODY="${EMAIL_BODY}\n- RAID array ${RAI:-LD} bad state: $STA"
+        fi
+        RAID_HTML_ACC="${RAID_HTML_ACC}<div class=\"raid-item\"><span><b>${RAI:-LD}</b> (${SIZ:-N/A})</span> ${STA_HTML}</div>"
+    fi
+    rm -f "$block"
+done
+rm -f /tmp/ld.tmp
+
 echo "------ ---------- --------" >> /tmp/hdd.rai
-megacli -PDList -a0 > /tmp/hdd.tmp
-echo "-------------------- ---------------- ---------- -------- --- -----" > $REP
-echo -e "\e[1;37mHDD-MODEL SERIAL SIZE PATH TEMP FAIL\e[0m" >> $REP
-echo "-------------------- ---------------- ---------- -------- --- -----" >> $REP
-SN=$(cat /tmp/hdd.tmp | grep "Inquiry Data" | awk '{print $NF}')
-for i in $SN;
-do
-    MODL=$(cat /tmp/hdd.tmp | grep -B15 $i | grep "Inquiry Data" | awk '{print $3"-"$4}')
-    SIZE=$(cat /tmp/hdd.tmp | grep -B15 $i | grep "Raw Size" | awk '{print $3"GB"}')
-    PORT=$(cat /tmp/hdd.tmp | grep -B15 $i | grep "Connected Port Number" | awk '{print $4}')
-    FAIL=$(cat /tmp/hdd.tmp | grep -B20 $i | grep "Predictive Failure Count" | awk '{print $NF}')
-    TEMP=$(cat /tmp/hdd.tmp | grep -A12 $i | grep "Temperature" | sed 's/://g' | awk '{print $3}')
-    if [[ "$FAIL" == "0" ]]; then
-        FAIL_HTML="<span class=\"status-ok\">0</span>"
-    else
-        FAIL_HTML="<span class=\"status-fail\">$FAIL</span>"
+echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" > $REP
+echo -e "\e[1;37mSLOT HDD-MODEL SERIAL SIZE TYPE TEMP MEDIA-ERR FAIL\e[0m" >> $REP
+echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
+
+megacli -PDList -aALL 2>/dev/null > /tmp/hdd.tmp
+csplit -s -f /tmp/pd_block_ /tmp/hdd.tmp '/Device Id:/' '{*}' 2>/dev/null
+for block in /tmp/pd_block_*; do
+    [ -f "$block" ] || continue
+    SN=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $NF}')
+    if [ -n "$SN" ]; then
+        MODL=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $1"-"$2}')
+        SIZE=$(grep "Raw Size" "$block" | awk -F':' '{print $2}' | awk '{print $1$2}')
+        SLOT=$(grep "Slot Number" "$block" | awk -F':' '{print $2}' | xargs)
+        FAIL=$(grep "Predictive Failure Count" "$block" | awk -F':' '{print $2}' | xargs)
+        MEDIA_ERR=$(grep "Media Error Count" "$block" | awk -F':' '{print $2}' | xargs)
+        TEMP=$(grep "Drive Temperature" "$block" | awk -F':' '{print $2}' | awk '{print $1}')
+        M_TYPE_RAW=$(grep "Media Type" "$block" | awk -F':' '{print $2}' | xargs)
+        I_TYPE_RAW=$(grep "PD Type" "$block" | awk -F':' '{print $2}' | xargs)
+        if echo "$M_TYPE_RAW" | grep -qi "Solid State"; then
+            M_TYPE="SSD"
+        else
+            M_TYPE="HDD"
+        fi
+        if echo "$I_TYPE_RAW" | grep -qi "SATA"; then
+            I_TYPE="SATA"
+        else
+            I_TYPE="SAS"
+        fi
+        BUS_TYPE="${I_TYPE}/${M_TYPE}"
+
+        [ -z "$SLOT" ] && SLOT="N/A"
+        [ -z "$FAIL" ] && FAIL="0"
+        [ -z "$MEDIA_ERR" ] && MEDIA_ERR="0"
+        [ -z "$TEMP" ] && TEMP="N/A"
+        
+        if [[ "$FAIL" == "0" ]]; then
+            FAIL_HTML="<span class=\"status-ok\">0</span>"
+        else
+            FAIL_HTML="<span class=\"status-fail\">$FAIL</span>"
+        fi
+
+        if [[ "$MEDIA_ERR" == "0" ]]; then
+            MEDIA_HTML="<span class=\"status-ok\">0</span>"
+        else
+            MEDIA_HTML="<span class=\"status-fail\">$MEDIA_ERR</span>"
+        fi
+
+        # Zachytenie chyby disku pre e-mail
+        if [[ "$FAIL" != "0" || "$MEDIA_ERR" != "0" ]]; then
+            EMAIL_ALERT=1
+            EMAIL_BODY="${EMAIL_BODY}\n- Disk Slot $SLOT ($MODL, SN: $SN): Media Errors = $MEDIA_ERR, Predictive Failures = $FAIL"
+        fi
+
+        cat >> $WEB/hdd.html <<EOF
+        <tr>
+            <td><b>$SLOT</b></td>
+            <td>$MODL</td>
+            <td>$SN</td>
+            <td>$SIZE</td>
+            <td><span class="type-tag">$BUS_TYPE</span></td>
+            <td>$TEMP</td>
+            <td>$MEDIA_HTML</td>
+            <td>$FAIL_HTML</td>
+        </tr>
+EOF
+        echo "$SLOT $MODL $SN $SIZE $BUS_TYPE $TEMP $MEDIA_ERR $FAIL" >> $REP
     fi
-    echo "<tr>" >> $WEB/hdd.html
-    echo "<td>$MODL</td>" >> $WEB/hdd.html
-    echo "<td>$i</td>" >> $WEB/hdd.html
-    echo "<td>$SIZE</td>" >> $WEB/hdd.html
-    echo "<td>$PORT</td>" >> $WEB/hdd.html
-    echo "<td>$TEMP</td>" >> $WEB/hdd.html
-    echo "<td>$FAIL_HTML</td>" >> $WEB/hdd.html
-    echo "</tr>" >> $WEB/hdd.html
-    echo "$MODL $i $SIZE $PORT $TEMP $FAIL" >> $REP
+    rm -f "$block"
 done
-echo "-------------------- ---------------- ---------- -------- --- -----" >> $REP
+
+echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
+rm -f /tmp/hdd.tmp
+}
+
+#------------------------------ WEB-STOP -------------------------------#
+webend(){
+cat >> $WEB/hdd.html <<EOF
+        </tbody>
+    </table>
+</div>
+</body>
+</html>
+EOF
+NUM=$(grep -vE "SERIAL|---" $REP | wc -l)
+SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}')
+sed -i "s/\[NUM\]/$NUM/g" $WEB/hdd.html
+sed -i "s/DATE/$(date '+%Y-%m-%d %H:%M:%S')/g" $WEB/hdd.html
+sed -i "s/SITE/$SITE/g" $WEB/hdd.html
+sed -i "s|CTRL_INFO|${CON:-N/A}|g" $WEB/hdd.html
+SAFE_BBU=$(echo "$BBU_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
+SAFE_TASK=$(echo "$TASK_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
+SAFE_RAID=$(echo "$RAID_HTML_ACC" | sed -e 's/[]\/$*.^[]/\\&/g')
+sed -i "s|BBU_INFO|$SAFE_BBU|g" $WEB/hdd.html
+sed -i "s|TASK_INFO|$SAFE_TASK|g" $WEB/hdd.html
+sed -i "s|RAID_INFO|$SAFE_RAID|g" $WEB/hdd.html
 }
 
 #------------------------------- CORE ----------------------------------#
 webstart
 checkhdd
 webend
-echo -e "- Final HTML report (\e[1;35mhttp://$(ifconfig vmbr0 | grep -w inet | awk '{print $2}' | sed 's/ //g')/hdd/hdd.html\e[0m)"
+IP=$(ip route get 1.1.1.1 2>/dev/null | awk '{print $7}')
+echo -e "- Final HTML report (\e[1;35mhttp://${IP:-localhost}/hdd/hdd.html\e[0m)"
 cat /tmp/hdd.rac
 cat /tmp/hdd.rai | column -t
-cat $REP | column -t
+cat $REP | column -t
+
+#------------------------------ NOTIFICATION ---------------------------#
+if [ "$EMAIL_ALERT" -eq 1 ]; then
+    if [ ! -f "$STATE" ]; then
+        SUBJECT="[CRITICAL] Storage Error on $SITE ($IP)"
+        echo -e "On $SITE was detected errors:\n$EMAIL_BODY\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
+        touch "$STATE"
+    fi
+else
+    if [ -f "$STATE" ]; then
+        SUBJECT="[RECOVERY] Storage OK on $SITE ($IP)"
+        echo -e "All drives and RAID array on $SITE are optimal state.\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
+        rm -f "$STATE"
+    fi
+fi
+EOGTR