|
@@ -2,7 +2,7 @@
|
|
|
#########################################################################
|
|
#########################################################################
|
|
|
# Script for check HDD & RAID state #
|
|
# Script for check HDD & RAID state #
|
|
|
# Created by Pavol Macko - DoIT #
|
|
# Created by Pavol Macko - DoIT #
|
|
|
-# ver 0.4 #
|
|
|
|
|
|
|
+# ver 0.7 #
|
|
|
#########################################################################
|
|
#########################################################################
|
|
|
|
|
|
|
|
#---------------------------- VARIABLES --------------------------------#
|
|
#---------------------------- VARIABLES --------------------------------#
|
|
@@ -10,9 +10,12 @@ REP=/tmp/hdd.rep
|
|
|
WEB=/var/www/hdd
|
|
WEB=/var/www/hdd
|
|
|
TMP=/tmp/hdd.tmp
|
|
TMP=/tmp/hdd.tmp
|
|
|
LOG=/tmp/hdd.log
|
|
LOG=/tmp/hdd.log
|
|
|
-test -d $WEB || mkdir -p $WEB
|
|
|
|
|
-test -f $TMP && rm -rf $TMP
|
|
|
|
|
-test -f $LOG && rm -rf $LOG
|
|
|
|
|
|
|
+STATE=/tmp/hdd.state
|
|
|
|
|
+MAIL_TO="info@doit.sk"
|
|
|
|
|
+EMAIL_ALERT=0
|
|
|
|
|
+EMAIL_BODY=""
|
|
|
|
|
+mkdir -p $WEB
|
|
|
|
|
+rm -f $TMP $LOG $REP
|
|
|
|
|
|
|
|
#----------------------------- WEB-START -------------------------------#
|
|
#----------------------------- WEB-START -------------------------------#
|
|
|
webstart(){
|
|
webstart(){
|
|
@@ -22,113 +25,401 @@ cat > $WEB/hdd.html <<\EOF
|
|
|
<head>
|
|
<head>
|
|
|
<meta charset="UTF-8">
|
|
<meta charset="UTF-8">
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
-<title>SITE HDD</title>
|
|
|
|
|
|
|
+<title>SITE - Storage</title>
|
|
|
<link rel="icon" href="logo.ico">
|
|
<link rel="icon" href="logo.ico">
|
|
|
<style>
|
|
<style>
|
|
|
- body { font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; background-color: #0f172a; color: #f8fafc; margin: 0; padding: 40px 20px; }
|
|
|
|
|
- .container { max-width: 1000px; margin: 0 auto; background: #1e293b; border-radius: 10px; box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.5); padding: 30px; border: 1px solid #334155; }
|
|
|
|
|
- h2 { text-align: center; color: #f1f5f9; margin-top: 0; margin-bottom: 6px; font-weight: 600; letter-spacing: -0.025em; }
|
|
|
|
|
- .subtitle { text-align: center; color: #94a3b8; font-size: 0.9em; margin-bottom: 30px; }
|
|
|
|
|
- table { width: 100%; border-collapse: collapse; }
|
|
|
|
|
- th, td { padding: 14px 16px; text-align: center; border-bottom: 1px solid #334155; }
|
|
|
|
|
- th { background-color: #0f172a; color: #cbd5e1; font-size: 0.8em; text-transform: uppercase; letter-spacing: 0.08em; font-weight: 700; }
|
|
|
|
|
- tr:hover { background-color: #334155; transition: background-color 0.15s ease; }
|
|
|
|
|
- .status-ok { background-color: rgba(16, 185, 129, 0.15); color: #34d399; border: 1px solid rgba(52, 211, 153, 0.3); padding: 4px 12px; border-radius: 20px; font-size: 0.85em; font-weight: 600; display: inline-block; }
|
|
|
|
|
- .status-fail { background-color: rgba(239, 68, 68, 0.15); color: #fca5a5; border: 1px solid rgba(252, 165, 165, 0.3); padding: 4px 12px; border-radius: 20px; font-size: 0.85em; font-weight: 600; display: inline-block; }
|
|
|
|
|
|
|
+ :root {
|
|
|
|
|
+ --bg-main: #0f172a;
|
|
|
|
|
+ --card-bg: #1e293b;
|
|
|
|
|
+ --border-color: #334155;
|
|
|
|
|
+ --text-muted: #94a3b8;
|
|
|
|
|
+ --accent-blue: #38bdf8;
|
|
|
|
|
+ }
|
|
|
|
|
+ body {
|
|
|
|
|
+ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
|
|
|
+ background-color: var(--bg-main);
|
|
|
|
|
+ color: #f8fafc;
|
|
|
|
|
+ margin: 0;
|
|
|
|
|
+ padding: 40px 20px;
|
|
|
|
|
+ line-height: 1.5;
|
|
|
|
|
+ }
|
|
|
|
|
+ .container {
|
|
|
|
|
+ max-width: 1150px;
|
|
|
|
|
+ margin: 0 auto;
|
|
|
|
|
+ background: var(--card-bg);
|
|
|
|
|
+ border-radius: 12px;
|
|
|
|
|
+ box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3);
|
|
|
|
|
+ padding: 32px;
|
|
|
|
|
+ border: 1px solid var(--border-color);
|
|
|
|
|
+ }
|
|
|
|
|
+ h2 {
|
|
|
|
|
+ text-align: center;
|
|
|
|
|
+ color: #f8fafc;
|
|
|
|
|
+ margin: 0 0 6px 0;
|
|
|
|
|
+ font-weight: 700;
|
|
|
|
|
+ font-size: 1.6em;
|
|
|
|
|
+ letter-spacing: -0.02em;
|
|
|
|
|
+ }
|
|
|
|
|
+ .subtitle {
|
|
|
|
|
+ text-align: center;
|
|
|
|
|
+ color: var(--text-muted);
|
|
|
|
|
+ font-size: 0.88em;
|
|
|
|
|
+ margin-bottom: 28px;
|
|
|
|
|
+ }
|
|
|
|
|
+ .cards-grid {
|
|
|
|
|
+ display: grid;
|
|
|
|
|
+ grid-template-columns: repeat(auto-fit, minmax(230px, 1fr));
|
|
|
|
|
+ gap: 16px;
|
|
|
|
|
+ margin-bottom: 30px;
|
|
|
|
|
+ }
|
|
|
|
|
+ .card {
|
|
|
|
|
+ background: #0f172a;
|
|
|
|
|
+ border: 1px solid var(--border-color);
|
|
|
|
|
+ border-radius: 8px;
|
|
|
|
|
+ padding: 16px;
|
|
|
|
|
+ transition: transform 0.2s ease, border-color 0.2s ease;
|
|
|
|
|
+ }
|
|
|
|
|
+ .card:hover {
|
|
|
|
|
+ border-color: #475569;
|
|
|
|
|
+ transform: translateY(-2px);
|
|
|
|
|
+ }
|
|
|
|
|
+ .card-title {
|
|
|
|
|
+ font-size: 0.72em;
|
|
|
|
|
+ text-transform: uppercase;
|
|
|
|
|
+ letter-spacing: 0.08em;
|
|
|
|
|
+ color: var(--text-muted);
|
|
|
|
|
+ font-weight: 700;
|
|
|
|
|
+ margin-bottom: 8px;
|
|
|
|
|
+ }
|
|
|
|
|
+ .card-content {
|
|
|
|
|
+ font-size: 0.98em;
|
|
|
|
|
+ font-weight: 600;
|
|
|
|
|
+ color: var(--accent-blue);
|
|
|
|
|
+ }
|
|
|
|
|
+ .raid-item {
|
|
|
|
|
+ display: flex;
|
|
|
|
|
+ justify-content: space-between;
|
|
|
|
|
+ align-items: center;
|
|
|
|
|
+ font-size: 0.85em;
|
|
|
|
|
+ padding: 4px 0;
|
|
|
|
|
+ border-bottom: 1px dashed var(--border-color);
|
|
|
|
|
+ }
|
|
|
|
|
+ .raid-item:last-child { border-bottom: none; }
|
|
|
|
|
+ table {
|
|
|
|
|
+ width: 100%;
|
|
|
|
|
+ border-collapse: separate;
|
|
|
|
|
+ border-spacing: 0;
|
|
|
|
|
+ border-radius: 8px;
|
|
|
|
|
+ overflow: hidden;
|
|
|
|
|
+ border: 1px solid var(--border-color);
|
|
|
|
|
+ }
|
|
|
|
|
+ th {
|
|
|
|
|
+ background-color: #0f172a;
|
|
|
|
|
+ color: #cbd5e1;
|
|
|
|
|
+ font-size: 0.75em;
|
|
|
|
|
+ text-transform: uppercase;
|
|
|
|
|
+ letter-spacing: 0.08em;
|
|
|
|
|
+ font-weight: 700;
|
|
|
|
|
+ padding: 14px;
|
|
|
|
|
+ border-bottom: 1px solid var(--border-color);
|
|
|
|
|
+ text-align: center;
|
|
|
|
|
+ }
|
|
|
|
|
+ td {
|
|
|
|
|
+ padding: 12px 14px;
|
|
|
|
|
+ text-align: center;
|
|
|
|
|
+ border-bottom: 1px solid var(--border-color);
|
|
|
|
|
+ font-size: 0.88em;
|
|
|
|
|
+ color: #e2e8f0;
|
|
|
|
|
+ }
|
|
|
|
|
+ tr:last-child td {
|
|
|
|
|
+ border-bottom: none;
|
|
|
|
|
+ }
|
|
|
|
|
+ tr:nth-child(even) {
|
|
|
|
|
+ background-color: rgba(255, 255, 255, 0.015);
|
|
|
|
|
+ }
|
|
|
|
|
+ tr:hover {
|
|
|
|
|
+ background-color: rgba(51, 65, 85, 0.5);
|
|
|
|
|
+ }
|
|
|
|
|
+ .status-ok {
|
|
|
|
|
+ background-color: rgba(16, 185, 129, 0.12);
|
|
|
|
|
+ color: #34d399;
|
|
|
|
|
+ border: 1px solid rgba(52, 211, 153, 0.25);
|
|
|
|
|
+ padding: 3px 10px;
|
|
|
|
|
+ border-radius: 20px;
|
|
|
|
|
+ font-size: 0.78em;
|
|
|
|
|
+ font-weight: 600;
|
|
|
|
|
+ display: inline-block;
|
|
|
|
|
+ }
|
|
|
|
|
+ .status-fail {
|
|
|
|
|
+ background-color: rgba(239, 68, 68, 0.15);
|
|
|
|
|
+ color: #fca5a5;
|
|
|
|
|
+ border: 1px solid rgba(252, 165, 165, 0.3);
|
|
|
|
|
+ padding: 3px 10px;
|
|
|
|
|
+ border-radius: 20px;
|
|
|
|
|
+ font-size: 0.78em;
|
|
|
|
|
+ font-weight: 600;
|
|
|
|
|
+ display: inline-block;
|
|
|
|
|
+ }
|
|
|
|
|
+ .status-warn {
|
|
|
|
|
+ background-color: rgba(245, 158, 11, 0.15);
|
|
|
|
|
+ color: #fbbf24;
|
|
|
|
|
+ border: 1px solid rgba(251, 191, 36, 0.3);
|
|
|
|
|
+ padding: 3px 10px;
|
|
|
|
|
+ border-radius: 20px;
|
|
|
|
|
+ font-size: 0.78em;
|
|
|
|
|
+ font-weight: 600;
|
|
|
|
|
+ display: inline-block;
|
|
|
|
|
+ }
|
|
|
|
|
+ .type-tag {
|
|
|
|
|
+ background-color: rgba(56, 189, 248, 0.08);
|
|
|
|
|
+ color: #38bdf8;
|
|
|
|
|
+ border: 1px solid rgba(56, 189, 248, 0.25);
|
|
|
|
|
+ padding: 2px 8px;
|
|
|
|
|
+ border-radius: 4px;
|
|
|
|
|
+ font-size: 0.78em;
|
|
|
|
|
+ font-weight: 600;
|
|
|
|
|
+ display: inline-block;
|
|
|
|
|
+ }
|
|
|
</style>
|
|
</style>
|
|
|
|
|
+
|
|
|
</head>
|
|
</head>
|
|
|
<body>
|
|
<body>
|
|
|
<div class="container">
|
|
<div class="container">
|
|
|
- <h2>Disk overview on server [SITE]</h2>
|
|
|
|
|
|
|
+ <h2>RAID Storage Status on [SITE]</h2>
|
|
|
<div class="subtitle">Number of disks: <b style="color: #f8fafc;">[NUM]</b> | Last update: DATE</div>
|
|
<div class="subtitle">Number of disks: <b style="color: #f8fafc;">[NUM]</b> | Last update: DATE</div>
|
|
|
|
|
+ <!-- Info Cards -->
|
|
|
|
|
+ <div class="cards-grid">
|
|
|
|
|
+ <div class="card">
|
|
|
|
|
+ <div class="card-title">HW RAID Controller</div>
|
|
|
|
|
+ <div class="card-content">CTRL_INFO</div>
|
|
|
|
|
+ </div>
|
|
|
|
|
+ <div class="card">
|
|
|
|
|
+ <div class="card-title">BBU / CacheVault</div>
|
|
|
|
|
+ <div class="card-content">BBU_INFO</div>
|
|
|
|
|
+ </div>
|
|
|
|
|
+ <div class="card">
|
|
|
|
|
+ <div class="card-title">Background Tasks</div>
|
|
|
|
|
+ <div class="card-content">TASK_INFO</div>
|
|
|
|
|
+ </div>
|
|
|
|
|
+ <div class="card">
|
|
|
|
|
+ <div class="card-title">Logical Arrays (RAID)</div>
|
|
|
|
|
+ RAID_INFO
|
|
|
|
|
+ </div>
|
|
|
|
|
+ </div>
|
|
|
|
|
+
|
|
|
<table>
|
|
<table>
|
|
|
<thead>
|
|
<thead>
|
|
|
<tr>
|
|
<tr>
|
|
|
|
|
+ <th>Slot</th>
|
|
|
<th>HDD-Model</th>
|
|
<th>HDD-Model</th>
|
|
|
<th>Serial</th>
|
|
<th>Serial</th>
|
|
|
<th>Size</th>
|
|
<th>Size</th>
|
|
|
- <th>Path</th>
|
|
|
|
|
|
|
+ <th>Type</th>
|
|
|
<th>Temp</th>
|
|
<th>Temp</th>
|
|
|
- <th>Fail</th>
|
|
|
|
|
|
|
+ <th>Media Err</th>
|
|
|
|
|
+ <th>Pred Fail</th>
|
|
|
</tr>
|
|
</tr>
|
|
|
</thead>
|
|
</thead>
|
|
|
<tbody>
|
|
<tbody>
|
|
|
EOF
|
|
EOF
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-#------------------------------ WEB-STOP -------------------------------#
|
|
|
|
|
-webend(){
|
|
|
|
|
-cat >> $WEB/hdd.html <<\EOF
|
|
|
|
|
- </tbody>
|
|
|
|
|
- </table>
|
|
|
|
|
-</div>
|
|
|
|
|
-</body>
|
|
|
|
|
-</html>
|
|
|
|
|
-EOF
|
|
|
|
|
-NUM=$(cat $REP | egrep -v "SERIAL|---" | wc -l)
|
|
|
|
|
-SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}')
|
|
|
|
|
-sed -i "s/NUM/$NUM/g" $WEB/hdd.html
|
|
|
|
|
-sed -i "s/DATE/`date`/g" $WEB/hdd.html
|
|
|
|
|
-sed -i "s/SITE/$SITE/g" $WEB/hdd.html
|
|
|
|
|
-}
|
|
|
|
|
-
|
|
|
|
|
#--------------------------- CHECK HDD INFO ----------------------------#
|
|
#--------------------------- CHECK HDD INFO ----------------------------#
|
|
|
checkhdd(){
|
|
checkhdd(){
|
|
|
echo -e "- \e[1;32mExtracting\e[0m HDD info from PVE hypervisor"
|
|
echo -e "- \e[1;32mExtracting\e[0m HDD info from PVE hypervisor"
|
|
|
|
|
+CON=$(megacli -ShowSummary -aALL 2>/dev/null | grep "ProductName" | head -n1 | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+BBU_RAW=$(megacli -AdpBbuCmd -aALL 2>/dev/null)
|
|
|
|
|
+if echo "$BBU_RAW" | grep -qi "Get BBU Status Failed\|Adapter.*Has No BBU"; then
|
|
|
|
|
+ BBU_HTML="<span style=\"color:#94a3b8; font-size:0.9em;\">N/A (No BBU)</span>"
|
|
|
|
|
+else
|
|
|
|
|
+ BBU_STATE=$(echo "$BBU_RAW" | grep "Battery State" | head -n1 | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ BBU_TEMP=$(echo "$BBU_RAW" | grep "Temperature" | head -n1 | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ [ -z "$BBU_STATE" ] && BBU_STATE="Optimal"
|
|
|
|
|
+ if [[ "$BBU_STATE" == "Optimal" || "$BBU_STATE" == "Operational" ]]; then
|
|
|
|
|
+ BBU_HTML="<span class=\"status-ok\">$BBU_STATE</span> <span style=\"font-size:0.85em; color:#94a3b8;\">(${BBU_TEMP:-N/A})</span>"
|
|
|
|
|
+ else
|
|
|
|
|
+ BBU_HTML="<span class=\"status-fail\">${BBU_STATE}</span> <span style=\"font-size:0.85em; color:#94a3b8;\">(${BBU_TEMP:-N/A})</span>"
|
|
|
|
|
+ EMAIL_ALERT=1
|
|
|
|
|
+ EMAIL_BODY="${EMAIL_BODY}\n- BBU/CacheVault error: State=$BBU_STATE, Temp=$BBU_TEMP"
|
|
|
|
|
+ fi
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
|
|
+REBUILD_INFO=""
|
|
|
|
|
+megacli -PDList -aALL 2>/dev/null > /tmp/pd_task.tmp
|
|
|
|
|
+csplit -s -f /tmp/pd_tblock_ /tmp/pd_task.tmp '/Device Id:/' '{*}' 2>/dev/null
|
|
|
|
|
+for block in /tmp/pd_tblock_*; do
|
|
|
|
|
+ [ -f "$block" ] || continue
|
|
|
|
|
+ if grep -q "Firmware state" "$block"; then
|
|
|
|
|
+ F_STATE=$(grep "Firmware state" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ if echo "$F_STATE" | grep -qi "Rebuild"; then
|
|
|
|
|
+ DEV_ID=$(grep "Device Id" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ ENC_ID=$(grep "Enclosure Device ID" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ PROG=$(megacli -PDRbld -ShowProg -PhysDrv "[$ENC_ID:$DEV_ID]" -aALL 2>/dev/null | grep -i "Completed" | awk -F'=' '{print $2}' | xargs)
|
|
|
|
|
+ REBUILD_INFO="Rebuild Dev $DEV_ID: ${PROG:-In Progress}"
|
|
|
|
|
+ break
|
|
|
|
|
+ fi
|
|
|
|
|
+ fi
|
|
|
|
|
+ rm -f "$block"
|
|
|
|
|
+done
|
|
|
|
|
+rm -f /tmp/pd_task.tmp /tmp/pd_tblock_*
|
|
|
|
|
+
|
|
|
|
|
+if [ -n "$REBUILD_INFO" ]; then
|
|
|
|
|
+ TASK_HTML="<span class=\"status-warn\">$REBUILD_INFO</span>"
|
|
|
|
|
+else
|
|
|
|
|
+ TASK_HTML="<span class=\"status-ok\">Idle</span>"
|
|
|
|
|
+fi
|
|
|
|
|
+
|
|
|
echo "--------------------------------------" > /tmp/hdd.rac
|
|
echo "--------------------------------------" > /tmp/hdd.rac
|
|
|
-echo -e "\e[1;37mHW-RAID-CONTROLER\e[0m" >> /tmp/hdd.rac
|
|
|
|
|
|
|
+echo -e "\e[1;37mHW-RAID-CONTROLLER\e[0m" >> /tmp/hdd.rac
|
|
|
echo "--------------------------------------" >> /tmp/hdd.rac
|
|
echo "--------------------------------------" >> /tmp/hdd.rac
|
|
|
-CON=$(megacli -ShowSummary -aALL | grep ProductName | awk '{$1=$2="";print $0}' | sed 's/ //g')
|
|
|
|
|
-echo -e "\e[1;36m$CON\e[0m" >>/tmp/hdd.rac
|
|
|
|
|
|
|
+echo -e "\e[1;36m${CON:-N/A}\e[0m" >> /tmp/hdd.rac
|
|
|
echo "------ ---------- --------" > /tmp/hdd.rai
|
|
echo "------ ---------- --------" > /tmp/hdd.rai
|
|
|
echo -e "\e[1;37mTYPE SIZE STATUS\e[0m" >> /tmp/hdd.rai
|
|
echo -e "\e[1;37mTYPE SIZE STATUS\e[0m" >> /tmp/hdd.rai
|
|
|
echo "------ ---------- --------" >> /tmp/hdd.rai
|
|
echo "------ ---------- --------" >> /tmp/hdd.rai
|
|
|
-megacli -LDInfo -L0 -a0 > /tmp/hdd.r0
|
|
|
|
|
-RAI=$(cat /tmp/hdd.r0 | grep Name | sed 's/://g' | awk '{print $2}')
|
|
|
|
|
-SIZ=$(cat /tmp/hdd.r0 | grep "Size" | egrep -v "Sector|Strip" | awk '{print $3""$4}')
|
|
|
|
|
-STA=$(cat /tmp/hdd.r0 | grep State | awk '{print $3}')
|
|
|
|
|
-echo -e "\e[1;36m$RAI $SIZ $STA\e[0m" >> /tmp/hdd.rai
|
|
|
|
|
-megacli -LDInfo -L1 -a0 > /tmp/hdd.r1
|
|
|
|
|
-RAI=$(cat /tmp/hdd.r1 | grep Name | sed 's/://g' | awk '{print $2}')
|
|
|
|
|
-SIZ=$(cat /tmp/hdd.r1 | grep "Size" | egrep -v "Sector|Strip" | awk '{print $3""$4}')
|
|
|
|
|
-STA=$(cat /tmp/hdd.r1 | grep State | awk '{print $3}')
|
|
|
|
|
-echo -e "\e[1;36m$RAI $SIZ $STA\e[0m" >> /tmp/hdd.rai
|
|
|
|
|
|
|
+
|
|
|
|
|
+RAID_HTML_ACC=""
|
|
|
|
|
+megacli -LDInfo -Lall -aALL 2>/dev/null > /tmp/ld.tmp
|
|
|
|
|
+csplit -s -f /tmp/ld_block_ /tmp/ld.tmp '/Virtual Drive:/' '{*}' 2>/dev/null
|
|
|
|
|
+for block in /tmp/ld_block_*; do
|
|
|
|
|
+ [ -f "$block" ] || continue
|
|
|
|
|
+ if grep -q "RAID Level" "$block"; then
|
|
|
|
|
+ RAI=$(grep -E "Name|RAID Level" "$block" | grep -v "User Name" | head -n1 | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ SIZ=$(grep "Size" "$block" | grep -vE "Sector|Strip" | awk -F':' '{print $2}' | xargs | tr -d ' ')
|
|
|
|
|
+ STA=$(grep "State" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ echo -e "\e[1;36m${RAI:-LD} ${SIZ:-N/A} ${STA:-N/A}\e[0m" >> /tmp/hdd.rai
|
|
|
|
|
+ if [[ "$STA" == "Optimal" ]]; then
|
|
|
|
|
+ STA_HTML="<span class=\"status-ok\">$STA</span>"
|
|
|
|
|
+ else
|
|
|
|
|
+ STA_HTML="<span class=\"status-fail\">$STA</span>"
|
|
|
|
|
+ EMAIL_ALERT=1
|
|
|
|
|
+ EMAIL_BODY="${EMAIL_BODY}\n- RAID array ${RAI:-LD} bad state: $STA"
|
|
|
|
|
+ fi
|
|
|
|
|
+ RAID_HTML_ACC="${RAID_HTML_ACC}<div class=\"raid-item\"><span><b>${RAI:-LD}</b> (${SIZ:-N/A})</span> ${STA_HTML}</div>"
|
|
|
|
|
+ fi
|
|
|
|
|
+ rm -f "$block"
|
|
|
|
|
+done
|
|
|
|
|
+rm -f /tmp/ld.tmp
|
|
|
|
|
+
|
|
|
echo "------ ---------- --------" >> /tmp/hdd.rai
|
|
echo "------ ---------- --------" >> /tmp/hdd.rai
|
|
|
-megacli -PDList -a0 > /tmp/hdd.tmp
|
|
|
|
|
-echo "-------------------- ---------------- ---------- -------- --- -----" > $REP
|
|
|
|
|
-echo -e "\e[1;37mHDD-MODEL SERIAL SIZE PATH TEMP FAIL\e[0m" >> $REP
|
|
|
|
|
-echo "-------------------- ---------------- ---------- -------- --- -----" >> $REP
|
|
|
|
|
-SN=$(cat /tmp/hdd.tmp | grep "Inquiry Data" | awk '{print $NF}')
|
|
|
|
|
-for i in $SN;
|
|
|
|
|
-do
|
|
|
|
|
- MODL=$(cat /tmp/hdd.tmp | grep -B15 $i | grep "Inquiry Data" | awk '{print $3"-"$4}')
|
|
|
|
|
- SIZE=$(cat /tmp/hdd.tmp | grep -B15 $i | grep "Raw Size" | awk '{print $3"GB"}')
|
|
|
|
|
- PORT=$(cat /tmp/hdd.tmp | grep -B15 $i | grep "Connected Port Number" | awk '{print $4}')
|
|
|
|
|
- FAIL=$(cat /tmp/hdd.tmp | grep -B20 $i | grep "Predictive Failure Count" | awk '{print $NF}')
|
|
|
|
|
- TEMP=$(cat /tmp/hdd.tmp | grep -A12 $i | grep "Temperature" | sed 's/://g' | awk '{print $3}')
|
|
|
|
|
- if [[ "$FAIL" == "0" ]]; then
|
|
|
|
|
- FAIL_HTML="<span class=\"status-ok\">0</span>"
|
|
|
|
|
- else
|
|
|
|
|
- FAIL_HTML="<span class=\"status-fail\">$FAIL</span>"
|
|
|
|
|
|
|
+echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" > $REP
|
|
|
|
|
+echo -e "\e[1;37mSLOT HDD-MODEL SERIAL SIZE TYPE TEMP MEDIA-ERR FAIL\e[0m" >> $REP
|
|
|
|
|
+echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
|
|
|
|
|
+
|
|
|
|
|
+megacli -PDList -aALL 2>/dev/null > /tmp/hdd.tmp
|
|
|
|
|
+csplit -s -f /tmp/pd_block_ /tmp/hdd.tmp '/Device Id:/' '{*}' 2>/dev/null
|
|
|
|
|
+for block in /tmp/pd_block_*; do
|
|
|
|
|
+ [ -f "$block" ] || continue
|
|
|
|
|
+ SN=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $NF}')
|
|
|
|
|
+ if [ -n "$SN" ]; then
|
|
|
|
|
+ MODL=$(grep "Inquiry Data" "$block" | awk -F':' '{print $2}' | awk '{print $1"-"$2}')
|
|
|
|
|
+ SIZE=$(grep "Raw Size" "$block" | awk -F':' '{print $2}' | awk '{print $1$2}')
|
|
|
|
|
+ SLOT=$(grep "Slot Number" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ FAIL=$(grep "Predictive Failure Count" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ MEDIA_ERR=$(grep "Media Error Count" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ TEMP=$(grep "Drive Temperature" "$block" | awk -F':' '{print $2}' | awk '{print $1}')
|
|
|
|
|
+ M_TYPE_RAW=$(grep "Media Type" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ I_TYPE_RAW=$(grep "PD Type" "$block" | awk -F':' '{print $2}' | xargs)
|
|
|
|
|
+ if echo "$M_TYPE_RAW" | grep -qi "Solid State"; then
|
|
|
|
|
+ M_TYPE="SSD"
|
|
|
|
|
+ else
|
|
|
|
|
+ M_TYPE="HDD"
|
|
|
|
|
+ fi
|
|
|
|
|
+ if echo "$I_TYPE_RAW" | grep -qi "SATA"; then
|
|
|
|
|
+ I_TYPE="SATA"
|
|
|
|
|
+ else
|
|
|
|
|
+ I_TYPE="SAS"
|
|
|
|
|
+ fi
|
|
|
|
|
+ BUS_TYPE="${I_TYPE}/${M_TYPE}"
|
|
|
|
|
+
|
|
|
|
|
+ [ -z "$SLOT" ] && SLOT="N/A"
|
|
|
|
|
+ [ -z "$FAIL" ] && FAIL="0"
|
|
|
|
|
+ [ -z "$MEDIA_ERR" ] && MEDIA_ERR="0"
|
|
|
|
|
+ [ -z "$TEMP" ] && TEMP="N/A"
|
|
|
|
|
+
|
|
|
|
|
+ if [[ "$FAIL" == "0" ]]; then
|
|
|
|
|
+ FAIL_HTML="<span class=\"status-ok\">0</span>"
|
|
|
|
|
+ else
|
|
|
|
|
+ FAIL_HTML="<span class=\"status-fail\">$FAIL</span>"
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ if [[ "$MEDIA_ERR" == "0" ]]; then
|
|
|
|
|
+ MEDIA_HTML="<span class=\"status-ok\">0</span>"
|
|
|
|
|
+ else
|
|
|
|
|
+ MEDIA_HTML="<span class=\"status-fail\">$MEDIA_ERR</span>"
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ # Zachytenie chyby disku pre e-mail
|
|
|
|
|
+ if [[ "$FAIL" != "0" || "$MEDIA_ERR" != "0" ]]; then
|
|
|
|
|
+ EMAIL_ALERT=1
|
|
|
|
|
+ EMAIL_BODY="${EMAIL_BODY}\n- Disk Slot $SLOT ($MODL, SN: $SN): Media Errors = $MEDIA_ERR, Predictive Failures = $FAIL"
|
|
|
|
|
+ fi
|
|
|
|
|
+
|
|
|
|
|
+ cat >> $WEB/hdd.html <<EOF
|
|
|
|
|
+ <tr>
|
|
|
|
|
+ <td><b>$SLOT</b></td>
|
|
|
|
|
+ <td>$MODL</td>
|
|
|
|
|
+ <td>$SN</td>
|
|
|
|
|
+ <td>$SIZE</td>
|
|
|
|
|
+ <td><span class="type-tag">$BUS_TYPE</span></td>
|
|
|
|
|
+ <td>$TEMP</td>
|
|
|
|
|
+ <td>$MEDIA_HTML</td>
|
|
|
|
|
+ <td>$FAIL_HTML</td>
|
|
|
|
|
+ </tr>
|
|
|
|
|
+EOF
|
|
|
|
|
+ echo "$SLOT $MODL $SN $SIZE $BUS_TYPE $TEMP $MEDIA_ERR $FAIL" >> $REP
|
|
|
fi
|
|
fi
|
|
|
- echo "<tr>" >> $WEB/hdd.html
|
|
|
|
|
- echo "<td>$MODL</td>" >> $WEB/hdd.html
|
|
|
|
|
- echo "<td>$i</td>" >> $WEB/hdd.html
|
|
|
|
|
- echo "<td>$SIZE</td>" >> $WEB/hdd.html
|
|
|
|
|
- echo "<td>$PORT</td>" >> $WEB/hdd.html
|
|
|
|
|
- echo "<td>$TEMP</td>" >> $WEB/hdd.html
|
|
|
|
|
- echo "<td>$FAIL_HTML</td>" >> $WEB/hdd.html
|
|
|
|
|
- echo "</tr>" >> $WEB/hdd.html
|
|
|
|
|
- echo "$MODL $i $SIZE $PORT $TEMP $FAIL" >> $REP
|
|
|
|
|
|
|
+ rm -f "$block"
|
|
|
done
|
|
done
|
|
|
-echo "-------------------- ---------------- ---------- -------- --- -----" >> $REP
|
|
|
|
|
|
|
+
|
|
|
|
|
+echo "---- -------------------- ---------------- ---------- ---------- --- ------- ----" >> $REP
|
|
|
|
|
+rm -f /tmp/hdd.tmp
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+#------------------------------ WEB-STOP -------------------------------#
|
|
|
|
|
+webend(){
|
|
|
|
|
+cat >> $WEB/hdd.html <<EOF
|
|
|
|
|
+ </tbody>
|
|
|
|
|
+ </table>
|
|
|
|
|
+</div>
|
|
|
|
|
+</body>
|
|
|
|
|
+</html>
|
|
|
|
|
+EOF
|
|
|
|
|
+NUM=$(grep -vE "SERIAL|---" $REP | wc -l)
|
|
|
|
|
+SITE=$(hostname -s | sed 's/-/ /g' | awk '{print $1}')
|
|
|
|
|
+sed -i "s/\[NUM\]/$NUM/g" $WEB/hdd.html
|
|
|
|
|
+sed -i "s/DATE/$(date '+%Y-%m-%d %H:%M:%S')/g" $WEB/hdd.html
|
|
|
|
|
+sed -i "s/SITE/$SITE/g" $WEB/hdd.html
|
|
|
|
|
+sed -i "s|CTRL_INFO|${CON:-N/A}|g" $WEB/hdd.html
|
|
|
|
|
+SAFE_BBU=$(echo "$BBU_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
|
|
|
|
|
+SAFE_TASK=$(echo "$TASK_HTML" | sed -e 's/[]\/$*.^[]/\\&/g')
|
|
|
|
|
+SAFE_RAID=$(echo "$RAID_HTML_ACC" | sed -e 's/[]\/$*.^[]/\\&/g')
|
|
|
|
|
+sed -i "s|BBU_INFO|$SAFE_BBU|g" $WEB/hdd.html
|
|
|
|
|
+sed -i "s|TASK_INFO|$SAFE_TASK|g" $WEB/hdd.html
|
|
|
|
|
+sed -i "s|RAID_INFO|$SAFE_RAID|g" $WEB/hdd.html
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
#------------------------------- CORE ----------------------------------#
|
|
#------------------------------- CORE ----------------------------------#
|
|
|
webstart
|
|
webstart
|
|
|
checkhdd
|
|
checkhdd
|
|
|
webend
|
|
webend
|
|
|
-echo -e "- Final HTML report (\e[1;35mhttp://$(ifconfig vmbr0 | grep -w inet | awk '{print $2}' | sed 's/ //g')/hdd/hdd.html\e[0m)"
|
|
|
|
|
|
|
+IP=$(ip route get 1.1.1.1 2>/dev/null | awk '{print $7}')
|
|
|
|
|
+echo -e "- Final HTML report (\e[1;35mhttp://${IP:-localhost}/hdd/hdd.html\e[0m)"
|
|
|
cat /tmp/hdd.rac
|
|
cat /tmp/hdd.rac
|
|
|
cat /tmp/hdd.rai | column -t
|
|
cat /tmp/hdd.rai | column -t
|
|
|
-cat $REP | column -t
|
|
|
|
|
|
|
+cat $REP | column -t
|
|
|
|
|
+
|
|
|
|
|
+#------------------------------ NOTIFICATION ---------------------------#
|
|
|
|
|
+if [ "$EMAIL_ALERT" -eq 1 ]; then
|
|
|
|
|
+ if [ ! -f "$STATE" ]; then
|
|
|
|
|
+ SUBJECT="[CRITICAL] Storage Error on $SITE ($IP)"
|
|
|
|
|
+ echo -e "On $SITE was detected errors:\n$EMAIL_BODY\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
|
|
|
|
|
+ touch "$STATE"
|
|
|
|
|
+ fi
|
|
|
|
|
+else
|
|
|
|
|
+ if [ -f "$STATE" ]; then
|
|
|
|
|
+ SUBJECT="[RECOVERY] Storage OK on $SITE ($IP)"
|
|
|
|
|
+ echo -e "All drives and RAID array on $SITE are optimal state.\n\nDetail report: http://${IP:-localhost}/hdd/hdd.html" | mail -s "$SUBJECT" "$MAIL_TO"
|
|
|
|
|
+ rm -f "$STATE"
|
|
|
|
|
+ fi
|
|
|
|
|
+fi
|
|
|
|
|
+EOGTR
|