#!/bin/bash
#
# Check Server Status
#
# (C) Copyright 2019-2026, Bogen Communications LLC. All rights reserved.
#

print_warn() {
    printf 'WARNING: %s\n' "$1"
}

print_info() {
    printf 'INFO: %s\n' "$1"
}

pkgver() {
    dpkg-query -W -f='${Version}' "$1" 2>/dev/null
}

check_drive() {
    local drive="$1"
    local smart_output
    local overall
    local critical=0

    echo
    echo "===== Checking $drive ====="

    # Collect SMART attributes
    if ! smart_output=$(sudo /sbin/smartctl -A "$drive" 2>/dev/null); then
        print_warn "$drive: unable to read SMART attributes"
        return
    fi

    # Optional: overall health line
    overall=$(sudo /sbin/smartctl -H "$drive" 2>/dev/null | grep -E 'SMART overall-health self-assessment test result|SMART Health Status')
    if [ -n "$overall" ]; then
        echo "$overall"
        if ! echo "$overall" | grep -qiE 'PASSED|OK'; then
            print_warn "$drive: SMART overall health check did not report PASSED/OK"
            critical=1
        fi
    fi

    # Parse interesting attributes by ID or name
    # Raw value is assumed to be last field on the line
    local realloc pending offline_unc realloc_events crc_err
    local prog_fail_chip erase_fail_chip prog_fail_total erase_fail_total
    local reserved_blocks avail_reserved temp

    realloc=$(echo "$smart_output" | awk '$1 == 5 {print $NF}')
    realloc_events=$(echo "$smart_output" | awk '$1 == 196 {print $NF}')
    pending=$(echo "$smart_output" | awk '$1 == 197 {print $NF}')
    offline_unc=$(echo "$smart_output" | awk '$1 == 198 {print $NF}')
    crc_err=$(echo "$smart_output" | awk '$1 == 199 {print $NF}')
    prog_fail_chip=$(echo "$smart_output" | awk '$1 == 175 {print $NF}')
    erase_fail_chip=$(echo "$smart_output" | awk '$1 == 176 {print $NF}')
    prog_fail_total=$(echo "$smart_output" | awk '$1 == 181 {print $NF}')
    erase_fail_total=$(echo "$smart_output" | awk '$1 == 182 {print $NF}')
    reserved_blocks=$(echo "$smart_output" | awk '$1 == 178 {print $NF}')
    avail_reserved=$(echo "$smart_output" | awk '$1 == 232 {print $NF}')
    temp=$(echo "$smart_output" | awk '$1 == 194 {print $NF}')

    # Default empty values to 0
    realloc=${realloc:-0}
    realloc_events=${realloc_events:-0}
    pending=${pending:-0}
    offline_unc=${offline_unc:-0}
    crc_err=${crc_err:-0}
    prog_fail_chip=${prog_fail_chip:-0}
    erase_fail_chip=${erase_fail_chip:-0}
    prog_fail_total=${prog_fail_total:-0}
    erase_fail_total=${erase_fail_total:-0}
    reserved_blocks=${reserved_blocks:-0}
    avail_reserved=${avail_reserved:-0}

    # Major issues
    if [ "$realloc" -gt 0 ]; then
        print_warn "$drive: Reallocated_Sector_Ct = $realloc"
        critical=1
    fi

    if [ "$pending" -gt 0 ]; then
        print_warn "$drive: Current_Pending_Sector = $pending"
        critical=1
    fi

    if [ "$offline_unc" -gt 0 ]; then
        print_warn "$drive: Offline_Uncorrectable = $offline_unc"
        critical=1
    fi

    if [ "$realloc_events" -gt 0 ]; then
        print_warn "$drive: Reallocated_Event_Count = $realloc_events"
        critical=1
    fi

    if [ "$prog_fail_chip" -gt 0 ]; then
        print_warn "$drive: Program_Fail_Count_Chip = $prog_fail_chip"
        critical=1
    fi

    if [ "$erase_fail_chip" -gt 0 ]; then
        print_warn "$drive: Erase_Fail_Count_Chip = $erase_fail_chip"
        critical=1
    fi

    if [ "$prog_fail_total" -gt 0 ]; then
        print_warn "$drive: Program_Fail_Cnt_Total = $prog_fail_total"
        critical=1
    fi

    if [ "$erase_fail_total" -gt 0 ]; then
        print_warn "$drive: Erase_Fail_Count_Total = $erase_fail_total"
        critical=1
    fi

    # SSD-specific cautionary items
    if [ "$reserved_blocks" -gt 0 ]; then
        print_warn "$drive: Used_Rsvd_Blk_Cnt_Chip = $reserved_blocks"
    fi

    if [ "$avail_reserved" -gt 0 ] && [ "$avail_reserved" -lt 100 ]; then
        print_warn "$drive: Available_Reservd_Space = $avail_reserved"
    fi

    # Usually not media failure, but still useful
    if [ "$crc_err" -gt 0 ]; then
        print_warn "$drive: UDMA_CRC_Error_Count = $crc_err (may indicate cable/interface issues)"
    fi

    # Temperature note
    if [ -n "$temp" ] && [ "$temp" -ge 60 ] 2>/dev/null; then
        print_warn "$drive: Temperature_Celsius = $temp"
    fi

    if [ "$critical" -eq 0 ]; then
        print_info "$drive: no major SMART media errors detected"
    fi
}
UPDATE_STATUS_LOG=/opt/bogen/last-update-status.txt
VERBOSE=$(find /tmp/nq-css -cmin 1 2>/dev/null)

read -d . DEBIAN_VERSION < /etc/debian_version

if [ ! -f /usr/bin/dig ]; then
   /usr/bin/apt-get -y -qq install dnsutils > /dev/null 2>&1
fi

#
# Get our Public IP address
#
myPubIpAddr="$(dig -4 TXT +short o-o.myaddr.l.google.com @ns1.google.com 2>/dev/null)"

if [ $? -ne 0 ]; then
   myPubIpAddr="$(dig -4 +short myip.opendns.com @resolver1.opendns.com 2>/dev/null)"

   if [ $? -ne 0 ]; then
      myPubIpAddr="FAILED"
   fi
fi

if [ "$myPubIpAddr" = "" ]; then
   myPubIpAddr="FAILED"
fi

NQPROD="$(/usr/local/bin/licinfo -p)"
NQVER="$(/usr/local/bin/nyquist-version)"

SERIAL_NUMBER=
if [ -e /home/bogen/serial_number ]; then
	SERIAL_NUMBER=$(< /home/bogen/serial_number)
else
	if [ -e /etc/asterisk/serial_number ]; then
		SERIAL_NUMBER=$(< /etc/asterisk/serial_number)
	fi
fi

echo ""
echo "-------------------------------------------------------------"
echo "Nyquist Server Status - $(date)"
if [ -n "$NQPROD" ]; then
echo "${NQPROD^^} $NQVER"
fi
echo "System Name: $(export MYSQL_PWD=A1B7N0Q0GEN00Z9;mysql --user=asterisk asterisk -s -s -e "SELECT name FROM system")"
if [ -n "$SERIAL_NUMBER" ]; then
	if [ "$SERIAL_NUMBER" != "CustomerServ" ]; then
		echo "System Controller Serial Number: $SERIAL_NUMBER"
	fi
fi
echo "-------------------------------------------------------------"
echo ""

if [ "$(licinfo -m)" = "INVALID" ]; then
	echo '<p style="color:red;">WARNING: Invalid License!</p>'
fi

if [ "$(licinfo -M)" = "0" ]; then
	echo '<p style="color:red;">WARNING: Your Software Update Subscription has expired.<br>Please contact your Bogen dealer to purchase a subscription renewal to continue your access to software updates.</p>'
fi

echo "Debian $(cat /etc/debian_version) - Linux kernel version:"
/bin/uname -r -v -o -m
echo ""
echo -n "Nyquist Server's Public IP Address: "
echo "${myPubIpAddr//\"/}"
echo -n "Nyquist Server's Hostname: "
hostname

if [ "$myPubIpAddr" = "FAILED" ] ;then
echo "* Failed to get server's public IP address. Check access to
ns1.google.com and resolver1.opendns.com, then try again."
fi

echo ""
echo "-------------------------"
echo "Network Interfaces Status"
echo "-------------------------"
if [ -n "$VERBOSE" ]; then
	nmcli dev status
	echo ""
	nmcli dev show
	echo ""
fi
/sbin/ip addr |tr '<' '[' |tr '>' ']'
echo ""
/sbin/ip -s link
echo ""
echo "Ethernet link speeds: "
for net in $(ls /sys/class/net)
do
	if [ "$net" != "lo" ]; then
		if [ -e /sys/class/net/$net/speed ]; then
			netspeed=$(</sys/class/net/$net/speed)
			if [ "$netspeed" = "-1" ]; then
				netspeed="No carrier detected"
			fi
			ifindex=$(</sys/class/net/$net/ifindex)
			case "$ifindex" in
				2) echo "Port A ($net) = $netspeed";;
				3) echo "Port B ($net) = $netspeed";;
				*) echo "Port ($net) = $netspeed";;
			esac
		fi
	fi
done
echo ""
echo "Routing table"
/sbin/ip route
if [ $(/sbin/ip route|grep -c "^default") -gt 1 ]; then
	echo ""
	echo "Warning: Two default routes detected, there should only be one."
	echo ""
fi
echo ""
echo "------------------------------
System Date & Time / RTC Clock
------------------------------"
echo "System Clock: $(date)"
echo "Hardware RTC: $(sudo /sbin/hwclock --show)"
echo ""
echo "-------------------------------------------"
echo "Network Time Protocol (NTP) Peers and State"
echo "-------------------------------------------"
echo ""
ntpq -p -W 128
echo ""
if [ "$(timedatectl show -p NTPSynchronized --value)" != "yes" ]; then
	echo "Warning: NTP is not syncronized, server date/time ($(date)) might be wrong."
	echo ""
fi
echo "---------------------"
echo "ARP Table (neighbors)"
echo "---------------------"
/sbin/ip neigh|awk 'BEGIN { print "IP Address       HW Address           Interface  State"} { if ($4 != "FAILED") {printf "%-16s %-20s %-10s %-20s\n", $1, $5, $3, $6} else {printf "%-16s %-20s %-10s %-20s\n", $1, $5, $3, $4}}' -
echo ""
echo "----------"
echo "Disk Space"
echo "----------"
df /
echo ""
df -h /
LOGFILE="/tmp/smartd.log"
sudo /usr/local/bin/nqts -S
SMARTD_TEMP_WARN=$(grep smartd $LOGFILE |grep "of 70"|wc -l)
#
# Find all "(Min/Max a/b)" occurrences, extract a and b, and print the max value
if [ "$SMARTD_TEMP_WARN" != "0" ]; then
	max_temp=$(sed -n 's/.*(Min\/Max \([0-9]\+\)\/\([0-9]\+\)).*/\1\n\2/p' "$LOGFILE" | sed -n '/^[0-9]\+$/p' | sort -n | tail -n 1)
	echo -e "\nWARNING: Maximum operating temperature of 70C exceeded on SSD recorded by SMART monitor $SMARTD_TEMP_WARN times during one week!"
	echo "         Maximum temperature recorded is $max_temp C"
fi

# Extract unique device names from smartd syslog lines
mapfile -t drives < <(
    grep 'smartd.*Device: /dev/' "$LOGFILE" \
    | grep -o '/dev/[[:alnum:]_./-]*' \
    | sort -u
)

if [ "${#drives[@]}" -eq 0 ]; then
    echo "No drive warnings detected by SMARTD."
else
   for drive in "${drives[@]}"; do
      check_drive "$drive"
   done
fi

rm -f $LOGFILE
echo ""
USB_INS=$(df -h|grep media)
if [ "$USB_INS" != "" ]; then
   echo -e "USB Drives:\n$(df -h|head -n 1)\n$USB_INS\n"
else
   echo -e "USB Drives: none detected\n"
fi
echo "---------------------"
echo "Display Message Space"
echo "---------------------"
df -h /srv/tftp/msg|awk '{print $2,$3,$4}' -
echo ""
echo "-------------------"
echo "Disk space used by:"
echo "-------------------"
spaces="      "
cd /var/lib/asterisk/sounds/user
du -s -h *|grep -v index.php
cd /var/opt/nyquist
du -s -h *|grep -v "database\|demo\|ha\|laravel\|use-manual-ntp\|logrotate.d\|etc\|custom_config.conf\|cksums\|mqtt.passwd"
cd /var
du -s -h log
cd /var/www/html/laravel/public
du -s -h maps
cd /var/spool/asterisk
du -s -h voicemail

CDR_SPACE="$(du -s -h /var/www/html/laravel/public/message_images|awk '{print $1}' -)"
echo "$CDR_SPACE ${spaces:1:6 - ${#CDR_SPACE}} display message images"

CDR_SPACE="$(du -s -h /var/log/asterisk/cdr-csv |awk '{print $1}' -)"
echo "$CDR_SPACE ${spaces:1:6 - ${#CDR_SPACE}} call detail records"

CDR_SPACE="$(du -s -h -c /var/spool/asterisk/monitor |grep total|awk '{print $1}' -)"
echo "$CDR_SPACE ${spaces:1:6 - ${#CDR_SPACE}} recorded calls"

FW_SPACE="$(du -s -h -c /srv/tftp/NyqUpdate* |grep total|awk '{print $1}' -)"
echo "$FW_SPACE ${spaces:1:6 - ${#FW_SPACE}} nyquist device firmware"

CFG_SPACE="$(du -s -h -c /srv/tftp/*.cfg /srv/tftp/*.xml /srv/tftp/backup |grep total|awk '{print $1}' -)"
echo "$CFG_SPACE ${spaces:1:6 - ${#CFG_SPACE}} nyquist device configuration files"

SYS_UPDATES="$(du -s -h /var/www/html/laravel/public/updates |awk '{print $1}' -)"
echo "$SYS_UPDATES ${spaces:1:6 - ${#SYS_UPDATES}} nyquist system updates"

PREV_INSTALLS="$(du -s -h /opt/bogen 2>/dev/null |awk '{print $1}' -)"
if [ "$PREV_INSTALLS" = "" ]; then
   PREV_INSTALLS="0K"
fi
echo "$PREV_INSTALLS ${spaces:1:6 - ${#PREV_INSTALLS}} previous nyquist system install/updates"

echo ""
echo "---------------
Uptime / Status
---------------"
echo "System $(uptime -p)"
echo -e "\nMemory/IO/CPU"
vmstat
free
echo ""
echo "Swap Configuration"
/sbin/swapon --show=NAME,TYPE,SIZE,USED,PRIO
SWAPSTAT=$(/sbin/swapon -s)
if [ "$SWAPSTAT" = "" ]; then
   echo '<p style="color:red;">WARNING: Swap not enabled, please enable swap!</p>'
fi
echo ""
echo "Temperature Sensors"
sensors
if [ "$SMARTD_TEMP_WARN" != "0" ]; then
	echo -e "\nWARNING: Maximum operating temperature of 70C exceeded on SSD recorded by SMART monitor $SMARTD_TEMP_WARN times during one week!"
	echo -e "         Maximum temperature recorded is $max_temp C\n"
fi
# Check CSTATE Config
if [ "$(grep GRUB_CMDLINE_LINUX= /etc/default/grub)" != "GRUB_CMDLINE_LINUX=\"intel_idle.max_cstate=0 processor.max_cstate=1\"" ]; then
	echo ""
	echo "*** WARNINIG: CSTATE throttle not disabled, this could cause random server shutdowns!"
	echo "*** You need to apply disable-cstate patch ASAP!"
	echo ""
fi
echo "--------------------------
Nyquist Operational Status
--------------------------"
/usr/local/bin/nqts -o -w -n
echo -e -n "\nChecking dpkg status..."
dpkg -C
echo -e "done\n"

if [ -e /etc/asterisk/weather_county ]; then
	echo "Cached NWS Alert County Code: $(</etc/asterisk/weather_county)"
	echo ""
fi

sudo /usr/local/bin/display-call-forwarding
echo ""

/usr/local/bin/find-bell-schedule-overlap -c -e -s

echo "Software Components:"
echo " $(/usr/sbin/asterisk -V)"
echo " apache2 $(pkgver apache2)"
echo " bash $(pkgver bash)"
echo " bind9 $(pkgver bind9-host)"
echo " binutils $(pkgver binutils)"
echo " corosync $(pkgver corosync)"
echo " cron $(pkgver cron)"
echo " curl $(pkgver curl)"
echo " dnsutils $(pkgver dnsutils)"
echo " exim4 $(pkgver exim4-base)"
echo " fail2ban $(pkgver fail2ban)"
echo " $(/usr/local/bin/ffmpeg -version | awk 'NR==1 {print $1,$2,$3}')"
echo " glibc $(pkgver libc6)"
echo " isc-dhcp-server $(pkgver isc-dhcp-server)"
echo " Laravel Framework 9.52.16"

gnutls_pkg=$(dpkg -l | awk '/^ii  libgnutls[0-9]+:/{print $2; exit}')
gnutls_ver=$(dpkg-query -W -f='${Version}' "$gnutls_pkg" 2>/dev/null)
echo " libgnutls ($gnutls_pkg) $gnutls_ver"

echo " libssh $(pkgver libssh-gcrypt-4)"
echo " libssl $(pkgver libssl3)"
echo " linux kernel $(uname -r -v -o -m)"
echo " lsyncd $(pkgver lsyncd)"
echo " mosquitto $(pkgver mosquitto)"
echo " msmtp $(pkgver msmtp)"
echo " $(mysql --version | awk -F, '{print $1}')"
echo " nmap $(pkgver nmap)"
echo " ntpsec $(pkgver ntpsec)"
echo " openssh $(pkgver openssh-server)"
echo " openssl $(pkgver openssl)"
echo " pacemaker $(pkgver pacemaker)"

if [ "$DEBIAN_VERSION" != "12" ]; then
    echo " php $(pkgver php7.4)"
else
    echo " php $(pkgver php8.2)"
fi

echo " php-ldap $(pkgver php-ldap)"
echo " pjsip 2.15.1"
echo " python3 $(pkgver python3)"
echo " rsync $(pkgver rsync)"
echo " rsyslog $(pkgver rsyslog)"
echo " smartmontools $(pkgver smartmontools)"
echo " snmpd $(pkgver snmpd)"
echo " sox $(pkgver sox)"
echo " sudo $(pkgver sudo)"
echo " systemd $(pkgver systemd)"
echo " tar $(pkgver tar)"
echo " tcpdump $(pkgver tcpdump)"
echo " tftpd-hpa $(pkgver tftpd-hpa)"
echo " wget $(pkgver wget)"
echo

if [ -e $UPDATE_STATUS_LOG ]; then
   echo "Status from last System Update:"
   cat $UPDATE_STATUS_LOG
fi

if [ -n "$VERBOSE" ]; then
   echo -e "\nLast few server reboots..."
   last -x | grep -E "shutdown|reboot|runlevel" | head -n 20

   echo -e "\nNyquist MySQL table status..."
   sudo /usr/local/bin/nqts -m
fi

# Update firmware table to make sure all stored firmware is in the table.
/usr/local/bin/update-firmware -t &> /dev/null

touch /tmp/nq-css
chown www-data:www-data /tmp/nq-css
