#!/usr/bin/env bash
# VICIfast Research, Report 01: "The Two Bottlenecks Inside a Busy VICIdial Server"
# Check 3: is Asterisk's AMI keeping up (event thread and connection queue), and is it leaking?
#
# READ-ONLY. Runs a few `asterisk -rx` show commands and reads two kernel counters for 10 seconds;
# safe on a live dialer.
# Usage:  sudo bash asterisk-check-ami.sh
#
# Every manager event VICIdial reads goes through one Asterisk thread. On
# Asterisk 21 and 22 each `dialplan reload` also leaves one more subscriber on
# the manager topic while /etc/asterisk/users.conf exists, and each subscriber
# handles every manager message again (section 7.7 of the report). VICIdial also opens a new AMI
# connection for every originate and every answered call, into a queue Asterisk keeps at 10 slots;
# when that queue overflows, connections wait 1-3 s and answered calls are lost (sections 7.1-7.3).
set -u
A() { asterisk -rx "$1" 2>/dev/null; }
command -v asterisk >/dev/null || { echo "asterisk not found: run this on a dialer." >&2; exit 1; }
A 'core show version' | grep -q Asterisk || { echo "Cannot reach the Asterisk console (run as root)." >&2; exit 1; }

echo "== AMI check on $(hostname -s), $(date '+%Y-%m-%d %H:%M:%S %Z')"
ver=$(A 'core show version' | awk '{print $2}')
up=$(A 'core show uptime seconds' | awk -F': ' '/System uptime/ {print $2}')
calls=$(A 'core show channels count' | awk '/active call/ {print $1}')
echo "-- Asterisk ${ver}, up $(( ${up:-0} / 86400 ))d $(( ${up:-0} % 86400 / 3600 ))h, ${calls:-0} active calls"

TP=$(A 'core show taskprocessors')
leaked=$(printf '%s\n' "$TP" | grep -c 'stasis/p:manager:core-')
router=$(printf '%s\n' "$TP" | awk '$1 ~ /^stasis\/m:manager:core/ { print "in queue " $3 ", max depth " $4 }' | head -1)
rq=$(printf '%s\n' "$TP" | awk '$1 ~ /^stasis\/m:manager:core/ { print $3 + 0; exit }')
worst=$(printf '%s\n' "$TP" | awk '$1 ~ /^stasis\/p:manager:core-/ && $4+0 > m { m = $4+0 } END { print m+0 }')
echo "-- manager topic router: ${router:-not found}"
echo "-- manager-topic pool subscribers: ${leaked} (deepest queue reached: ${worst})"
if [ -e /etc/asterisk/users.conf ]; then
  echo "-- /etc/asterisk/users.conf is present"
else
  echo "-- /etc/asterisk/users.conf is absent"
fi
trig=$(awk -F'=' '/^[[:space:]]*taskprocessor_overload_trigger/ { gsub(/[[:space:]]/, "", $2); print $2 }' /etc/asterisk/asterisk.conf 2>/dev/null)
echo "-- taskprocessor_overload_trigger: ${trig:-global (default)}"
ami=$(A 'manager show connected' | grep -cE '^[[:space:]]*[A-Za-z0-9_.-]+[[:space:]]+[0-9]')
echo "-- AMI connections: ${ami}"
port=$(awk -F'=' '/^[[:space:]]*port[[:space:]]*=/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /etc/asterisk/manager.conf 2>/dev/null)
port=${port:-5038}
lq=$(ss -ltnH "sport = :${port}" 2>/dev/null | awk '{ print $2 " waiting, queue size " $3; exit }')
echo "-- AMI listen queue on port ${port}: ${lq:-not found}"
ovf() { awk '/^TcpExt:/ { if (!h) { for (i = 1; i <= NF; i++) n[i] = $i; h = 1 } else { for (i = 1; i <= NF; i++) if (n[i] == "ListenOverflows") print $i } }' /proc/net/netstat; }
o1=$(ovf); sleep 10; o2=$(ovf)
drops=$(( ${o2:-0} - ${o1:-0} ))
echo "-- connection attempts the kernel dropped on full listen queues: ${drops} in 10 s (${o2:-?} since boot, all ports)"
echo
echo "-- Verdict"
major=${ver%%.*}
qsize=$(printf '%s' "$lq" | awk '{ print $NF }')
if [ "${drops:-0}" -gt 0 ] && [ "${qsize:-0}" -le 128 ]; then
  echo "   ${drops} dropped connection attempts in 10 s, and the AMI queue holds only ${qsize:-?}."
  echo "   AMI clients wait 1-3 s to connect. On VICIdial that loses answered calls (section 7.2);"
  echo "   see section 7.3 for the fix. Other services count in this number too: run it again at your"
  echo "   busiest minute, and compare with 'ss -ltn' for port ${port}."
fi
if [ "$leaked" -gt 0 ] && { [ "$major" = 21 ] || [ "$major" = 22 ]; }; then
  echo "   ${leaked} leaked manager-topic subscriber(s): every AMI message is handled $((leaked + 1)) times."
  echo "   Stop new ones: move /etc/asterisk/users.conf aside if you do not use it"
  echo "   (mv /etc/asterisk/users.conf /etc/asterisk/users.conf.disabled). The existing ones clear at"
  echo "   the next Asterisk restart; do that with no calls up."
elif [ "${rq:-0}" -ge 3000 ]; then
  echo "   The manager router is ${rq} events behind: VICIdial sees answers and hangups late."
  echo "   Apply the AMI trim (section 7.5); if it stays behind at your peak, see section 7.3."
elif [ "${worst:-0}" -ge 300 ]; then
  echo "   A manager subscriber's queue reached ${worst}. Near 500 Asterisk raises its overload alert."
else
  echo "   No leak, and the manager queues are shallow."
fi