#!/usr/bin/env bash
# VICIfast Research, Report 01: The Two Bottlenecks Inside a Busy VICIdial Server
# Test harness. FOR A TEST SERVER, NOT A LIVE FLOOR. Read it before you run it.
# AMI load generator. It ADDS an AMI user to manager.conf and drives synthetic load.
# Test servers only.
#
# env: LABEL, LOOPS (space list), SECS. Emulator must be stopped by caller or here.
set +e
W=/root/vf-engine-ab/ami; M=/etc/asterisk/manager.conf
P=$(pidof asterisk | awk '{print $1}'); HZ=$(getconf CLK_TCK)
TP=$(asterisk -rx 'core show taskprocessors' | awk '/stasis\/m:manager:core/{print $1; exit}')
pr(){ asterisk -rx 'core show taskprocessors' | awk -v n="$TP" '$1==n{print $2, $3, $4}'; }
thr(){ for t in /proc/$P/task/*; do awk -v t=${t##*/} '{print t, $14+$15}' $t/stat 2>/dev/null; done; }
asterisk -rx 'dialplan remove context vf-amiload' >/dev/null 2>&1
# Every executed priority is logged at the box's verbosity: keep that out of the measurement (same for every run).
cp -n /etc/asterisk/logger.conf $W/logger.conf.orig
sed -i 's/^messages => .*/messages => notice,warning,error,dtmf/' /etc/asterisk/logger.conf; asterisk -rx 'logger reload' >/dev/null
asterisk -rx 'dialplan add extension s,1,Set(I=0) into vf-amiload' >/dev/null
asterisk -rx 'dialplan add extension "s,2,UserEvent(VFLoad,N: ${I})" into vf-amiload' >/dev/null
asterisk -rx 'dialplan add extension s,3,Set(I=$[${I}+1]) into vf-amiload' >/dev/null
asterisk -rx 'dialplan add extension s,4,GotoIf($[${I}<50000000]?2) into vf-amiload' >/dev/null
asterisk -rx 'dialplan show vf-amiload' | grep -cE '^ +([0-9]\.|.s. =>)' | sed 's/^/dialplan lines: /'
echo "$(asterisk -rx 'manager show settings' | grep -E 'Event threads|Disabled events' | tr -s ' ' | tr '\n' ' ')"
S=$(head -c 24 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 20)
sed -i '/^\[vfabcount\]/,/^\[/{/^\[vfabcount\]/d;/^\[/!d}' $M
printf '\n[vfabcount]\nsecret = %s\ndeny = 0.0.0.0/0.0.0.0\npermit = 127.0.0.1/255.255.255.255\nread = all\nwrite = command\n' "$S" >> $M
asterisk -rx 'manager reload' >/dev/null; sleep 1
for K in $LOOPS; do
  asterisk -rx "core reset taskprocessor $TP" >/dev/null 2>&1
  thr > $W/thr0; k0=$(awk '{print $14+$15}' /proc/$P/stat); read p0 q0 m0 < <(pr); t0=$(date +%s.%N)
  python3 $W/tally2.py "$S" $SECS > $W/bench-$LABEL-$K.txt &
  TP2=$!
  for i in $(seq 1 $K); do asterisk -rx 'channel originate Local/s@vf-amiload application Wait 600' >/dev/null; done
  end=$(( $(date +%s) + SECS )); TRIP=""; BPMAX=0; RSSMAX=0
  while [ $(date +%s) -lt $end ]; do
    read pq qq mq < <(pr)
    bp=$(asterisk -rx 'manager show settings' | awk '/Event batches/{print $3}'); bp=${bp:-0}
    rss=$(awk '/VmRSS/{print $2}' /proc/$P/status); [ "$bp" -gt "$BPMAX" ] && BPMAX=$bp; [ "$rss" -gt "$RSSMAX" ] && RSSMAX=$rss
    if [ "${qq:-0}" -gt 200000 ] || [ "$bp" -gt 200000 ] || [ "$rss" -gt 3000000 ]; then TRIP="GUARD TRIPPED (queue $qq, batches $bp, rss $((rss/1024))MB)"; break; fi
    sleep 0.5
  done
  read p1 q1 m1 < <(pr); t1=$(date +%s.%N); k1=$(awk '{print $14+$15}' /proc/$P/stat); thr > $W/thr1
  for c in $(asterisk -rx 'core show channels concise' | awk -F'!' '$2=="vf-amiload" || $1 ~ /^Local\/s@vf-amiload/ {print $1}'); do asterisk -rx "channel request hangup $c" >/dev/null; done
  wait $TP2
  # how long the router needs to drain what it is behind on
  d0=$(date +%s.%N); for i in $(seq 1 300); do read pq qq mq < <(pr); [ "${qq:-0}" -le 5 ] && break; sleep 0.2; done; dr=$(echo "$(date +%s.%N)-$d0" | bc)
  EL=$(echo "$t1-$t0" | bc)
  TOPT=$(join <(sort $W/thr0) <(sort $W/thr1) | awk -v hz=$HZ -v el=$EL '{printf "%.2f\n", ($3-$2)/hz/el}' | sort -rn | head -4 | tr '\n' ' ')
  printf "%s K=%d: router processed %.0f/s, queue at end %d (max %d), drain after stop %.1fs | asterisk %.2f cores, top threads (cores): %s\n" $LABEL $K $(echo "($p1-$p0)/$EL" | bc) $q1 $m1 $dr $(echo "scale=2; ($k1-$k0)/$HZ/$EL" | bc) "$TOPT"
  echo "     batches pending max $BPMAX | asterisk RSS max $((RSSMAX/1024))MB ${TRIP}"
  grep -E "VFLoad|TOTAL" $W/bench-$LABEL-$K.txt | sed 's/^/     /'
  sleep 3
done
sed -i '/^\[vfabcount\]/,/^\[/{/^\[vfabcount\]/d;/^\[/!d}' $M; sed -i -e :a -e '/^\n*$/{$d;N;ba' -e '}' $M; asterisk -rx 'manager reload' >/dev/null
asterisk -rx 'dialplan remove context vf-amiload' >/dev/null 2>&1
cp $W/logger.conf.orig /etc/asterisk/logger.conf; asterisk -rx 'logger reload' >/dev/null
echo "logger restored: $(grep '^messages' /etc/asterisk/logger.conf)"
echo "left: vf-amiload channels $(asterisk -rx 'core show channels concise' | grep -c vf-amiload) | vfabcount $(grep -c vfabcount $M)"