#!/usr/bin/env bash
# VICIfast Research, Report 01: The Two Bottlenecks Inside a Busy VICIdial Server
# Test harness. FOR A TEST SERVER, NOT A LIVE FLOOR. Read it before you run it.
# Pressure-step sampler. With LEVEL set it CHANGES the dial level of two campaigns: edit
# the campaign ids (MEDOUT, BLEND) for your server.
#
# env: LEVEL (blank = leave), SETTLE, SECS, LABEL
set +e
DB=$(awk -F'=> *' '/^VARDB_database/{print $2}' /etc/astguiclient.conf | tr -d ' \r'); M="mysql -N $DB"
W=/root/vf-engine-ab/pressure; mkdir -p $W
[ -n "$LEVEL" ] && $M -e "update vicidial_campaigns set auto_dial_level='$LEVEL' where campaign_id in ('MEDOUT','BLEND')"
sleep ${SETTLE:-0}
t0=$(date '+%F %T'); q0=$(mysql -N -e "show global status like 'Questions'" | awk '{print $2}'); lw0=$(mysql -N -e "show global status like 'Table_locks_waited'" | awk '{print $2}')
vmstat 5 $(( SECS/5 + 1 )) | tail -n +4 > $W/vm.$LABEL &
end=$(( $(date +%s) + SECS ))
: > $W/s.$LABEL
while [ $(date +%s) -lt $end ]; do
  ch=$(asterisk -rx 'core show channels count' 2>/dev/null | awk '/active channels/{print $1}')
  echo "$ch $(mysql -N -e "select (select count(*) from $DB.vicidial_auto_calls), (select count(*) from $DB.vicidial_live_agents where status in ('READY','CLOSER')), (select variable_value from information_schema.global_status where variable_name='THREADS_RUNNING'), (select count(*) from information_schema.processlist where command='Query' and time>=1 and user not in ('system user')), (select count(*) from information_schema.processlist where state='Waiting for table level lock'), (select count(*) from $DB.vicidial_hopper)")" >> $W/s.$LABEL
  mysql -N -e "select user, left(replace(info,'\\n',' '),110) from information_schema.processlist where state in ('Waiting for table level lock','Locked') union all select concat('HOLDER:',user), left(replace(info,'\\n',' '),110) from information_schema.processlist where command='Query' and time>=1 and user not in ('system user') and state not in ('Waiting for table level lock')" >> $W/lk.$LABEL 2>/dev/null
  sleep 3
done
wait
t1=$(date '+%F %T'); q1=$(mysql -N -e "show global status like 'Questions'" | awk '{print $2}'); lw1=$(mysql -N -e "show global status like 'Table_locks_waited'" | awk '{print $2}')
read calls drops < <($M -e "select count(*), sum(status='DROP') from vicidial_log where call_date between '$t0' and '$t1'")
read dials < <($M -e "select count(*) from vicidial_dial_log where call_date between '$t0' and '$t1'")
lvl=$($M -e "select group_concat(concat(campaign_id,'=',auto_dial_level)) from vicidial_campaigns where campaign_id in ('MEDOUT','BLEND')")
avail=$(free -m | awk '/Mem/{print $7}'); swp=$(free -m | awk '/Swap/{print $3}')
awk '{us+=$13; sy+=$14; id+=$15; wa+=$16; si+=$7; so+=$8; n++} END{printf "cpu us %.0f sy %.0f idle %.0f wa %.0f | swap in/out %d/%d", us/n, sy/n, id/n, wa/n, si, so}' $W/vm.$LABEL > $W/cpu.$LABEL
awk -v L="$LABEL" -v lvl="$lvl" -v S=$SECS -v c=$calls -v d=${drops:-0} -v dl=$dials -v q=$((q1-q0)) -v lw=$((lw1-lw0)) -v av=$avail -v sw=$swp -v cpu="$(cat $W/cpu.$LABEL)" -v la="$(cut -d' ' -f1-3 /proc/loadavg)" '
 {ch+=$1; if($1>mch)mch=$1; ac+=$2; fr+=$3; tr+=$4; if($4>mtr)mtr=$4; st+=$5; if($5>mst)mst=$5; tl+=$6; if($6>mtl)mtl=$6; hp+=$7; n++}
 END{printf "%s [%s] %s | load %s | mem avail %dMB swap used %dMB\n   channels avg %.0f max %d | auto_calls avg %.0f | dials/s %.2f | vicidial_log calls/s %.2f drops %d (%.1f%%) | free agents %.1f | qps %.0f | threads_running avg %.1f max %d | stmts>=1s avg %.1f max %d | tbl-lock waiting avg %.1f max %d, waits/s %.1f | hopper avg %.0f\n",
  L, lvl, cpu, la, av, sw, ch/n, mch, ac/n, dl/S, c/S, d, (c?100*d/c:0), fr/n, q/S, tr/n, mtr, st/n, mst, tl/n, mtl, lw/S, hp/n}' $W/s.$LABEL
echo "   waiting/holding statement fingerprints (top 8):"
sed -E "s/'[^']*'/?/g; s/[0-9]+(\.[0-9]+)?/N/g" $W/lk.$LABEL 2>/dev/null | sort | uniq -c | sort -rn | head -8 | cut -c1-150 | sed 's/^/     /'