#!/bin/bash
#
# rss_watch.sh
# Lightweight periodic memory (RSS) logger for suspect PHP processes.
# Designed to run via cron every 5 minutes, appending one line per matching
# process to a CSV log. Negligible overhead — just parses /proc, no daemon.
#
# Setup (as root or via the owning user's crontab):
#   chmod +x rss_watch.sh
#   */5 * * * * /path/to/rss_watch.sh
#
# Log file: /var/log/rss_watch.csv (change LOGFILE below if you'd rather it
# live under the ottar account's home directory instead of /var/log).
#
# Columns: timestamp,pid,etime,rss_kb,cmd
#
# To graph/inspect later:
#   grep "datadomain=43332" /var/log/rss_watch.csv
#   awk -F',' '{print $1, $4}' /var/log/rss_watch.csv | column -t

set -uo pipefail

LOGFILE="/var/log/rss_watch.csv"
MEMLOGFILE="/var/log/mem_watch.csv"

# Added php-fpm pool workers for faretrak.net (the master process line is
# deliberately excluded further down via grep -v "master process", since we
# only want the actual worker children that serve requests and hold per-
# request memory).
PATTERN="realtime_daemon.php|getRealtimeOTTAR.php|realtime_stop_recorder.php|php-fpm: pool faretrak_net"

# Write header once if the file doesn't exist yet
if [[ ! -f "$LOGFILE" ]]; then
    echo "timestamp,pid,etime,rss_kb,cmd" > "$LOGFILE"
fi
if [[ ! -f "$MEMLOGFILE" ]]; then
    echo "timestamp,total_kb,used_kb,free_kb,available_kb,swap_used_kb,fpm_worker_count,fpm_worker_rss_total_kb" > "$MEMLOGFILE"
fi

TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S %Z')

# ps output: PID ELAPSED RSS(kb) CMD  -- filtered to our suspect processes,
# excluding the bash wrapper/launcher lines, grep itself, and the php-fpm
# master process (we only want worker children, which is where per-request
# memory usage actually shows up).
# Uses awk throughout (not cut) because ps right-pads columns with variable
# whitespace; cut -d' ' misparses that and leaves stray numbers in the cmd field.
ps -eo pid,etime,rss,cmd | grep -E "$PATTERN" | grep -v grep | grep -v "/bin/bash -c" | grep -v "master process" | \
awk -v ts="$TIMESTAMP" '{
    pid = $1; etime = $2; rss = $3;
    cmd = "";
    for (i = 4; i <= NF; i++) { cmd = cmd $i (i < NF ? " " : "") }
    gsub(/,/, ";", cmd);
    printf "%s,%s,%s,%s,\"%s\"\n", ts, pid, etime, rss, cmd
}' >> "$LOGFILE"

# System-wide memory snapshot each run, so mem_watch.csv can be directly
# correlated against "connection refused" incident times without having to
# cross-reference free -h output captured manually after the fact.
read -r TOTAL_KB USED_KB FREE_KB AVAIL_KB <<< "$(free -k | awk '/^Mem:/ {print $2, $3, $4, $7}')"
SWAP_USED_KB=$(free -k | awk '/^Swap:/ {print $3}')

# Count and sum RSS of faretrak.net PHP-FPM worker children specifically,
# so we can see pool-level pressure (e.g. all 5 pm.max_children busy at
# once) even on runs where individual worker PIDs churn too fast for the
# per-process CSV above to catch a steady trend.
read -r FPM_COUNT FPM_RSS_TOTAL <<< "$(ps -eo rss,cmd | grep "php-fpm: pool faretrak_net" | grep -v grep | \
    awk '{sum += $1; count++} END {print count+0, sum+0}')"

echo "${TIMESTAMP},${TOTAL_KB:-0},${USED_KB:-0},${FREE_KB:-0},${AVAIL_KB:-0},${SWAP_USED_KB:-0},${FPM_COUNT},${FPM_RSS_TOTAL}" >> "$MEMLOGFILE"