#!/bin/sh -e
# 2026-10-10 Hyperling
# Stop a system from going into swap due to LLMs.

### Tuning ###

typeset -i sleep=10 ram_pct_restart=98

### Functions ###

restart_service() {
	if [ -z $1 ]; then
		echo "ERROR: Service name must be passed to restart_service()."
		return 1
	fi
	sv restart $1 || echo "$1 failed to restart, ignoring."
}

# While we could loop over `ollama ps` and stop them, this ensures that the UIs
#   do not sit around waiting for responses from the dead LLMs. It bring them up
#   in a fresh state so that work can continue from a more clean slate.
restart_llms() {
	restart_service ollama
	restart_service zeroclaw
	restart_service open-webui
}

### Main ###

typeset -i {total,used}_{ram,swap}=0 \
	used_{ram,swap}_pct=0 \
	last_{ram,swap}_pct=100 \
	count=0

while [[ true ]]; do

	count=$(expr $count + 1 )
	echo "$(date +'%Y-%m-%d @ %H:%M:%S') - Loop $count"

	# RAM #

	total_ram=$(free | awk '/^Mem:/ {print $2}')
	used_ram=$(free | awk '/^Mem:/ {print $3}')
	used_ram_pct=$( \
		awk -v used="$used_ram" -v total="$total_ram" \
		'BEGIN {printf "%d",(used/total)*100}' \
	)

	echo "Current RAM usage is $used_ram_pct%, was $last_ram_pct% last loop."

	# Check if RAM is over 95%
	if [ $used_ram_pct -gt $ram_pct_restart ]; then
		echo "WARNING: RAM is over ${ram_pct_restart}% used. Restarting LLMs."
		restart_llms
		continue
	fi

	# Swap #

	total_swap=$(free | awk '/^Swap:/ {print $2}')
	used_swap=$(free | awk '/^Swap:/ {print $3}')
	used_swap_pct=$( \
		awk -v used="$used_swap" -v total="$total_swap" \
		'BEGIN {printf "%d",(used/total)*100}' \
	)

	echo "Current swap usage is $used_swap_pct%, was $last_swap_pct% last loop."

	# Check if swap is growing
	if [ $used_swap_pct -gt $last_swap_pct ]; then
		echo "WARNING: Swap is growing. Restarting LLMs."
		restart_llms
		continue
	fi

	# Finish #

	echo -e "Check finished, sleeping for ${sleep}...\n"

	last_ram_pct=$used_ram_pct
	last_swap_pct=$used_swap_pct

	sleep $sleep
done