90 lines
2.1 KiB
Bash
Executable File
90 lines
2.1 KiB
Bash
Executable File
#!/bin/sh -e
|
|
# 2026-10-10 Hyperling
|
|
# Stop a system from going into swap due to LLMs.
|
|
|
|
### Tuning ###
|
|
|
|
typeset -i sleep=10 full_pct=98 high_pct=90
|
|
|
|
### Functions ###
|
|
|
|
restart_service() {
|
|
if [ -z $1 ]; then
|
|
echo "ERROR: Service name must be passed to restart_service()."
|
|
return 1
|
|
fi
|
|
sv restart $1 || echo "$1 failed to restart, ignoring."
|
|
}
|
|
|
|
# While we could loop over `ollama ps` and stop them, this ensures that the UIs
|
|
# do not sit around waiting for responses from the dead LLMs. It bring them up
|
|
# in a fresh state so that work can continue from a more clean slate.
|
|
restart_llms() {
|
|
restart_service ollama
|
|
#restart_service zeroclaw
|
|
#restart_service open-webui
|
|
#restart_service qwen-code
|
|
}
|
|
|
|
### Main ###
|
|
|
|
typeset -i {total,used}_{ram,swap}=0 \
|
|
used_{ram,swap}_pct=0 \
|
|
last_{ram,swap}_pct=100 \
|
|
count=0
|
|
|
|
while [[ true ]]; do
|
|
|
|
count=$(expr $count + 1 )
|
|
echo "$(date +'%Y-%m-%d @ %H:%M:%S') - Loop $count"
|
|
|
|
# RAM #
|
|
|
|
total_ram=$(free | awk '/^Mem:/ {print $2}')
|
|
used_ram=$(free | awk '/^Mem:/ {print $3}')
|
|
used_ram_pct=$( \
|
|
awk -v used="$used_ram" -v total="$total_ram" \
|
|
'BEGIN {printf "%d",(used/total)*100}' \
|
|
)
|
|
|
|
echo "Current RAM usage is $used_ram_pct%, was $last_ram_pct% last loop."
|
|
|
|
# Swap #
|
|
|
|
total_swap=$(free | awk '/^Swap:/ {print $2}')
|
|
used_swap=$(free | awk '/^Swap:/ {print $3}')
|
|
used_swap_pct=$( \
|
|
awk -v used="$used_swap" -v total="$total_swap" \
|
|
'BEGIN {printf "%d",(used/total)*100}' \
|
|
)
|
|
|
|
echo "Current swap usage is $used_swap_pct%, was $last_swap_pct% last loop."
|
|
|
|
# If RAM is full and swap is growing then restart LLM services.
|
|
if [ $used_ram_pct -gt $full_pct ] &&
|
|
[ $used_swap_pct -gt $last_swap_pct ]
|
|
then
|
|
echo "WARNING: RAM full and swap is growing. Restarting LLMs."
|
|
restart_llms
|
|
continue
|
|
fi
|
|
|
|
# If both are high then restart LLM services.
|
|
if [ $used_ram_pct -ge $high_pct ] &&
|
|
[ $used_swap_pct -ge $high_pct ]
|
|
then
|
|
echo "WARNING: RAM and swap both high. Restarting LLMs."
|
|
restart_llms
|
|
continue
|
|
fi
|
|
|
|
# Finish #
|
|
|
|
echo -e "Check finished, sleeping for ${sleep}...\n"
|
|
|
|
last_ram_pct=$used_ram_pct
|
|
last_swap_pct=$used_swap_pct
|
|
|
|
sleep $sleep
|
|
done
|