diff --git a/files/scripts/llm_swap_stopper.sh b/files/scripts/llm_swap_stopper.sh index dd302d8..588d4fb 100755 --- a/files/scripts/llm_swap_stopper.sh +++ b/files/scripts/llm_swap_stopper.sh @@ -4,7 +4,7 @@ ### Tuning ### -typeset -i sleep=10 ram_pct_restart=98 +typeset -i sleep=10 full_pct=95 high_pct=90 ### Functions ### @@ -48,15 +48,6 @@ while [[ true ]]; do echo "Current RAM usage is $used_ram_pct%, was $last_ram_pct% last loop." - # Check if RAM is over 95% - if [ $used_ram_pct -gt $ram_pct_restart ]; then - echo "WARNING: RAM is over ${ram_pct_restart}% used. Restarting LLMs." - restart_llms - echo -e "Done, setting last RAM to 100 and continuing.\n" - last_ram_pct=100 - continue - fi - # Swap # total_swap=$(free | awk '/^Swap:/ {print $2}') @@ -68,13 +59,20 @@ while [[ true ]]; do echo "Current swap usage is $used_swap_pct%, was $last_swap_pct% last loop." - # Check if swap is growing - if [ $used_swap_pct -gt $last_swap_pct ]; then - echo "WARNING: Swap is growing. Restarting LLMs." + # If RAM is full and swap is growing then restart LLM services. + if [ $used_ram_pct -gt $full_pct ] && + [ $used_swap_pct -gt $last_swap_pct ] + then + echo "WARNING: RAM full and swap is growing. Restarting LLMs." + restart_llms + fi + + # If both are high then restart LLM services. + if [ $used_ram_pct -ge $high_pct ] && + [ $used_swap_pct -ge $high_pct ] + then + echo "WARNING: RAM and swap both high. Restarting LLMs." restart_llms - echo -e "Done, setting last swap to 100 and continuing.\n" - last_swap_pct=100 - continue fi # Finish #