diff --git a/rc_shared.sh b/rc_shared.sh index 99d4e28..0157a65 100644 --- a/rc_shared.sh +++ b/rc_shared.sh @@ -1475,7 +1475,7 @@ function ollama_sizes() { for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do #ollama run $model "" --format json >/dev/null 2>&1 curl -s http://localhost:11434/api/generate \ - -d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":1'$context'}' \ + -d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":"1m"'$context'}' \ >/dev/null 2>&1 status=$? @@ -1501,6 +1501,8 @@ function ollama_sizes() { } alias ollama-sizes="ollama_sizes " alias model-sizes="ollama_sizes " +alias model-memory="ollama_sizes " +alias model-mem="ollama_sizes " function ollama_threads() { # VAR1 - Model @@ -1545,7 +1547,7 @@ function ollama_threads() { echo -n "`date` - $model w/ $t Thread(s): " speed=$(curl -s http://localhost:11434/api/generate \ - -d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \ + -d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_thread":'$t$context'}}' \ | jq '.eval_count * 1000000000 / .eval_duration' ) finish=$SECONDS @@ -1562,6 +1564,251 @@ alias ollama-threads="ollama_threads " alias ollama-cpus="ollama_threads " alias ollama-cpu="ollama_threads " +function ollama_benchmark_usage() { + cat <<- EOF + USAGE: ollama-benchmark CONTEXT [THREADS] [iGPU] [MODEL] [-h | --help] + + Tests installed models at a certain context, thread count, and iGPU setting. + + EXAMPLES: + + Test models with at a context of 32000, default threads, and iGPU enabled. + ollama-benchmark 32000 0 1 + + Test qwen3.5:4b with at a context of 128k, 4 threads, and iGPU disabled. + ollama-benchmark 128000 4 0 qwen3.5:4b + + Variables can also be specified through variables: + + context=64000 threads=7 igpu=0 ollama-benchmark + context=64000 threads=7 ollama-benchmark + context=64000 igpu=0 ollama-benchmark + context=64000 model=qwen3.5:4b ollama-benchmark + + Aliases: + + ollama-test + EOF +} + +function ollama_benchmark() { + # Parameters + + if [[ "$@" == *"-h"* || "$@" == *"--help"* ]]; then + ollama_benchmark_usage + return 0 + fi + + typeset -i context + if [[ -z $context || $context == 0 ]]; then + context="$1" + fi + if [[ -z "$context" ]]; then + echo " ERROR: Context is required." + ollama_benchmark_usage + return 1 + fi + + typeset -i threads + if [[ -z $threads ]]; then + threads="$2" + fi + if [[ -z $threads ]]; then + threads=0 + fi + + declare igpu + if [[ -z "$igpu" ]] ; then + igpu="$3" + fi + if [[ -z "$igpu" ]] ; then + igpu=1 + fi + export OLLAMA_IGPU_ENABLE=$igpu + export OLLAMA_VULKAN=$igpu + + declare model + if [[ -z "$model" ]] ; then + model="$4" + fi + + echo -e "\n Sharing run parameters..." + cat <<- EOF + DEBUG=$DEBUG + context=$context + threads=$threads + igpu=$igpu + OLLAMA_IGPU_ENABLE=$OLLAMA_IGPU_ENABLE + OLLAMA_VULKAN=$OLLAMA_IGPU_ENABLE + model=$model + EOF + echo " Done!" + + # Variables + + col_mdl=25 + col_hdd=6 + col_ctx=8 + col_ram=6 + col_thr=7 + col_cpu=9 + col_spd=11 col_spd_round=2 + col_tim=8 + + log_file="ollama_benchmark.log.txt" + + declare champ_{speed,time}_{model,num} + + # Functions + + function wait_for_ollama() { + while ! ollama ps >/dev/null 2>&1; do + sleep 1 + done + } + + # Main + + if [[ ! -d /sdcard && -d /etc/sv ]]; then + echo -e "\n Stopping ollama runit service..." + sudo sv disable ollama + sudo sv stop ollama + sleep 5 + echo " Done!" + fi + + echo -e "\n Stopping any ollama processes..." + killall ollama + sleep 5 + echo " Done!" + + echo -e "\n Starting background ollama service, logging to '$log_file'..." + (ollama serve >$log_file 2>&1 &) + wait_for_ollama + echo " Done!" + + echo -e "\n Starting main loop... \n" + + printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \ + "Model @ CTX=$context" "HDD" "Context" "RAM" "Threads" "Processor" "Speed (t/s)" "Time (s)" + + printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \ + ":------------------------" "-----:" "-------:" "-----:" "------:" "--------:" "----------:" "-------:" + + model_focus="$model" + unset model + for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do + # If a model was specified then only focus on it. + if [[ -n "$model_focus" ]]; then + if [[ "$model" != "$model_focus" ]]; then + continue; + fi + fi + + hdd="`ollama ls | grep $model | awk '{print $3 $4}'`" + + printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd" + + # Start the model and store the response. + start=$SECONDS + speed_json=$(curl -s http://localhost:11434/api/generate \ + -d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$threads'}}' \ + 2>/dev/null + ) + status_ollama=$? + finish=$SECONDS + + [[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama" + + time=$(( $finish - $start )) + + # Calculate the values if we got a successful response. + declare ram cpu ctx speed status_jq=0 + if [[ "$status_ollama" == 0 ]]; then + speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null) + status_jq=$? + [[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq" + if [[ "$status_jq" != 0 ]]; then + speed=0 + fi + else + [[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)" + ctx="OLLAMA" + ram="DIED" + cpu="FAIL" + speed=0 + killall ollama + (ollama serve >>$log_file 2>&1 &) + wait_for_ollama + fi + + if [[ "$status_jq" != 0 ]]; then + [[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)" + ctx="RESPONSE" + ram="MLFRMD" + cpu="FAIL" + speed=0 + fi + + if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then + [[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values." + ram="`ollama ps | grep $model | awk '{print $3 $4}'`" + cpu="`ollama ps | grep $model | awk '{print $5 $6}'`" + ctx="`ollama ps | grep $model | awk '{print $7}'`" + + [[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed" + if [[ -z "$champ_speed_num" ]] \ + || awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }' + then + [[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!" + champ_speed_num="$speed" + champ_speed_model="$model" + fi + + [[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time" + if [[ -z "$champ_time_num" ]] \ + || awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a > b) }' + then + [[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!" + champ_time_num="$time" + champ_time_model="$model" + fi + fi + + printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \ + "$ctx" "$ram" "$threads" "$cpu" "$speed" "$time" + + ollama stop $model >/dev/null 2>&1 + while ollama ps | grep "$model"; do + [[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..." + sleep 1 + done + done + + echo -e "\n Done!! :D \n" + cat <<- EOF + Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}! + Time Champion: $champ_time_model with a time of ${champ_time_num}! + EOF + + # Finish + + echo -e "\n Killing local ollama serve..." + killall ollama + echo " Done!" + + if [[ ! -d /sdcard && -d /etc/sv ]]; then + echo -e "\n Restarting runit service..." + sudo sv enable ollama + sudo sv start ollama + echo " Done!" + fi + + echo -e "\n Done!!! :)" +} +alias ollama-benchmark="ollama_benchmark " +alias ollama-test="ollama_benchmark " + ### Finalize ### # Export all functions! #