diff --git a/rc_shared.sh b/rc_shared.sh index 12d3ef2..01de8e1 100644 --- a/rc_shared.sh +++ b/rc_shared.sh @@ -1566,10 +1566,21 @@ alias ollama-cpu="ollama_threads " function ollama_benchmark_usage() { cat <<- EOF - USAGE: ollama-benchmark CONTEXT [THREADS] [iGPU] [MODEL] [-h | --help] + USAGE: ollama-benchmark CONTEXTS [THREADS] [IGPUS] [MODELS] [-h | --help] Tests installed models at a certain context, thread count, and iGPU setting. + PARAMETERS + + CONTEXTS + Sent as num_ctx to ollama. No default. + THREADS + Sent as num_threads to ollama. Defaults to 0. + IGPUS + Values to set OLLAMA_IGPU_ENABLE to. Defaults to 1. + MODELS + Model names to test. Defaults to all of `ollama ls`. + EXAMPLES: Test models with at a context of 32000, default threads, and iGPU enabled. @@ -1580,10 +1591,15 @@ function ollama_benchmark_usage() { Variables can also be specified through variables: - context=64000 threads=7 igpu=0 ollama-benchmark - context=64000 threads=7 ollama-benchmark - context=64000 igpu=0 ollama-benchmark - context=64000 model=qwen3.5:4b ollama-benchmark + context=64000 threads=7 igpu=0 ollama-benchmark + context=64000 threads=7 ollama-benchmark + context=64000 igpu=0 ollama-benchmark + context=64000 model=qwen3.5:4b ollama-benchmark + + The parameters can also act as arrays to cause extra looping. + + ollama-benchmark "32000 64000 128000 256000" "4 7 0" "0 1" \ + "qwen3.8:27b qwen3.6:35b qwen3.6:27b qwen3.5:35b qwen3.5:27b" Aliases: @@ -1599,50 +1615,40 @@ function ollama_benchmark() { return 0 fi - typeset -i context - if [[ -z $context || $context == 0 ]]; then - context="$1" - fi - if [[ -z "$context" ]]; then + contexts="${1:-$contexts}" + if [[ -z "$contexts" ]]; then echo " ERROR: Context is required." ollama_benchmark_usage return 1 fi - typeset -i threads - if [[ -z $threads ]]; then - threads="$2" - fi + threads="${2:-$threads}" if [[ -z $threads ]]; then threads=0 fi - declare igpu - if [[ -z "$igpu" ]] ; then - igpu="$3" - fi - if [[ -z "$igpu" ]] ; then - igpu=1 - fi - export OLLAMA_IGPU_ENABLE=$igpu - export OLLAMA_VULKAN=$igpu - - declare model - if [[ -z "$model" ]] ; then - model="$4" + igpus="${3:-$igpus}" + if [[ -z "$igpus" ]] ; then + igpus=1 fi - echo -e "\n Sharing run parameters..." - cat <<- EOF + models="${4:-$models}" + if [[ -z "$models" ]]; then + models="`ollama ls | grep -v NAME | awk '{print $1}'`" + fi + + DEBUG="${DEBUG}" + QUIET="${QUIET}" + [[ -z "$QUIET" ]] && echo -e "\n Sharing run parameters..." + [[ -z "$QUIET" ]] && cat <<- EOF DEBUG=$DEBUG + QUIET=$QUIET context=$context threads=$threads igpu=$igpu - OLLAMA_IGPU_ENABLE=$OLLAMA_IGPU_ENABLE - OLLAMA_VULKAN=$OLLAMA_IGPU_ENABLE model=$model EOF - echo " Done!" + [[ -z "$QUIET" ]] && echo " Done!" # Variables @@ -1651,11 +1657,13 @@ function ollama_benchmark() { col_ctx=8 col_ram=6 col_thr=7 + col_gpu=4 col_cpu=9 col_spd=11 col_spd_round=2 col_tim=8 log_file="ollama_benchmark.log.txt" + [[ -z "$QUIET" ]] && echo -e "\n Logging ollama to '$log_file'..." declare champ_{speed,time}_{model,num} @@ -1670,141 +1678,134 @@ function ollama_benchmark() { # Main if [[ ! -d /sdcard && -d /etc/sv ]]; then - echo -e "\n Stopping ollama runit service..." - sudo sv disable ollama - sudo sv stop ollama + [[ -z "$QUIET" ]] && echo -e "\n Stopping ollama runit service..." + sudo sv disable ollama >/dev/null 2>&1 + sudo sv stop ollama >/dev/null 2>&1 sleep 5 - echo " Done!" + [[ -z "$QUIET" ]] && echo " Done!" fi - echo -e "\n Stopping any ollama processes..." - killall ollama - sleep 5 - echo " Done!" + [[ -z "$QUIET" ]] && echo -e "\n Starting main loop... \n" - echo -e "\n Starting background ollama service, logging to '$log_file'..." - (ollama serve >$log_file 2>&1 &) - wait_for_ollama - echo " Done!" + printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \ + "Model" "HDD" "Context" "RAM" "Threads" "iGPU" "Processor" "Speed (t/s)" "Time (s)" - echo -e "\n Starting main loop... \n" + printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \ + ":------------------------" "-----:" "-------:" "-----:" "---:" "------:" "--------:" "----------:" "-------:" - printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \ - "Model @ CTX=$context" "HDD" "Context" "RAM" "Threads" "Processor" "Speed (t/s)" "Time (s)" + for model in $models; do + for context in $contexts; do + for thread in $threads; do + for igpu in $igpus; do + (killall ollama >/dev/null 2>&1) + sleep 5 + export OLLAMA_IGPU_ENABLE=$igpu + export OLLAMA_VULKAN=$igpu + (ollama serve >$log_file 2>&1 &) + wait_for_ollama - printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \ - ":------------------------" "-----:" "-------:" "-----:" "------:" "--------:" "----------:" "-------:" + hdd="`ollama ls | grep $model | awk '{print $3 $4}'`" + printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd" - model_focus="$model" - unset model - for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do - # If a model was specified then only focus on it. - if [[ -n "$model_focus" ]]; then - if [[ "$model" != "$model_focus" ]]; then - continue; - fi - fi + # Start the model and store the response. + start=$SECONDS + speed_json=$(curl -s http://localhost:11434/api/generate \ + -d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$thread'}}' \ + 2>/dev/null + ) + status_ollama=$? + finish=$SECONDS - hdd="`ollama ls | grep $model | awk '{print $3 $4}'`" + [[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama" - printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd" + time=$(( $finish - $start )) - # Start the model and store the response. - start=$SECONDS - speed_json=$(curl -s http://localhost:11434/api/generate \ - -d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$threads'}}' \ - 2>/dev/null - ) - status_ollama=$? - finish=$SECONDS + # Calculate the values if we got a successful response. + declare ram cpu ctx speed status_jq=0 + if [[ "$status_ollama" == 0 ]]; then + speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null) + status_jq=$? + [[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq" + if [[ "$status_jq" != 0 ]]; then + speed=0 + fi + else + [[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)" + ctx="OLLAMA" + ram="DIED" + cpu="FAIL" + speed=0 + killall ollama + (ollama serve >>$log_file 2>&1 &) + wait_for_ollama + fi - [[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama" + if [[ "$status_jq" != 0 ]]; then + [[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)" + ctx="RESPONSE" + ram="MLFRMD" + cpu="FAIL" + speed=0 + fi - time=$(( $finish - $start )) + if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then + [[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values." + ram="`ollama ps | grep $model | awk '{print $3 $4}'`" + cpu="`ollama ps | grep $model | awk '{print $5 $6}'`" + ctx="`ollama ps | grep $model | awk '{print $7}'`" - # Calculate the values if we got a successful response. - declare ram cpu ctx speed status_jq=0 - if [[ "$status_ollama" == 0 ]]; then - speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null) - status_jq=$? - [[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq" - if [[ "$status_jq" != 0 ]]; then - speed=0 - fi - else - [[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)" - ctx="OLLAMA" - ram="DIED" - cpu="FAIL" - speed=0 - killall ollama - (ollama serve >>$log_file 2>&1 &) - wait_for_ollama - fi + [[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed" + if [[ -z "$champ_speed_num" ]] \ + || awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }' + then + [[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!" + champ_speed_num="$speed" + champ_speed_model="$model" + fi - if [[ "$status_jq" != 0 ]]; then - [[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)" - ctx="RESPONSE" - ram="MLFRMD" - cpu="FAIL" - speed=0 - fi + [[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time" + if [[ -z "$champ_time_num" ]] \ + || awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a < b) }' + then + [[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!" + champ_time_num="$time" + champ_time_model="$model" + fi + fi - if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then - [[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values." - ram="`ollama ps | grep $model | awk '{print $3 $4}'`" - cpu="`ollama ps | grep $model | awk '{print $5 $6}'`" - ctx="`ollama ps | grep $model | awk '{print $7}'`" + printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \ + "$ctx" "$ram" "$thread" "$igpu" "$cpu" "$speed" "$time" - [[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed" - if [[ -z "$champ_speed_num" ]] \ - || awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }' - then - [[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!" - champ_speed_num="$speed" - champ_speed_model="$model" - fi - - [[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time" - if [[ -z "$champ_time_num" ]] \ - || awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a < b) }' - then - [[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!" - champ_time_num="$time" - champ_time_model="$model" - fi - fi - - printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \ - "$ctx" "$ram" "$threads" "$cpu" "$speed" "$time" - - ollama stop $model >/dev/null 2>&1 - while ollama ps | grep "$model"; do - [[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..." - sleep 1 + ollama stop $model >/dev/null 2>&1 + while ollama ps | grep "$model"; do + [[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..." + sleep 1 + done + done + done done done - echo -e "\n Done!! :D \n" - cat <<- EOF + [[ -z "$QUIET" ]] && echo -e "\n Done!! :D \n" + [[ -z "$QUIET" ]] && cat <<- EOF Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}! Time Champion: $champ_time_model with a time of ${champ_time_num}! EOF # Finish - echo -e "\n Killing local ollama serve..." + [[ -z "$QUIET" ]] && echo -e "\n Killing local ollama serve..." killall ollama - echo " Done!" + [[ -z "$QUIET" ]] && echo " Done!" if [[ ! -d /sdcard && -d /etc/sv ]]; then - echo -e "\n Restarting runit service..." + [[ -z "$QUIET" ]] && echo -e "\n Restarting runit service..." sudo sv enable ollama sudo sv start ollama - echo " Done!" + [[ -z "$QUIET" ]] && echo " Done!" fi - echo -e "\n Done!!! :)" + [[ -z "$QUIET" ]] && echo -e "\n Done!!! :)" } alias ollama-benchmark="ollama_benchmark " alias ollama-test="ollama_benchmark "