generated from me/template-mit
ollama-test, enhance to me like pv-test with the parameter looping, also fix some variables being set via env and add QUIET to suppress all but the table.
This commit is contained in:
me committed
1 parent
933e00556c
commit
f3da185fb6
1 file changed
+135
-134
+135
-134
@@ -1566,10 +1566,21 @@ alias ollama-cpu="ollama_threads "
|
|||||||
|
|
||||||
function ollama_benchmark_usage() {
|
function ollama_benchmark_usage() {
|
||||||
cat <<- EOF
|
cat <<- EOF
|
||||||
USAGE: ollama-benchmark CONTEXT [THREADS] [iGPU] [MODEL] [-h | --help]
|
USAGE: ollama-benchmark CONTEXTS [THREADS] [IGPUS] [MODELS] [-h | --help]
|
||||||
|
|
||||||
Tests installed models at a certain context, thread count, and iGPU setting.
|
Tests installed models at a certain context, thread count, and iGPU setting.
|
||||||
|
|
||||||
|
PARAMETERS
|
||||||
|
|
||||||
|
CONTEXTS
|
||||||
|
Sent as num_ctx to ollama. No default.
|
||||||
|
THREADS
|
||||||
|
Sent as num_threads to ollama. Defaults to 0.
|
||||||
|
IGPUS
|
||||||
|
Values to set OLLAMA_IGPU_ENABLE to. Defaults to 1.
|
||||||
|
MODELS
|
||||||
|
Model names to test. Defaults to all of `ollama ls`.
|
||||||
|
|
||||||
EXAMPLES:
|
EXAMPLES:
|
||||||
|
|
||||||
Test models with at a context of 32000, default threads, and iGPU enabled.
|
Test models with at a context of 32000, default threads, and iGPU enabled.
|
||||||
@@ -1580,10 +1591,15 @@ function ollama_benchmark_usage() {
|
|||||||
|
|
||||||
Variables can also be specified through variables:
|
Variables can also be specified through variables:
|
||||||
|
|
||||||
context=64000 threads=7 igpu=0 ollama-benchmark
|
context=64000 threads=7 igpu=0 ollama-benchmark
|
||||||
context=64000 threads=7 ollama-benchmark
|
context=64000 threads=7 ollama-benchmark
|
||||||
context=64000 igpu=0 ollama-benchmark
|
context=64000 igpu=0 ollama-benchmark
|
||||||
context=64000 model=qwen3.5:4b ollama-benchmark
|
context=64000 model=qwen3.5:4b ollama-benchmark
|
||||||
|
|
||||||
|
The parameters can also act as arrays to cause extra looping.
|
||||||
|
|
||||||
|
ollama-benchmark "32000 64000 128000 256000" "4 7 0" "0 1" \
|
||||||
|
"qwen3.8:27b qwen3.6:35b qwen3.6:27b qwen3.5:35b qwen3.5:27b"
|
||||||
|
|
||||||
Aliases:
|
Aliases:
|
||||||
|
|
||||||
@@ -1599,50 +1615,40 @@ function ollama_benchmark() {
|
|||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
typeset -i context
|
contexts="${1:-$contexts}"
|
||||||
if [[ -z $context || $context == 0 ]]; then
|
if [[ -z "$contexts" ]]; then
|
||||||
context="$1"
|
|
||||||
fi
|
|
||||||
if [[ -z "$context" ]]; then
|
|
||||||
echo " ERROR: Context is required."
|
echo " ERROR: Context is required."
|
||||||
ollama_benchmark_usage
|
ollama_benchmark_usage
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
typeset -i threads
|
threads="${2:-$threads}"
|
||||||
if [[ -z $threads ]]; then
|
|
||||||
threads="$2"
|
|
||||||
fi
|
|
||||||
if [[ -z $threads ]]; then
|
if [[ -z $threads ]]; then
|
||||||
threads=0
|
threads=0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
declare igpu
|
igpus="${3:-$igpus}"
|
||||||
if [[ -z "$igpu" ]] ; then
|
if [[ -z "$igpus" ]] ; then
|
||||||
igpu="$3"
|
igpus=1
|
||||||
fi
|
|
||||||
if [[ -z "$igpu" ]] ; then
|
|
||||||
igpu=1
|
|
||||||
fi
|
|
||||||
export OLLAMA_IGPU_ENABLE=$igpu
|
|
||||||
export OLLAMA_VULKAN=$igpu
|
|
||||||
|
|
||||||
declare model
|
|
||||||
if [[ -z "$model" ]] ; then
|
|
||||||
model="$4"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo -e "\n Sharing run parameters..."
|
models="${4:-$models}"
|
||||||
cat <<- EOF
|
if [[ -z "$models" ]]; then
|
||||||
|
models="`ollama ls | grep -v NAME | awk '{print $1}'`"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEBUG="${DEBUG}"
|
||||||
|
QUIET="${QUIET}"
|
||||||
|
[[ -z "$QUIET" ]] && echo -e "\n Sharing run parameters..."
|
||||||
|
[[ -z "$QUIET" ]] && cat <<- EOF
|
||||||
DEBUG=$DEBUG
|
DEBUG=$DEBUG
|
||||||
|
QUIET=$QUIET
|
||||||
context=$context
|
context=$context
|
||||||
threads=$threads
|
threads=$threads
|
||||||
igpu=$igpu
|
igpu=$igpu
|
||||||
OLLAMA_IGPU_ENABLE=$OLLAMA_IGPU_ENABLE
|
|
||||||
OLLAMA_VULKAN=$OLLAMA_IGPU_ENABLE
|
|
||||||
model=$model
|
model=$model
|
||||||
EOF
|
EOF
|
||||||
echo " Done!"
|
[[ -z "$QUIET" ]] && echo " Done!"
|
||||||
|
|
||||||
# Variables
|
# Variables
|
||||||
|
|
||||||
@@ -1651,11 +1657,13 @@ function ollama_benchmark() {
|
|||||||
col_ctx=8
|
col_ctx=8
|
||||||
col_ram=6
|
col_ram=6
|
||||||
col_thr=7
|
col_thr=7
|
||||||
|
col_gpu=4
|
||||||
col_cpu=9
|
col_cpu=9
|
||||||
col_spd=11 col_spd_round=2
|
col_spd=11 col_spd_round=2
|
||||||
col_tim=8
|
col_tim=8
|
||||||
|
|
||||||
log_file="ollama_benchmark.log.txt"
|
log_file="ollama_benchmark.log.txt"
|
||||||
|
[[ -z "$QUIET" ]] && echo -e "\n Logging ollama to '$log_file'..."
|
||||||
|
|
||||||
declare champ_{speed,time}_{model,num}
|
declare champ_{speed,time}_{model,num}
|
||||||
|
|
||||||
@@ -1670,141 +1678,134 @@ function ollama_benchmark() {
|
|||||||
# Main
|
# Main
|
||||||
|
|
||||||
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
||||||
echo -e "\n Stopping ollama runit service..."
|
[[ -z "$QUIET" ]] && echo -e "\n Stopping ollama runit service..."
|
||||||
sudo sv disable ollama
|
sudo sv disable ollama >/dev/null 2>&1
|
||||||
sudo sv stop ollama
|
sudo sv stop ollama >/dev/null 2>&1
|
||||||
sleep 5
|
sleep 5
|
||||||
echo " Done!"
|
[[ -z "$QUIET" ]] && echo " Done!"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo -e "\n Stopping any ollama processes..."
|
[[ -z "$QUIET" ]] && echo -e "\n Starting main loop... \n"
|
||||||
killall ollama
|
|
||||||
sleep 5
|
|
||||||
echo " Done!"
|
|
||||||
|
|
||||||
echo -e "\n Starting background ollama service, logging to '$log_file'..."
|
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||||
(ollama serve >$log_file 2>&1 &)
|
"Model" "HDD" "Context" "RAM" "Threads" "iGPU" "Processor" "Speed (t/s)" "Time (s)"
|
||||||
wait_for_ollama
|
|
||||||
echo " Done!"
|
|
||||||
|
|
||||||
echo -e "\n Starting main loop... \n"
|
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||||
|
":------------------------" "-----:" "-------:" "-----:" "---:" "------:" "--------:" "----------:" "-------:"
|
||||||
|
|
||||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
for model in $models; do
|
||||||
"Model @ CTX=$context" "HDD" "Context" "RAM" "Threads" "Processor" "Speed (t/s)" "Time (s)"
|
for context in $contexts; do
|
||||||
|
for thread in $threads; do
|
||||||
|
for igpu in $igpus; do
|
||||||
|
(killall ollama >/dev/null 2>&1)
|
||||||
|
sleep 5
|
||||||
|
export OLLAMA_IGPU_ENABLE=$igpu
|
||||||
|
export OLLAMA_VULKAN=$igpu
|
||||||
|
(ollama serve >$log_file 2>&1 &)
|
||||||
|
wait_for_ollama
|
||||||
|
|
||||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
|
||||||
":------------------------" "-----:" "-------:" "-----:" "------:" "--------:" "----------:" "-------:"
|
printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd"
|
||||||
|
|
||||||
model_focus="$model"
|
# Start the model and store the response.
|
||||||
unset model
|
start=$SECONDS
|
||||||
for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
|
speed_json=$(curl -s http://localhost:11434/api/generate \
|
||||||
# If a model was specified then only focus on it.
|
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$thread'}}' \
|
||||||
if [[ -n "$model_focus" ]]; then
|
2>/dev/null
|
||||||
if [[ "$model" != "$model_focus" ]]; then
|
)
|
||||||
continue;
|
status_ollama=$?
|
||||||
fi
|
finish=$SECONDS
|
||||||
fi
|
|
||||||
|
|
||||||
hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
|
[[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama"
|
||||||
|
|
||||||
printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd"
|
time=$(( $finish - $start ))
|
||||||
|
|
||||||
# Start the model and store the response.
|
# Calculate the values if we got a successful response.
|
||||||
start=$SECONDS
|
declare ram cpu ctx speed status_jq=0
|
||||||
speed_json=$(curl -s http://localhost:11434/api/generate \
|
if [[ "$status_ollama" == 0 ]]; then
|
||||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$threads'}}' \
|
speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null)
|
||||||
2>/dev/null
|
status_jq=$?
|
||||||
)
|
[[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq"
|
||||||
status_ollama=$?
|
if [[ "$status_jq" != 0 ]]; then
|
||||||
finish=$SECONDS
|
speed=0
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)"
|
||||||
|
ctx="OLLAMA"
|
||||||
|
ram="DIED"
|
||||||
|
cpu="FAIL"
|
||||||
|
speed=0
|
||||||
|
killall ollama
|
||||||
|
(ollama serve >>$log_file 2>&1 &)
|
||||||
|
wait_for_ollama
|
||||||
|
fi
|
||||||
|
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama"
|
if [[ "$status_jq" != 0 ]]; then
|
||||||
|
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)"
|
||||||
|
ctx="RESPONSE"
|
||||||
|
ram="MLFRMD"
|
||||||
|
cpu="FAIL"
|
||||||
|
speed=0
|
||||||
|
fi
|
||||||
|
|
||||||
time=$(( $finish - $start ))
|
if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then
|
||||||
|
[[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values."
|
||||||
|
ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
|
||||||
|
cpu="`ollama ps | grep $model | awk '{print $5 $6}'`"
|
||||||
|
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||||
|
|
||||||
# Calculate the values if we got a successful response.
|
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed"
|
||||||
declare ram cpu ctx speed status_jq=0
|
if [[ -z "$champ_speed_num" ]] \
|
||||||
if [[ "$status_ollama" == 0 ]]; then
|
|| awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }'
|
||||||
speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null)
|
then
|
||||||
status_jq=$?
|
[[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!"
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq"
|
champ_speed_num="$speed"
|
||||||
if [[ "$status_jq" != 0 ]]; then
|
champ_speed_model="$model"
|
||||||
speed=0
|
fi
|
||||||
fi
|
|
||||||
else
|
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)"
|
|
||||||
ctx="OLLAMA"
|
|
||||||
ram="DIED"
|
|
||||||
cpu="FAIL"
|
|
||||||
speed=0
|
|
||||||
killall ollama
|
|
||||||
(ollama serve >>$log_file 2>&1 &)
|
|
||||||
wait_for_ollama
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$status_jq" != 0 ]]; then
|
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time"
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)"
|
if [[ -z "$champ_time_num" ]] \
|
||||||
ctx="RESPONSE"
|
|| awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a < b) }'
|
||||||
ram="MLFRMD"
|
then
|
||||||
cpu="FAIL"
|
[[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!"
|
||||||
speed=0
|
champ_time_num="$time"
|
||||||
fi
|
champ_time_model="$model"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then
|
printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values."
|
"$ctx" "$ram" "$thread" "$igpu" "$cpu" "$speed" "$time"
|
||||||
ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
|
|
||||||
cpu="`ollama ps | grep $model | awk '{print $5 $6}'`"
|
|
||||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
|
||||||
|
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed"
|
ollama stop $model >/dev/null 2>&1
|
||||||
if [[ -z "$champ_speed_num" ]] \
|
while ollama ps | grep "$model"; do
|
||||||
|| awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }'
|
[[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..."
|
||||||
then
|
sleep 1
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!"
|
done
|
||||||
champ_speed_num="$speed"
|
done
|
||||||
champ_speed_model="$model"
|
done
|
||||||
fi
|
|
||||||
|
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time"
|
|
||||||
if [[ -z "$champ_time_num" ]] \
|
|
||||||
|| awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a < b) }'
|
|
||||||
then
|
|
||||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!"
|
|
||||||
champ_time_num="$time"
|
|
||||||
champ_time_model="$model"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \
|
|
||||||
"$ctx" "$ram" "$threads" "$cpu" "$speed" "$time"
|
|
||||||
|
|
||||||
ollama stop $model >/dev/null 2>&1
|
|
||||||
while ollama ps | grep "$model"; do
|
|
||||||
[[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..."
|
|
||||||
sleep 1
|
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
echo -e "\n Done!! :D \n"
|
[[ -z "$QUIET" ]] && echo -e "\n Done!! :D \n"
|
||||||
cat <<- EOF
|
[[ -z "$QUIET" ]] && cat <<- EOF
|
||||||
Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}!
|
Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}!
|
||||||
Time Champion: $champ_time_model with a time of ${champ_time_num}!
|
Time Champion: $champ_time_model with a time of ${champ_time_num}!
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# Finish
|
# Finish
|
||||||
|
|
||||||
echo -e "\n Killing local ollama serve..."
|
[[ -z "$QUIET" ]] && echo -e "\n Killing local ollama serve..."
|
||||||
killall ollama
|
killall ollama
|
||||||
echo " Done!"
|
[[ -z "$QUIET" ]] && echo " Done!"
|
||||||
|
|
||||||
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
||||||
echo -e "\n Restarting runit service..."
|
[[ -z "$QUIET" ]] && echo -e "\n Restarting runit service..."
|
||||||
sudo sv enable ollama
|
sudo sv enable ollama
|
||||||
sudo sv start ollama
|
sudo sv start ollama
|
||||||
echo " Done!"
|
[[ -z "$QUIET" ]] && echo " Done!"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo -e "\n Done!!! :)"
|
[[ -z "$QUIET" ]] && echo -e "\n Done!!! :)"
|
||||||
}
|
}
|
||||||
alias ollama-benchmark="ollama_benchmark "
|
alias ollama-benchmark="ollama_benchmark "
|
||||||
alias ollama-test="ollama_benchmark "
|
alias ollama-test="ollama_benchmark "
|
||||||
|
|||||||
Reference in new issue
Block a user