generated from me/template-mit
ollama_benchmark, add a much more capable function for testing ollama models.
This commit is contained in:
me committed
1 parent
9a85425c3f
commit
a47d1ab9f8
1 file changed
+249
-2
+249
-2
@@ -1475,7 +1475,7 @@ function ollama_sizes() {
|
||||
for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
|
||||
#ollama run $model "" --format json >/dev/null 2>&1
|
||||
curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":1'$context'}' \
|
||||
-d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":"1m"'$context'}' \
|
||||
>/dev/null 2>&1
|
||||
status=$?
|
||||
|
||||
@@ -1501,6 +1501,8 @@ function ollama_sizes() {
|
||||
}
|
||||
alias ollama-sizes="ollama_sizes "
|
||||
alias model-sizes="ollama_sizes "
|
||||
alias model-memory="ollama_sizes "
|
||||
alias model-mem="ollama_sizes "
|
||||
|
||||
function ollama_threads() {
|
||||
# VAR1 - Model
|
||||
@@ -1545,7 +1547,7 @@ function ollama_threads() {
|
||||
echo -n "`date` - $model w/ $t Thread(s): "
|
||||
|
||||
speed=$(curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_thread":'$t$context'}}' \
|
||||
| jq '.eval_count * 1000000000 / .eval_duration'
|
||||
)
|
||||
finish=$SECONDS
|
||||
@@ -1562,6 +1564,251 @@ alias ollama-threads="ollama_threads "
|
||||
alias ollama-cpus="ollama_threads "
|
||||
alias ollama-cpu="ollama_threads "
|
||||
|
||||
function ollama_benchmark_usage() {
|
||||
cat <<- EOF
|
||||
USAGE: ollama-benchmark CONTEXT [THREADS] [iGPU] [MODEL] [-h | --help]
|
||||
|
||||
Tests installed models at a certain context, thread count, and iGPU setting.
|
||||
|
||||
EXAMPLES:
|
||||
|
||||
Test models with at a context of 32000, default threads, and iGPU enabled.
|
||||
ollama-benchmark 32000 0 1
|
||||
|
||||
Test qwen3.5:4b with at a context of 128k, 4 threads, and iGPU disabled.
|
||||
ollama-benchmark 128000 4 0 qwen3.5:4b
|
||||
|
||||
Variables can also be specified through variables:
|
||||
|
||||
context=64000 threads=7 igpu=0 ollama-benchmark
|
||||
context=64000 threads=7 ollama-benchmark
|
||||
context=64000 igpu=0 ollama-benchmark
|
||||
context=64000 model=qwen3.5:4b ollama-benchmark
|
||||
|
||||
Aliases:
|
||||
|
||||
ollama-test
|
||||
EOF
|
||||
}
|
||||
|
||||
function ollama_benchmark() {
|
||||
# Parameters
|
||||
|
||||
if [[ "$@" == *"-h"* || "$@" == *"--help"* ]]; then
|
||||
ollama_benchmark_usage
|
||||
return 0
|
||||
fi
|
||||
|
||||
typeset -i context
|
||||
if [[ -z $context || $context == 0 ]]; then
|
||||
context="$1"
|
||||
fi
|
||||
if [[ -z "$context" ]]; then
|
||||
echo " ERROR: Context is required."
|
||||
ollama_benchmark_usage
|
||||
return 1
|
||||
fi
|
||||
|
||||
typeset -i threads
|
||||
if [[ -z $threads ]]; then
|
||||
threads="$2"
|
||||
fi
|
||||
if [[ -z $threads ]]; then
|
||||
threads=0
|
||||
fi
|
||||
|
||||
declare igpu
|
||||
if [[ -z "$igpu" ]] ; then
|
||||
igpu="$3"
|
||||
fi
|
||||
if [[ -z "$igpu" ]] ; then
|
||||
igpu=1
|
||||
fi
|
||||
export OLLAMA_IGPU_ENABLE=$igpu
|
||||
export OLLAMA_VULKAN=$igpu
|
||||
|
||||
declare model
|
||||
if [[ -z "$model" ]] ; then
|
||||
model="$4"
|
||||
fi
|
||||
|
||||
echo -e "\n Sharing run parameters..."
|
||||
cat <<- EOF
|
||||
DEBUG=$DEBUG
|
||||
context=$context
|
||||
threads=$threads
|
||||
igpu=$igpu
|
||||
OLLAMA_IGPU_ENABLE=$OLLAMA_IGPU_ENABLE
|
||||
OLLAMA_VULKAN=$OLLAMA_IGPU_ENABLE
|
||||
model=$model
|
||||
EOF
|
||||
echo " Done!"
|
||||
|
||||
# Variables
|
||||
|
||||
col_mdl=25
|
||||
col_hdd=6
|
||||
col_ctx=8
|
||||
col_ram=6
|
||||
col_thr=7
|
||||
col_cpu=9
|
||||
col_spd=11 col_spd_round=2
|
||||
col_tim=8
|
||||
|
||||
log_file="ollama_benchmark.log.txt"
|
||||
|
||||
declare champ_{speed,time}_{model,num}
|
||||
|
||||
# Functions
|
||||
|
||||
function wait_for_ollama() {
|
||||
while ! ollama ps >/dev/null 2>&1; do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Main
|
||||
|
||||
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
||||
echo -e "\n Stopping ollama runit service..."
|
||||
sudo sv disable ollama
|
||||
sudo sv stop ollama
|
||||
sleep 5
|
||||
echo " Done!"
|
||||
fi
|
||||
|
||||
echo -e "\n Stopping any ollama processes..."
|
||||
killall ollama
|
||||
sleep 5
|
||||
echo " Done!"
|
||||
|
||||
echo -e "\n Starting background ollama service, logging to '$log_file'..."
|
||||
(ollama serve >$log_file 2>&1 &)
|
||||
wait_for_ollama
|
||||
echo " Done!"
|
||||
|
||||
echo -e "\n Starting main loop... \n"
|
||||
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||
"Model @ CTX=$context" "HDD" "Context" "RAM" "Threads" "Processor" "Speed (t/s)" "Time (s)"
|
||||
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||
":------------------------" "-----:" "-------:" "-----:" "------:" "--------:" "----------:" "-------:"
|
||||
|
||||
model_focus="$model"
|
||||
unset model
|
||||
for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
|
||||
# If a model was specified then only focus on it.
|
||||
if [[ -n "$model_focus" ]]; then
|
||||
if [[ "$model" != "$model_focus" ]]; then
|
||||
continue;
|
||||
fi
|
||||
fi
|
||||
|
||||
hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
|
||||
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd"
|
||||
|
||||
# Start the model and store the response.
|
||||
start=$SECONDS
|
||||
speed_json=$(curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$threads'}}' \
|
||||
2>/dev/null
|
||||
)
|
||||
status_ollama=$?
|
||||
finish=$SECONDS
|
||||
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama"
|
||||
|
||||
time=$(( $finish - $start ))
|
||||
|
||||
# Calculate the values if we got a successful response.
|
||||
declare ram cpu ctx speed status_jq=0
|
||||
if [[ "$status_ollama" == 0 ]]; then
|
||||
speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null)
|
||||
status_jq=$?
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq"
|
||||
if [[ "$status_jq" != 0 ]]; then
|
||||
speed=0
|
||||
fi
|
||||
else
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)"
|
||||
ctx="OLLAMA"
|
||||
ram="DIED"
|
||||
cpu="FAIL"
|
||||
speed=0
|
||||
killall ollama
|
||||
(ollama serve >>$log_file 2>&1 &)
|
||||
wait_for_ollama
|
||||
fi
|
||||
|
||||
if [[ "$status_jq" != 0 ]]; then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)"
|
||||
ctx="RESPONSE"
|
||||
ram="MLFRMD"
|
||||
cpu="FAIL"
|
||||
speed=0
|
||||
fi
|
||||
|
||||
if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values."
|
||||
ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
|
||||
cpu="`ollama ps | grep $model | awk '{print $5 $6}'`"
|
||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed"
|
||||
if [[ -z "$champ_speed_num" ]] \
|
||||
|| awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }'
|
||||
then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!"
|
||||
champ_speed_num="$speed"
|
||||
champ_speed_model="$model"
|
||||
fi
|
||||
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time"
|
||||
if [[ -z "$champ_time_num" ]] \
|
||||
|| awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a > b) }'
|
||||
then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!"
|
||||
champ_time_num="$time"
|
||||
champ_time_model="$model"
|
||||
fi
|
||||
fi
|
||||
|
||||
printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \
|
||||
"$ctx" "$ram" "$threads" "$cpu" "$speed" "$time"
|
||||
|
||||
ollama stop $model >/dev/null 2>&1
|
||||
while ollama ps | grep "$model"; do
|
||||
[[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..."
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
|
||||
echo -e "\n Done!! :D \n"
|
||||
cat <<- EOF
|
||||
Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}!
|
||||
Time Champion: $champ_time_model with a time of ${champ_time_num}!
|
||||
EOF
|
||||
|
||||
# Finish
|
||||
|
||||
echo -e "\n Killing local ollama serve..."
|
||||
killall ollama
|
||||
echo " Done!"
|
||||
|
||||
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
||||
echo -e "\n Restarting runit service..."
|
||||
sudo sv enable ollama
|
||||
sudo sv start ollama
|
||||
echo " Done!"
|
||||
fi
|
||||
|
||||
echo -e "\n Done!!! :)"
|
||||
}
|
||||
alias ollama-benchmark="ollama_benchmark "
|
||||
alias ollama-test="ollama_benchmark "
|
||||
|
||||
### Finalize ###
|
||||
|
||||
# Export all functions! #
|
||||
|
||||
Reference in new issue
Block a user