llm, add elapsed time to ollama_threads.
This commit is contained in:
1 parent
7f7f2a2edb
commit
c604859947
1 file changed
+5
-2
@@ -927,15 +927,18 @@
|
|||||||
|
|
||||||
# Benchmark Thread Count
|
# Benchmark Thread Count
|
||||||
for t in {1..{{ ansible_processor_vcpus | int }}}; do
|
for t in {1..{{ ansible_processor_vcpus | int }}}; do
|
||||||
echo -n "`date` - Model $model w/ $t Thread(s), tokens/sec: "
|
start=$SECONDS
|
||||||
|
echo -n "`date` - Model $model w/ $t Thread(s): "
|
||||||
|
|
||||||
speed=$(curl -s http://localhost:11434/api/generate \
|
speed=$(curl -s http://localhost:11434/api/generate \
|
||||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \
|
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \
|
||||||
| jq '.eval_count * 1000000000 / .eval_duration'
|
| jq '.eval_count * 1000000000 / .eval_duration'
|
||||||
)
|
)
|
||||||
|
finish=$SECONDS
|
||||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||||
|
time=$(( $finish - $start ))
|
||||||
|
|
||||||
echo "$speed @ context=$ctx"
|
echo ", tokens/sec: $speed @ context=$ctx, time: $time secs"
|
||||||
|
|
||||||
ollama stop $model >/dev/null 2>&1
|
ollama stop $model >/dev/null 2>&1
|
||||||
done
|
done
|
||||||
|
|||||||
Reference in new issue
Block a user