llm, move some functions to env-shared project.
This commit is contained in:
me committed
1 parent
41f306f7d7
commit
ef9566c76c
1 file changed
-156
@@ -905,18 +905,6 @@
|
||||
alias pull-models="ollama_pull "
|
||||
alias pull-ai="ollama_pull "
|
||||
|
||||
function ollama_stop() {
|
||||
for model in `ollama ps | tail -n +2 | awk '{print $1}'`; do
|
||||
echo "Stopping ${model}..."
|
||||
ollama stop $model
|
||||
done
|
||||
}
|
||||
alias ollama-stop="ollama_stop"
|
||||
alias ollama-kill="ollama_stop"
|
||||
alias ollama-killall="ollama_stop"
|
||||
alias ollama-kill-all="ollama_stop"
|
||||
alias ollama-stop-all="ollama_stop"
|
||||
|
||||
function ollama_chat() {
|
||||
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen3.5:4b' }}
|
||||
}
|
||||
@@ -927,21 +915,6 @@
|
||||
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen2.5-coder:7b' }}
|
||||
}
|
||||
alias ollama-code="ollama_code "
|
||||
alias code="ollama_code "
|
||||
|
||||
function ollama_models() {
|
||||
for model in `ollama ls | grep -v NAME | awk '{print $1;}'`; do
|
||||
echo $model
|
||||
ollama show $model | grep -E "architecture|context length|quantization"
|
||||
done
|
||||
}
|
||||
alias ollama-models="ollama_models "
|
||||
alias ollama-show-models="ollama_models "
|
||||
alias ollama-show="ollama_models "
|
||||
alias ollama-dense="ollama_models "
|
||||
alias ollama-moe="ollama_models "
|
||||
alias show-models="ollama_models "
|
||||
alias list-models="ollama_models "
|
||||
|
||||
function ollama_purge() {
|
||||
echo "TBD: Purge all models not in llm_installs."
|
||||
@@ -966,135 +939,6 @@
|
||||
alias restart-llms="restart_ai "
|
||||
alias restart-models="restart_ai "
|
||||
alias reload-ai="restart_ai "
|
||||
|
||||
function ollama_sizes() {
|
||||
# VAR1 - Format. Choices are Markdown or Spaces.
|
||||
typeset -l fmt="$1"
|
||||
if [[ -z "$fmt" ]]; then
|
||||
fmt=s
|
||||
fi
|
||||
if [[ "$fmt" != "s"* && "$fmt" != "m"* ]]; then
|
||||
echo "ERROR: Format incorrect, choose Markdown or Spaces."
|
||||
return 1
|
||||
fi
|
||||
sep=""
|
||||
if [[ $fmt == "m"* ]]; then
|
||||
sep="|"
|
||||
fi
|
||||
|
||||
# VAR2 - Context
|
||||
typeset -i num_ctx="$2"
|
||||
context=""
|
||||
if [[ -n $num_ctx ]]; then
|
||||
context=',"options":{"num_ctx":'$num_ctx'}'
|
||||
fi
|
||||
|
||||
# Column Formatting
|
||||
col1=25
|
||||
col2=12
|
||||
col3=8
|
||||
col4=12
|
||||
spacing="$sep %-${col1}s $sep %${col2}s $sep %${col3}s $sep %${col4}s $sep"
|
||||
|
||||
#echo " Stopping all ollama usage..."
|
||||
#sudo sv restart ollama >/dev/null 2>&1
|
||||
|
||||
#echo -e "\n Checking model disk and RAM sizes...\n"
|
||||
|
||||
printf "$spacing\n" "Model Name" "HDD (ls)" "Context" "RAM (ps)"
|
||||
if [[ $fmt == "m"* ]]; then
|
||||
printf "$spacing\n" ":------------------------" "-----------:" "-------:" "-----------:"
|
||||
fi
|
||||
|
||||
for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
|
||||
#ollama run $model "" --format json >/dev/null 2>&1
|
||||
curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":1'$context'}' \
|
||||
>/dev/null 2>&1
|
||||
status=$?
|
||||
|
||||
hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
|
||||
ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
|
||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||
|
||||
if [[ "$status" != 0 ]]; then
|
||||
ram="TOO LARGE $ram"
|
||||
ctx="FAIL $ctx"
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
printf "$spacing\n" "$model" "$hdd" "$ctx" "$ram"
|
||||
|
||||
ollama stop $model >/dev/null 2>&1
|
||||
while ollama ps | grep $model; do
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
|
||||
#echo "Done!!"
|
||||
}
|
||||
alias ollama-sizes="ollama_sizes "
|
||||
alias model-sizes="ollama_sizes "
|
||||
|
||||
function ollama_threads() {
|
||||
# VAR1 - Model
|
||||
model="$1"
|
||||
if [[ -z $model ]]; then
|
||||
model="qwen3.5:4b"
|
||||
fi
|
||||
|
||||
# VAR2 - Context
|
||||
typeset -i num_ctx="$2"
|
||||
context=""
|
||||
if [[ -n $num_ctx ]]; then
|
||||
context=',"num_ctx":'$num_ctx''
|
||||
fi
|
||||
|
||||
# Physical CPU Count (no longer using ansible_processor_vcpus, {{ ansible_processor_vcpus }})
|
||||
# Will be something like 0-7, then get expanded to 0 1 2 3 4 5 6 7.
|
||||
# Runnning with 0 causes default behavior, 1 per physical core, 8!
|
||||
cores=$(cat /sys/devices/cpu_core/cpus | tr ',' '\n' | sed 's/-/ /' | xargs -L1 seq)
|
||||
|
||||
# Flatten the output (trims newlines and extra spacing).
|
||||
cores=$(echo $cores)
|
||||
|
||||
# Take out useless tests. 1 CPU is silly. If we have 8 physical then
|
||||
# only need to look at 4, 6, 7, and 8 (which is done by 0/default).
|
||||
cores=${cores/ 1 / }
|
||||
if [[ "$cores" == *" 7" ]]; then
|
||||
cores=${cores/ 2 / }
|
||||
cores=${cores/ 3 / }
|
||||
cores=${cores/ 5 / }
|
||||
fi
|
||||
|
||||
# Do the 0 test last.
|
||||
cores="${cores/0 / } 0"
|
||||
|
||||
#echo " Stopping all ollama usage..."
|
||||
#sudo sv restart ollama >/dev/null 2>&1
|
||||
|
||||
# Benchmark Thread Count
|
||||
for t in $cores; do
|
||||
start=$SECONDS
|
||||
echo -n "`date` - $model w/ $t Thread(s): "
|
||||
|
||||
speed=$(curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \
|
||||
| jq '.eval_count * 1000000000 / .eval_duration'
|
||||
)
|
||||
finish=$SECONDS
|
||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||
time=$(( $finish - $start ))
|
||||
|
||||
#echo "tokens/sec: $speed @ context=$ctx, time: $time secs"
|
||||
printf "%-32s %18s %16s\n" "tokens/sec: $speed" "@ context=$ctx," "time: $time secs"
|
||||
|
||||
ollama stop $model >/dev/null 2>&1
|
||||
done
|
||||
}
|
||||
alias ollama-threads="ollama_threads "
|
||||
alias ollama-cpus="ollama_threads "
|
||||
alias ollama-cpu="ollama_threads "
|
||||
state: present
|
||||
create: false
|
||||
backup: true
|
||||
|
||||
Reference in new issue
Block a user