diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index dd5a5af..00f90a4 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -178,6 +178,120 @@ #- deepseek-r1:1.5b #- deepseek-r1:7b + ### 2026-10-18, checking the actual RAM usage of the models... ### + + # `ollama_sizes m 16000` + #| Model Name | HDD (ls) | Context | RAM (ps) | + #| :------------------------ | -----------: | -------: | -----------: | + #| mistral-nemo:12b | 7.1GB | 16000 | 10GB | + #| ministral-3:14b | 9.1GB | 16000 | 11GB | + #| ministral-3:8b | 6.0GB | 16000 | 7.8GB | + #| ministral-3:3b | 3.0GB | 16000 | 4.1GB | + #| mistral-small3.2:24b | 15GB | 16000 | 17GB | + #| mistral:7b | 4.4GB | 16000 | 6.8GB | + #| codestral:22b | 12GB | 16000 | 16GB | + #| devstral-small-2:24b | 15GB | 16000 | 17GB | + #| qwen2.5-coder:32b | 19GB | 16000 | 24GB | + #| qwen2.5-coder:14b | 9.0GB | 16000 | 12GB | + #| qwen2.5-coder:7b | 4.7GB | 16000 | 5.9GB | + #| qwen2.5-coder:3b | 1.9GB | 16000 | 2.7GB | + #| qwen2.5-coder:1.5b | 986MB | 16000 | 1.6GB | + #| qwen2.5-coder:0.5b | 397MB | 16000 | 701MB | + #| qwen3-coder:30b | 18GB | 16000 | 20GB | + #| qwen3.5:35b | 22GB | 16000 | 22GB | + #| qwen3.5:27b | 17GB | 16000 | 18GB | + #| qwen3.5:9b | 6.6GB | 16000 | 6.7GB | + #| qwen3.5:4b | 3.3GB | 16000 | 3.7GB | + #| qwen3.5:2b | 2.7GB | 16000 | 2.6GB | + #| qwen3.5:0.8b | 1.3GB | 16000 | 630MB | + #| qwen3.6:35b | 22GB | 16000 | 22GB | + #| qwen3.6:27b | 17GB | 16000 | 18GB | + #| qwen3.8:27b | 17GB | 16000 | 18GB | + + # `ollama_sizes m 32000` + #| Model Name | HDD (ls) | Context | RAM (ps) | + #| :------------------------ | -----------: | -------: | -----------: | + #| mistral-nemo:12b | 7.1GB | 32000 | 12GB | + #| ministral-3:14b | 9.1GB | 32000 | 13GB | + #| ministral-3:8b | 6.0GB | 32000 | 10GB | + #| ministral-3:3b | 3.0GB | 32000 | 5.9GB | + #| mistral-small3.2:24b | 15GB | 32000 | 20GB | + #| mistral:7b | 4.4GB | 32000 | 8.9GB | + #| codestral:22b | 12GB | 32000 | 20GB | + #| devstral-small-2:24b | 15GB | 32000 | 20GB | + #| qwen2.5-coder:32b | 19GB | 32000 | 28GB | + #| qwen2.5-coder:14b | 9.0GB | 32000 | 15GB | + #| qwen2.5-coder:7b | 4.7GB | 32000 | 6.9GB | + #| qwen2.5-coder:3b | 1.9GB | 32000 | 3.3GB | + #| qwen2.5-coder:1.5b | 986MB | 32000 | 2.1GB | + #| qwen2.5-coder:0.5b | 397MB | 32000 | 929MB | + #| qwen3-coder:30b | 18GB | 32000 | 21GB | + #| qwen3.5:35b | 22GB | 32000 | 22GB | + #| qwen3.5:27b | 17GB | 32000 | 19GB | + #| qwen3.5:9b | 6.6GB | 32000 | 7.3GB | + #| qwen3.5:4b | 3.3GB | 32000 | 4.2GB | + #| qwen3.5:2b | 2.7GB | 32000 | 2.8GB | + #| qwen3.5:0.8b | 1.3GB | 32000 | 695MB | + #| qwen3.6:35b | 22GB | 32000 | 22GB | + #| qwen3.6:27b | 17GB | 32000 | 18GB | + #| qwen3.8:27b | 17GB | 32000 | 18GB | + + # `ollama_sizes m 64000` + #| Model Name | HDD (ls) | Context | RAM (ps) | + #| :------------------------ | -----------: | -------: | -----------: | + #| mistral-nemo:12b | 7.1GB | 64000 | 18GB | + #| ministral-3:14b | 9.1GB | 64000 | 19GB | + #| ministral-3:8b | 6.0GB | 64000 | 14GB | + #| ministral-3:3b | 3.0GB | 64000 | 9.6GB | + #| mistral-small3.2:24b | 15GB | 64000 | 26GB | + #| mistral:7b | 4.4GB | 32768 | 9.0GB | + #| codestral:22b | 12GB | 32768 | 20GB | + #| devstral-small-2:24b | 15GB | 64000 | 26GB | + #| qwen2.5-coder:32b | 19GB | 32768 | 28GB | + #| qwen2.5-coder:14b | 9.0GB | 32768 | 15GB | + #| qwen2.5-coder:7b | 4.7GB | 32768 | 6.9GB | + #| qwen2.5-coder:3b | 1.9GB | 32768 | 3.4GB | + #| qwen2.5-coder:1.5b | 986MB | 32768 | 2.1GB | + #| qwen2.5-coder:0.5b | 397MB | 32768 | 940MB | + #| qwen3-coder:30b | 18GB | 64000 | 25GB | + #| qwen3.5:35b | 22GB | 64000 | 23GB | + #| qwen3.5:27b | 17GB | 64000 | 22GB | + #| qwen3.5:9b | 6.6GB | 64000 | 8.8GB | + #| qwen3.5:4b | 3.3GB | 64000 | 5.6GB | + #| qwen3.5:2b | 2.7GB | 64000 | 3.5GB | + #| qwen3.5:0.8b | 1.3GB | 64000 | 999MB | + #| qwen3.6:35b | 22GB | 64000 | 22GB | + #| qwen3.6:27b | 17GB | 64000 | 18GB | + #| qwen3.8:27b | 17GB | 64000 | 18GB | + + # `ollama_sizes m 256000` + #| Model Name | HDD (ls) | Context | RAM (ps) | + #| :------------------------ | -----------: | -------: | -----------: | + #| mistral-nemo:12b | 7.1GB | 256000 | 50GB | + #| ministral-3:14b | 9.1GB | 256000 | 51GB | + #| ministral-3:8b | 6.0GB | 256000 | 42GB | + #| ministral-3:3b | 3.0GB | 256000 | 30GB | + #| mistral-small3.2:24b | 15GB | 131072 | 37GB | + #| mistral:7b | 4.4GB | 32768 | 9.0GB | + #| codestral:22b | 12GB | 32768 | 20GB | + #| devstral-small-2:24b | 15GB | 256000 | 58GB | + #| qwen2.5-coder:32b | 19GB | 32768 | 28GB | + #| qwen2.5-coder:14b | 9.0GB | 32768 | 15GB | + #| qwen2.5-coder:7b | 4.7GB | 32768 | 6.9GB | + #| qwen2.5-coder:3b | 1.9GB | 32768 | 3.4GB | + #| qwen2.5-coder:1.5b | 986MB | 32768 | 2.1GB | + #| qwen2.5-coder:0.5b | 397MB | 32768 | 940MB | + #| qwen3-coder:30b | 18GB | 256000 | 45GB | + #| qwen3.5:35b | 22GB | 256000 | 28GB | + #| qwen3.5:27b | 17GB | 256000 | 35GB | + #| qwen3.5:9b | 6.6GB | 256000 | 15GB | + #| qwen3.5:4b | 3.3GB | 256000 | 12GB | + #| qwen3.5:2b | 2.7GB | 256000 | 6.7GB | + #| qwen3.5:0.8b | 1.3GB | 256000 | 2.2GB | + #| qwen3.6:35b | 22GB | 256000 | 23GB | + #| qwen3.6:27b | 17GB | 256000 | 20GB | + #| qwen3.8:27b | 17GB | 256000 | 20GB | + - name: Workstation | Linux | Software | LLMs | Facts | Merge | Print ansible.builtin.debug: var: "{{ item }}" @@ -841,34 +955,63 @@ alias restart-models="restart_ai " function ollama_sizes() { - echo " Stopping all ollama usage..." - sudo sv restart ollama + # VAR1 - Format. Choices are Markdown or Spaces. + typeset -l fmt="$1" + if [[ -z "$fmt" ]]; then + fmt=s + fi + if [[ "$fmt" != "s"* && "$fmt" != "m"* ]]; then + echo "ERROR: Format incorrect, choose Markdown or Spaces." + return 1 + fi + sep="" + if [[ $fmt == "m"* ]]; then + sep="|" + fi + + # VAR2 - Context + typeset -i num_ctx="$2" + context="" + if [[ -n $num_ctx ]]; then + context=',"options":{"num_ctx":'$num_ctx'}' + fi + + # Column Formatting + col1=25 + col2=12 + col3=8 + col4=12 + spacing="$sep %-${col1}s $sep %${col2}s $sep %${col3}s $sep %${col4}s $sep" + + #echo " Stopping all ollama usage..." + sudo sv restart ollama >/dev/null 2>&1 + + #echo -e "\n Checking model disk and RAM sizes...\n" + + printf "$spacing\n" "Model Name" "HDD (ls)" "Context" "RAM (ps)" + if [[ $fmt == "m"* ]]; then + printf "$spacing\n" ":------------------------" "-----------:" "-------:" "-----------:" + fi - echo -e "\n Checking model disk and RAM sizes...\n" - echo "| Model Name | HDD (ls) | RAM (ps) |" - echo "| +---- | --+ | --+ |" for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do - echo -n "| $model" #ollama run $model "" --format json >/dev/null 2>&1 - #set -x curl -s http://localhost:11434/api/generate \ - -d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":1}' \ + -d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":1'$context'}' \ >/dev/null 2>&1 - set +x hdd="`ollama ls | grep $model | awk '{print $3 $4}'`" - echo -n " | $hdd" ram="`ollama ps | grep $model | awk '{print $3 $4}'`" - echo -n " | $ram" + ctx="`ollama ps | grep $model | awk '{print $7}'`" + + printf "$spacing\n" "$model" "$hdd" "$ctx" "$ram" ollama stop $model >/dev/null 2>&1 while ollama ps | grep $model; do sleep 1 done - echo " |" done - echo "Done!!" + #echo "Done!!" } alias ollama-sizes="ollama_sizes " alias model-sizes="ollama_sizes "