From c6cc631d8c193bb71212319d7a35e407b7159107 Mon Sep 17 00:00:00 2001 From: Hyperling Date: Sat, 10 Oct 2026 13:10:29 -0700 Subject: [PATCH] llm, tuning the context and token parameters. --- tasks/workstation/linux/software/llm.yml | 114 ++++++++++++++--------- 1 file changed, 69 insertions(+), 45 deletions(-) diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index a52d28e..840fb11 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -59,7 +59,7 @@ # - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000. llm_size_coef: 1.3 # Take the GB of RAM and use it as the MB of context. - llm_context: "{{ ansible_memtotal_mb | int }}" + llm_context: "{{ [(ansible_memtotal_mb | round(0, method='ceil')), 32000] | max | int }}" - name: Workstation | Linux | Software | LLMs | Facts 02 ansible.builtin.set_fact: @@ -67,6 +67,8 @@ ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}" zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}" zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}" + # Amount of output tokens allowed per response. Half of context is ok. + llm_tokens: "{{ (llm_context / 4) | int }}" - name: Workstation | Linux | Software | LLMs | Facts 03 ansible.builtin.set_fact: @@ -86,33 +88,33 @@ ansible.builtin.set_fact: llm_installs: [] llm_removals: - - { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000 } - - { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000 } - - { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}" } + - { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } + - { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 } + - { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 } llm_choices: ### QWEN >3.5 - Need to try! - - { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000 } - - { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000 } + - { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 } + - { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 } ### QWEN 3.5 - Good for generic usage and a little coding. - - { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}" } - - { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}" } - - { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}" } - - { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}" } + - { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 } + - { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 } + - { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 } + - { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 } # skip 27b and use 35b - - { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}" } - - { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}" } + - { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 } + - { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 } ### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it. - - { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}" } - - { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}" } + - { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 } + - { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 } ### QWEN 2.5 Coder - Ol'reliable! - - { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}" } - - { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}" } - - { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}" } - - { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}" } - - { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}" } - - { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}" } + - { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 } + - { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 } + - { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 } + - { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 } + - { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 } + - { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 } ### Mistral, Ministral, and Devstral - By a French company? - - { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000 } + - { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 } #- mistral:7b #- ministral-3:3b #- ministral-3:8b @@ -141,6 +143,9 @@ - ansible_memtotal_mb - llm_concurrency - llm_parallel + - llm_size_coef + - llm_context + - llm_tokens - llm_installs - llm_removals - llm_choices @@ -336,14 +341,14 @@ while [ ! -S /var/run/docker.sock ]; do sleep 1; done # Ensure the old container is gone. - echo "Stopping..." + echo "RUN: Stopping..." docker stop open-webui && - echo "Container stopped" || - echo "Container already stopped." - echo "Deleting..." + echo "RUN: Container stopped" || + echo "RUN: Container already stopped." + echo "RUN: Deleting..." docker rm -f open-webui && - echo "Container deleted" || - echo "Container already deleted." + echo "RUN: Container deleted" || + echo "RUN: Container already deleted." # Start the open-webui program. set -e @@ -367,14 +372,22 @@ # $1 is the exit code, $2 is the exit status byte (signal number if killed) echo "Service finished with exit code: $1 and status: $2" - echo "Stopping..." + echo "FINISH: Stopping..." docker stop open-webui && - echo "Container stopped" || - echo "Container already stopped." - echo "Deleting..." + echo "FINISH: Container stopped" || + echo "FINISH: Container already stopped." + echo "FINISH: Deleting..." docker rm -f open-webui && - echo "Container deleted" || - echo "Container already deleted." + echo "FINISH: Container deleted" || + echo "FINISH: Container already deleted." + + sleep=30 + if [[ "$1" != 0 ]]; then + echo "FINISH: Sleeping for $sleep seconds due to error..." + sleep $sleep + fi + + echo "FINISH: Done!" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log @@ -571,10 +584,13 @@ block: | {% for llm in llm_installs %} {% if llm.model in llm_models.stdout %} + [providers.models.ollama.{{ llm.provider }}] model = "{{ llm.model }}" timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though. - context_window = {{ llm_context }} + context_window = {{ [llm_context, llm.context] | min | int }} + max_tokens = {{ [llm_tokens, (llm.context / 4), 16000] | min | int }} + {% endif %} {% endfor %} state: present @@ -591,23 +607,31 @@ version: 1.0.0 schema: v1 models: + + # | Role | `contextLength` | `maxTokens` | Notes | + # |------|-----------------|-------------|-------| + # | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight | + # | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs | + {% for llm in llm_installs %} {% if llm.model in llm_models.stdout %} - - name: {{ llm.provider }} + - name: {{ llm.provider }}_chat-edit provider: ollama model: {{ llm.model }} defaultCompletionOptions: - contextLength: {{ llm_context }} - maxTokens: {{ llm_context / 4 }} - #- name: {{ llm.provider }}_autocomplete - # provider: ollama - # model: {{ llm.model }} - # defaultCompletionOptions: - # contextLength: {{ llm_context }} - # maxTokens: {{ llm_context }} - # roles: - # - autocomplete + contextLength: {{ [llm_context, llm.context] | min | int }} + maxTokens: {{ [llm_tokens, (llm.context / 4), 16000] | min | int }} + + - name: {{ llm.provider }}_autocomplete + provider: ollama + model: {{ llm.model }} + defaultCompletionOptions: + contextLength: {{ [llm_context, llm.context] | min | int }} + maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }} + roles: + - autocomplete + {% endif %} {% endfor %} mode: "0644"