me/env-ansible
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm, tuning the context and token parameters.

Browse Source
This commit is contained in:
me
2026-10-10 13:10:29 -07:00
parent 3381eb6d70
commit c6cc631d8c
1 changed files with 69 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tasks/workstation/linux/software/llm.yml
+69 -45
View File
@@ -59,7 +59,7 @@
# - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
llm_size_coef: 1.3
# Take the GB of RAM and use it as the MB of context.
llm_context: "{{ ansible_memtotal_mb | int }}"
llm_context: "{{ [(ansible_memtotal_mb | round(0, method='ceil')), 32000] | max | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 02
ansible.builtin.set_fact:
@@ -67,6 +67,8 @@
ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}"
zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}"
zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}"
# Amount of output tokens allowed per response. Half of context is ok.
llm_tokens: "{{ (llm_context / 4) | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 03
ansible.builtin.set_fact:
@@ -86,33 +88,33 @@
ansible.builtin.set_fact:
llm_installs: []
llm_removals:
- { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000 }
- { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000 }
- { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}" }
- { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 }
- { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 }
- { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 }
llm_choices:
### QWEN >3.5 - Need to try!
- { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000 }
- { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000 }
- { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 }
- { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 }
### QWEN 3.5 - Good for generic usage and a little coding.
- { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}" }
- { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}" }
- { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}" }
- { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}" }
- { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
- { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
- { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
- { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
# skip 27b and use 35b
- { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}" }
- { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}" }
- { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 }
- { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 }
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
- { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}" }
- { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}" }
- { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 }
- { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 }
### QWEN 2.5 Coder - Ol'reliable!
- { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}" }
- { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}" }
- { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}" }
- { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}" }
- { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}" }
- { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}" }
- { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
- { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
- { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
- { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
- { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
- { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
### Mistral, Ministral, and Devstral - By a French company?
- { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000 }
- { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 }
#- mistral:7b
#- ministral-3:3b
#- ministral-3:8b
@@ -141,6 +143,9 @@
- ansible_memtotal_mb
- llm_concurrency
- llm_parallel
- llm_size_coef
- llm_context
- llm_tokens
- llm_installs
- llm_removals
- llm_choices
@@ -336,14 +341,14 @@
while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone.
echo "Stopping..."
echo "RUN: Stopping..."
docker stop open-webui &&
echo "Container stopped" ||
echo "Container already stopped."
echo "Deleting..."
echo "RUN: Container stopped" ||
echo "RUN: Container already stopped."
echo "RUN: Deleting..."
docker rm -f open-webui &&
echo "Container deleted" ||
echo "Container already deleted."
echo "RUN: Container deleted" ||
echo "RUN: Container already deleted."
# Start the open-webui program.
set -e
@@ -367,14 +372,22 @@
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "Service finished with exit code: $1 and status: $2"
echo "Stopping..."
echo "FINISH: Stopping..."
docker stop open-webui &&
echo "Container stopped" ||
echo "Container already stopped."
echo "Deleting..."
echo "FINISH: Container stopped" ||
echo "FINISH: Container already stopped."
echo "FINISH: Deleting..."
docker rm -f open-webui &&
echo "Container deleted" ||
echo "Container already deleted."
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
sleep=30
if [[ "$1" != 0 ]]; then
echo "FINISH: Sleeping for $sleep seconds due to error..."
sleep $sleep
fi
echo "FINISH: Done!"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log
@@ -571,10 +584,13 @@
block: |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
[providers.models.ollama.{{ llm.provider }}]
model = "{{ llm.model }}"
timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
context_window = {{ llm_context }}
context_window = {{ [llm_context, llm.context] | min | int }}
max_tokens = {{ [llm_tokens, (llm.context / 4), 16000] | min | int }}
{% endif %}
{% endfor %}
state: present
@@ -591,23 +607,31 @@
version: 1.0.0
schema: v1
models:
# | Role | `contextLength` | `maxTokens` | Notes |
# |------|-----------------|-------------|-------|
# | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight |
# | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
- name: {{ llm.provider }}
- name: {{ llm.provider }}_chat-edit
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ llm_context }}
maxTokens: {{ llm_context / 4 }}
#- name: {{ llm.provider }}_autocomplete
# provider: ollama
# model: {{ llm.model }}
# defaultCompletionOptions:
# contextLength: {{ llm_context }}
# maxTokens: {{ llm_context }}
# roles:
# - autocomplete
contextLength: {{ [llm_context, llm.context] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 16000] | min | int }}
- name: {{ llm.provider }}_autocomplete
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }}
roles:
- autocomplete
{% endif %}
{% endfor %}
mode: "0644"