me/env-ansible
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm, tuning the context and token parameters.

Browse Source
This commit is contained in:
me
2026-10-10 13:10:29 -07:00
parent 3381eb6d70
commit c6cc631d8c
1 changed files with 69 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tasks/workstation/linux/software/llm.yml
+69 -45
View File
@@ -59,7 +59,7 @@
# - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000. # - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
llm_size_coef: 1.3 llm_size_coef: 1.3
# Take the GB of RAM and use it as the MB of context. # Take the GB of RAM and use it as the MB of context.
llm_context: "{{ ansible_memtotal_mb | int }}" llm_context: "{{ [(ansible_memtotal_mb | round(0, method='ceil')), 32000] | max | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 02 - name: Workstation | Linux | Software | LLMs | Facts 02
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -67,6 +67,8 @@
ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}" ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}"
zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}" zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}"
zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}" zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}"
# Amount of output tokens allowed per response. Half of context is ok.
llm_tokens: "{{ (llm_context / 4) | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 03 - name: Workstation | Linux | Software | LLMs | Facts 03
ansible.builtin.set_fact: ansible.builtin.set_fact:
@@ -86,33 +88,33 @@
ansible.builtin.set_fact: ansible.builtin.set_fact:
llm_installs: [] llm_installs: []
llm_removals: llm_removals:
- { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000 } - { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 }
- { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000 } - { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 }
- { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}" } - { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 }
llm_choices: llm_choices:
### QWEN >3.5 - Need to try! ### QWEN >3.5 - Need to try!
- { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000 } - { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 }
- { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000 } - { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 }
### QWEN 3.5 - Good for generic usage and a little coding. ### QWEN 3.5 - Good for generic usage and a little coding.
- { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}" } - { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
- { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}" } - { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
- { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}" } - { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
- { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}" } - { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
# skip 27b and use 35b # skip 27b and use 35b
- { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}" } - { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 }
- { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}" } - { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 }
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it. ### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
- { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}" } - { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 }
- { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}" } - { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 }
### QWEN 2.5 Coder - Ol'reliable! ### QWEN 2.5 Coder - Ol'reliable!
- { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}" } - { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
- { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}" } - { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
- { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}" } - { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
- { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}" } - { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
- { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}" } - { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
- { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}" } - { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
### Mistral, Ministral, and Devstral - By a French company? ### Mistral, Ministral, and Devstral - By a French company?
- { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000 } - { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 }
#- mistral:7b #- mistral:7b
#- ministral-3:3b #- ministral-3:3b
#- ministral-3:8b #- ministral-3:8b
@@ -141,6 +143,9 @@
- ansible_memtotal_mb - ansible_memtotal_mb
- llm_concurrency - llm_concurrency
- llm_parallel - llm_parallel
- llm_size_coef
- llm_context
- llm_tokens
- llm_installs - llm_installs
- llm_removals - llm_removals
- llm_choices - llm_choices
@@ -336,14 +341,14 @@
while [ ! -S /var/run/docker.sock ]; do sleep 1; done while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone. # Ensure the old container is gone.
echo "Stopping..." echo "RUN: Stopping..."
docker stop open-webui && docker stop open-webui &&
echo "Container stopped" || echo "RUN: Container stopped" ||
echo "Container already stopped." echo "RUN: Container already stopped."
echo "Deleting..." echo "RUN: Deleting..."
docker rm -f open-webui && docker rm -f open-webui &&
echo "Container deleted" || echo "RUN: Container deleted" ||
echo "Container already deleted." echo "RUN: Container already deleted."
# Start the open-webui program. # Start the open-webui program.
set -e set -e
@@ -367,14 +372,22 @@
# $1 is the exit code, $2 is the exit status byte (signal number if killed) # $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "Service finished with exit code: $1 and status: $2" echo "Service finished with exit code: $1 and status: $2"
echo "Stopping..." echo "FINISH: Stopping..."
docker stop open-webui && docker stop open-webui &&
echo "Container stopped" || echo "FINISH: Container stopped" ||
echo "Container already stopped." echo "FINISH: Container already stopped."
echo "Deleting..." echo "FINISH: Deleting..."
docker rm -f open-webui && docker rm -f open-webui &&
echo "Container deleted" || echo "FINISH: Container deleted" ||
echo "Container already deleted." echo "FINISH: Container already deleted."
sleep=30
if [[ "$1" != 0 ]]; then
echo "FINISH: Sleeping for $sleep seconds due to error..."
sleep $sleep
fi
echo "FINISH: Done!"
mode: "0755" mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log - name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log
@@ -571,10 +584,13 @@
block: | block: |
{% for llm in llm_installs %} {% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %} {% if llm.model in llm_models.stdout %}
[providers.models.ollama.{{ llm.provider }}] [providers.models.ollama.{{ llm.provider }}]
model = "{{ llm.model }}" model = "{{ llm.model }}"
timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though. timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
context_window = {{ llm_context }} context_window = {{ [llm_context, llm.context] | min | int }}
max_tokens = {{ [llm_tokens, (llm.context / 4), 16000] | min | int }}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
state: present state: present
@@ -591,23 +607,31 @@
version: 1.0.0 version: 1.0.0
schema: v1 schema: v1
models: models:
# | Role | `contextLength` | `maxTokens` | Notes |
# |------|-----------------|-------------|-------|
# | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight |
# | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs |
{% for llm in llm_installs %} {% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %} {% if llm.model in llm_models.stdout %}
- name: {{ llm.provider }} - name: {{ llm.provider }}_chat-edit
provider: ollama provider: ollama
model: {{ llm.model }} model: {{ llm.model }}
defaultCompletionOptions: defaultCompletionOptions:
contextLength: {{ llm_context }} contextLength: {{ [llm_context, llm.context] | min | int }}
maxTokens: {{ llm_context / 4 }} maxTokens: {{ [llm_tokens, (llm.context / 4), 16000] | min | int }}
#- name: {{ llm.provider }}_autocomplete
# provider: ollama - name: {{ llm.provider }}_autocomplete
# model: {{ llm.model }} provider: ollama
# defaultCompletionOptions: model: {{ llm.model }}
# contextLength: {{ llm_context }} defaultCompletionOptions:
# maxTokens: {{ llm_context }} contextLength: {{ [llm_context, llm.context] | min | int }}
# roles: maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }}
# - autocomplete roles:
- autocomplete
{% endif %} {% endif %}
{% endfor %} {% endfor %}
mode: "0644" mode: "0644"