me/env-ansible
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm, add sleepy finishes to each service, further edit facts, fix qwen config file and add validator task, move non-ollama web services to non-critical check, start refactoring shell functions, make all context 3200 so programs don;t have to restart the models.

Browse Source
This commit is contained in:
me
2026-10-14 10:23:03 -07:00
parent f3632fa13a
commit 03d0978a64
1 changed files with 87 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tasks/workstation/linux/software/llm.yml
+87 -63
View File
@@ -54,9 +54,8 @@
dummy_website: 'https://hyperling.com'
# Good to keep low to prevent swap usage. Going to have a minimum of 3
# though, one for Cline, two for Continue (Agent + Autocomplete).
# 20261014: Going down to 2. Either 1 strong and 1 weak (autocomplete),
# or 1 coder and 1 general. No need to exceed this even on good HW.
llm_concurrency: 2 #"{{ [(ansible_memtotal_mb / 32000) | round(0, method='ceil'), 3] | max | int }}"
# 20261014: Going down to a static of 3 since we're focusing on dense.
llm_concurrency: 3 #"{{ [(ansible_memtotal_mb / 32000) | round(0, method='ceil'), 3] | max | int }}"
# Allow having multiple chats open without them just waiting each other.
# Also setting this to a minimum of 3 for using Codium with extensions.
# 20261014: Going down to 1 since only using CPUs, no GPUs. Otherwise
@@ -72,7 +71,18 @@
# Use the MB of RAM as the context, with a maximum to prevent swapping.
#llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}"
# Let's just be consistent for now, only having 8000 on the Macbook was not usable.
llm_context: 34543
llm_context: 32000
runit_finish_sleep: |
#!/bin/sh
exec 2>&1
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "FINISH: Service returned with exit code: $1 and status: $2"
sleep=30
if [[ "$1" != 0 ]]; then
echo "FINISH: Sleeping for $sleep seconds due to error..."
sleep $sleep
fi
- name: Workstation | Linux | Software | LLMs | Facts 02
ansible.builtin.set_fact:
@@ -102,16 +112,15 @@
llm_installs: []
llm_removals:
##### QWEN #####
- { "purpose": "chat", "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded
- { "purpose": ["chat"], "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded
- { "purpose": ["chat", "code"], "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended
- { "purpose": ["chat", "code"], "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 } # succeeded
##### Mistral #####
- { "purpose": "code", "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded
- { "purpose": ["code"], "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded
# https://ollama.com/library/mistral-small : Last Updated 2025-01-30, succeeded
- { "purpose": "chat", "model": "mistral-small:24b", "provider": "mistral_small30_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 32000 }
- { "purpose": "chat", "model": "mistral-small:22b", "provider": "mistral_small30_0220", "size_mb": "{{ llm_size_coef * 13000 }}", "context": 128000 } # Yes, the smaller one has larger context.
- { "purpose": ["chat"], "model": "mistral-small:24b", "provider": "mistral_small30_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 32000 }
- { "purpose": ["chat"], "model": "mistral-small:22b", "provider": "mistral_small30_0220", "size_mb": "{{ llm_size_coef * 13000 }}", "context": 128000 } # Yes, the smaller one has larger context.
# https://ollama.com/library/mistral-small3.1 : Last Update 2025-04-07, succeeded
- { "purpose": "chat", "model": "mistral-small3.1:24b", "provider": "mistral_small31_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 }
- { "purpose": ["chat"], "model": "mistral-small3.1:24b", "provider": "mistral_small31_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 }
llm_choices:
##### QWEN #####
### QWEN >3.5 - Need to try!
@@ -123,33 +132,34 @@
# https://ollama.com/library/qwen3.6 : Last Updated 2026-10-13 (yesterday as of this commit!!)
# Has been succeeded but dense and does not overthink so still worthy
- { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
# https://ollama.com/library/qwen3.5 : Last Updated 2026-10-13 (yesterday as of this commit!!)
### QWEN 3.5 - Good for generic usage and a little coding.
- { "purpose": "chat", "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
- { "purpose": "chat", "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
- { "purpose": "chat", "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
- { "purpose": "chat", "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
# skip 27b and use 35b if going this high.
- { "purpose": ["chat"], "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
- { "purpose": ["chat"], "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
- { "purpose": ["chat"], "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
- { "purpose": ["chat"], "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
- { "purpose": ["chat", "code"], "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 256000 } # succeeded
- { "purpose": ["chat", "code"], "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 }
- { "purpose": ["chat", "code"], "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 }
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
- { "purpose": ["code", "chat"], "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 }
- { "purpose": ["code", "chat"], "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 }
### QWEN 2.5 Coder - Ol'reliable!
- { "purpose": "code", "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
- { "purpose": ["code"], "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
- { "purpose": ["code"], "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
- { "purpose": ["code"], "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
- { "purpose": ["code"], "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
- { "purpose": ["code"], "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
- { "purpose": ["code"], "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
##### Mistral, Ministral, and Devstral - By a French company? #####
# https://ollama.com/library/devstral-small-2 : Last Updated 2025-12-13
# Most recommended in Mistral family for agentic coding.
- { "purpose": "code", "model": "devstral-small-2:24b", "provider": "mistral_dev20_0240_small", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 384000 }
- { "purpose": ["code"], "model": "devstral-small-2:24b", "provider": "mistral_dev20_0240_small", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 384000 }
# https://ollama.com/library/codestral : Last Updated 2024-09-03
# Mistral's first ever coder agent, recommended for autocomplete (FIM).
- { "purpose": "code", "model": "codestral:22b", "provider": "mistral_code_0220", "size_mb": "{{ llm_size_coef * 12000 }}", "context": 32000 }
- { "purpose": ["code"], "model": "codestral:22b", "provider": "mistral_code_0220", "size_mb": "{{ llm_size_coef * 12000 }}", "context": 32000 }
# https://ollama.com/library/mistral : Last Updated 2025-07-13
- { "purpose": "chat", "model": "mistral:7b", "provider": "mistral_gen_0070", "size_mb": "{{ llm_size_coef * 4400 }}", "context": 32000 }
- { "purpose": ["chat"], "model": "mistral:7b", "provider": "mistral_gen_0070", "size_mb": "{{ llm_size_coef * 4400 }}", "context": 32000 }
# https://ollama.com/library/mistral-small3.2 : Last Updated 2025-06-20
- { "purpose": ["chat", "lang"], "model": "mistral-small3.2:24b", "provider": "mistral_small32_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 } # rated best for Croatian
# https://ollama.com/library/ministral-3 : Last Updated 2025-12-13
@@ -282,6 +292,13 @@
exec chpst -u {{ user }} ollama serve
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ ollama }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log
ansible.builtin.copy:
dest: "{{ ollama_runit_log_script }}"
@@ -337,6 +354,13 @@
exec chpst -u {{ user }} zeroclaw daemon -v --log-level info
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ zeroclaw }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_script }}"
@@ -452,10 +476,7 @@
ansible.builtin.copy:
dest: /etc/sv/open-webui/finish
content: |
#!/bin/sh
exec 2>&1
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "FINISH: Service returned with exit code: $1 and status: $2"
{{ runit_finish_sleep }}
echo "FINISH: Stopping..."
docker stop open-webui &&
@@ -467,12 +488,6 @@
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
sleep=30
if [[ "$1" != 0 ]]; then
echo "FINISH: Sleeping for $sleep seconds due to error..."
sleep $sleep
fi
echo "FINISH: Done!"
mode: "0755"
@@ -530,10 +545,7 @@
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/finish
content: |
#!/bin/sh -e
exec 2>&1
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "FINISH: Service returned with exit code: $1 and status: $2"
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log
@@ -584,10 +596,7 @@
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/finish
content: |
#!/bin/sh -e
exec 2>&1
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "FINISH: Service returned with exit code: $1 and status: $2"
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log
@@ -626,7 +635,7 @@
- { 'name': 'swap-stopper' , 'status': true }
- { 'name': '{{ qwen_code }}', 'status': "{{ qwen_code_enabled }}" }
- name: Workstation | Linux | Software | LLMs | Services | Verify
- name: Workstation | Linux | Software | LLMs | Services | Verify | Critical
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
@@ -637,8 +646,6 @@
delay: 5
loop:
- "{{ 'http://127.0.0.1:11434' if ollama_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:42617/health' if zeroclaw_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:4170' if qwen_code_enabled else dummy_website }}"
- name: Workstation | Linux | Software | LLMs | Services | Verify | Okay to Fail
ansible.builtin.uri:
@@ -650,7 +657,9 @@
retries: 2
delay: 5
loop:
- "{{ 'http://127.0.0.1:42617/health' if zeroclaw_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:' ~ open_webui_port ~ '/health' if open_webui_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:4170' if qwen_code_enabled else dummy_website }}"
ignore_errors: true
# Services - End #
@@ -662,16 +671,23 @@
path: "{{ user_user.home }}/{{ item }}"
marker: "# {mark} ANSIBLE MANAGED BLOCK | ollama | Manual Script"
content: |
function pull_llms() {
echo "Manually Pulling Models"
function list_llms() {
{% for llm in llm_installs %}
echo -e "\n\n Starting 'ollama pull {{ llm.model }}' \n\n"
ollama pull {{ llm.model }}
status=$?
echo "Finished with status $status."
echo "{{ llm.model }}"
{% endfor %}
}
function pull_llms() {
echo "Manually Pulling Models"
for model in `list_llms`; do
echo -e "\n\n Starting 'ollama pull $model' \n\n"
ollama pull $model
status=$?
echo "Finished with status $status."
done
echo "Done!!"
}
alias pull-llms="pull_llms "
@@ -697,7 +713,7 @@
alias chat="ollama_chat "
function ollama_code() {
ollama run {{ 'qwen3-coder:30b' if ansible_memtotal_mb > 32000 else 'qwen2.5-coder:7b' }}
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen2.5-coder:7b' }}
}
alias ollama-code="ollama_code "
alias code="ollama_code "
@@ -847,14 +863,14 @@
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context, 24000] | min | int }}
contextLength: {{ [llm_context, llm.context, 32000] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 8000] | min | int }}
- name: {{ llm.provider }}_autocomplete
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context, 12000] | min | int }}
contextLength: {{ [llm_context, llm.context, 32000] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }}
roles:
- autocomplete
@@ -881,18 +897,17 @@
"modelProviders": {
"openai": [
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "code" in llm.purpose %}
{
"baseUrl": "http://127.0.0.1:11434/v1",
"envKey": "OLLAMA_API_KEY",
"id": "{{ llm.model }}",
"name": "{{ llm.provider }}"
}{% if not loop.last %},{% endif %}
"name": "{{ '' if not llm.model in llm_models.stdout else 'NA_'
}}{{ '' if not "code" in llm.purpose else 'ZZ_'
}}{{ llm.provider }}"
,"baseUrl": "http://127.0.0.1:11434/v1"
,"envKey": "OLLAMA_API_KEY"
,"id": "{{ llm.model }}"
}{{ ',' if not loop.last else '' }}
{% endif %}
{% endif %}
{% endfor %}
]
},
@@ -913,6 +928,15 @@
#- "{{ user_user.home }}/Code/.qwen/settings.json"
when: qwen_code_enabled
- name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config | Validate
ansible.builtin.shell: |
set -o pipefail
cat {{ user_user.home }}/.qwen/settings.json | jq
register: shell_check
changed_when: shell_check.rc == 0
failed_when: shell_check.rc != 0
when: qwen_code_enabled
# Models - End #
when: ansible_pkg_mgr == "pacman"