me/env-ansible
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
41d01c70f5f4a1b85d7c21f75826292592d03f68
env-ansible/tasks/workstation/linux/software/llm.yml
T

1050 lines
43 KiB
YAML
Raw Blame History

---
# 2026-10-05 Hyperling
# Install a local LLM for use offline.
# - https://zeroclaw.net/blog/install-zeroclaw
# - https://zeroclaw.net/blog/set-up-zeroclaw-bot-ollama-telegram
#
# Packages Details
# - https://archlinux.org/packages/extra/x86_64/zeroclaw/
# - https://archlinux.org/packages/extra/x86_64/ollama/
#
# Individual Guides
# - https://github.com/zeroclaw-labs/zeroclaw
# - https://wiki.archlinux.org/title/Ollama
#
# Model Explanations
# - https://webscraft.org/blog/ollama-na-8-gb-ram-yaki-modeli-pratsyuyut-u-2026?lang=en
#
# Models
# - https://ollama.com/search
#
# References
# - https://docs.zeroclaw.com/master/en/reference/config.html
# - https://docs.ollama.com
# - https://docs.ollama.com/faq#setting-environment-variables-on-linux
# - https://modelpiper.com/blog/ollama-environment-variables
# - https://qwenlm.github.io/qwen-code-docs/en/users/configuration/settings/#environment-variables--env-files
# - https://qwenlm.github.io/qwen-code-docs/en/users/qwen-serve/
# Currently only testing this on Arch based distros (pacman).
- name: Workstation | Linux | Software | LLMs
block:
# Facts - Start #
- name: Workstation | Linux | Software | LLMs | Facts | Provisioned
ansible.builtin.debug:
var: "{{ item }}"
loop:
- ollama_enabled
- manage_models
- hf_token
- zeroclaw_enabled
- open_webui_enabled
- open_webui_port
- open_webui_key
- codium_config
- qwen_code_enabled
- qwen_code_key
- open_terminal_enabled
- open_terminal_port
- open_terminal_key
- name: Workstation | Linux | Software | LLMs | Facts 01
ansible.builtin.set_fact:
ollama: ollama
zeroclaw: zeroclaw
zerocode: zerocode
qwen_code: qwen-code
dummy_service: 'cronie'
dummy_website: 'https://hyperling.com'
# Good to keep low to prevent swap usage. Going to have a minimum of 3
# though, one for Cline, two for Continue (Agent + Autocomplete).
# 20261014: Going down to a static of 3 since we're focusing on dense.
llm_concurrency: 3 #"{{ [(ansible_memtotal_mb / 32000) | round(0, method='ceil'), 3] | max | int }}"
# Allow having multiple chats open without them just waiting each other.
# Also setting this to a minimum of 3 for using Codium with extensions.
# 20261014: Going down to 1 since only using CPUs, no GPUs. Otherwise
# the models seem to take excessively long to answer even small things.
llm_parallel: 1 #"{{ [(ansible_processor_vcpus / 2) | round(0, method='ceil'), 3] | max | int }}"
# Most LLMs take a coefficient of about 1.2 to 1.5 of their file size in
# RAM, which is usually less than guessing a usage of weight*1000.
# Example:
# - qwen3.5:9b would be 6.6GB * 1.3 = 8580, rather than 9000.
# Conversely:
# - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
llm_size_coef: 1.3
# Base the context on MB of RAM, with a minumum 64k for decent thinking.
llm_context: "{{ [(ansible_memtotal_mb * 4) | round(-3, method='ceil'), 64000] | max | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 02
ansible.builtin.set_fact:
ollama_runit_dir: "{{ runit_script_dir }}/{{ ollama }}"
ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}"
zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}"
zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}"
# Amount of output tokens allowed per response. Half of context is ok.
llm_tokens: "{{ (llm_context / 8) | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 03
ansible.builtin.set_fact:
ollama_runit_script: "{{ ollama_runit_dir }}/{{ runit_service_script }}"
ollama_runit_log_dir: "{{ ollama_runit_dir }}/{{ runit_log_dir }}"
ollama_runit_log_config: "{{ ollama_runit_log_dest }}/{{ global_log_config }}"
zeroclaw_runit_script: "{{ zeroclaw_runit_dir }}/{{ runit_service_script }}"
zeroclaw_runit_log_dir: "{{ zeroclaw_runit_dir }}/{{ runit_log_dir }}"
zeroclaw_runit_log_config: "{{ zeroclaw_runit_log_dest }}/{{ global_log_config }}"
- name: Workstation | Linux | Software | LLMs | Facts 04
ansible.builtin.set_fact:
ollama_runit_log_script: "{{ ollama_runit_log_dir }}/{{ runit_log_script }}"
zeroclaw_runit_log_script: "{{ zeroclaw_runit_log_dir }}/{{ runit_log_script }}"
- name: Workstation | Linux | Software | LLMs | Facts | LLMs
ansible.builtin.set_fact:
llm_installs: []
llm_removals:
##### QWEN #####
- { "purpose": ["chat"], "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded
##### Mistral #####
- { "purpose": ["code"], "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded
# https://ollama.com/library/mistral-small : Last Updated 2025-01-30, succeeded
- { "purpose": ["chat"], "model": "mistral-small:24b", "provider": "mistral_small30_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 32000 }
- { "purpose": ["chat"], "model": "mistral-small:22b", "provider": "mistral_small30_0220", "size_mb": "{{ llm_size_coef * 13000 }}", "context": 128000 } # Yes, the smaller one has larger context.
# https://ollama.com/library/mistral-small3.1 : Last Update 2025-04-07, succeeded
- { "purpose": ["chat"], "model": "mistral-small3.1:24b", "provider": "mistral_small31_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 }
llm_choices:
##### QWEN #####
### QWEN >3.5 - Need to try!
# Supposedly great for coding over any previous version.
# Prefer 27b (dense) over anything higher which likely uses MoE (essentially only uses 3b inside of a 35b, dumber but faster and potentially has access to more info).
# https://ollama.com/library/qwen3.8 : Last Updated 2026-09-25
# Known to overthink, suggested to prefer 3.6:27b unless setting temp to low/medium.
- { "purpose": ["code", "chat", "lang"], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
# https://ollama.com/library/qwen3.6 : Last Updated 2026-10-13 (yesterday as of this commit!!)
# Has been succeeded but dense and does not overthink so still worthy
- { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
- { "purpose": ["chat", "code", "lang"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended
# https://ollama.com/library/qwen3.5 : Last Updated 2026-10-13 (yesterday as of this commit!!)
### QWEN 3.5 - Good for generic usage and a little coding.
- { "purpose": ["chat"], "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 262144 }
- { "purpose": ["chat"], "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 262144 }
- { "purpose": ["chat"], "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 262144 }
- { "purpose": ["chat"], "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 262144 }
- { "purpose": ["chat", "code"], "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 262144 } # succeeded
- { "purpose": ["chat", "code"], "model": "qwen3.5:35b", "provider": "qwen_35_0350_moe", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 262144 }
- { "purpose": ["chat", "code"], "model": "qwen3.5:122b", "provider": "qwen_35_1220_moe", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 262144 }
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
- { "purpose": ["code", "chat"], "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300_moe", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 262144 }
- { "purpose": ["code", "chat"], "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200_moe", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 262144 }
### QWEN 2.5 Coder - Ol'reliable!
- { "purpose": ["code"], "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32768 }
##### Mistral, Ministral, and Devstral - By a French company? #####
# https://ollama.com/library/devstral-small-2 : Last Updated 2025-12-13
# Most recommended in Mistral family for agentic coding.
- { "purpose": ["code"], "model": "devstral-small-2:24b", "provider": "mistral_dev20_0240_small", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 393216 }
# https://ollama.com/library/codestral : Last Updated 2024-09-03
# Mistral's first ever coder agent, recommended for autocomplete (FIM).
- { "purpose": ["code"], "model": "codestral:22b", "provider": "mistral_code_0220", "size_mb": "{{ llm_size_coef * 12000 }}", "context": 32768 }
# https://ollama.com/library/mistral : Last Updated 2025-07-13
- { "purpose": ["chat"], "model": "mistral:7b", "provider": "mistral_gen_0070", "size_mb": "{{ llm_size_coef * 4400 }}", "context": 32768 }
# https://ollama.com/library/mistral-small3.2 : Last Updated 2025-06-20
- { "purpose": ["chat", "lang"], "model": "mistral-small3.2:24b", "provider": "mistral_small32_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 } # rated best for Croatian
# https://ollama.com/library/ministral-3 : Last Updated 2025-12-13
- { "purpose": ["chat", "lang"], "model": "ministral-3:3b", "provider": "mistral_mini30_0030", "size_mb": "{{ llm_size_coef * 3000 }}", "context": 256000 }
- { "purpose": ["chat", "lang"], "model": "ministral-3:8b", "provider": "mistral_mini30_0080", "size_mb": "{{ llm_size_coef * 6000 }}", "context": 256000 }
- { "purpose": ["chat", "lang"], "model": "ministral-3:14b", "provider": "mistral_mini30_0140", "size_mb": "{{ llm_size_coef * 9100 }}", "context": 256000 } # also rated super high for Croatian, gives more head room.
# https://ollama.com/library/mistral-nemo : Last Updated 2025-07-23
# Supposed to be a drop-in replacement for mistral:7b.
- { "purpose": ["chat", "lang"], "model": "mistral-nemo:12b", "provider": "mistral_nemo_0120", "size_mb": "{{ llm_size_coef * 7100 }}", "context": 128000 } # says 1000K in table but 128k in desc, going with desc.
##### TBD #####
### QWEN 2.5 - Older, but does not have "thinking", tempting!
#- qwen2.5:0.5b
#- qwen2.5:1.5b
#- qwen2.5:3b
#- qwen2.5:7b
#- qwen2.5:14b
#- qwen2.5:32b
#- qwen2.5:72b
### Z.ai
#- GLM-5.x
#- glm4-flash
### Others
#- deepseek-r1:1.5b
#- deepseek-r1:7b
- name: Workstation | Linux | Software | LLMs | Facts | Merge | Print
ansible.builtin.debug:
var: "{{ item }}"
loop:
- ansible_processor_vcpus
- ansible_memtotal_mb
- llm_concurrency
- llm_parallel
- llm_size_coef
- llm_context
- llm_tokens
- llm_installs
- llm_removals
- llm_choices
- name: Workstation | Linux | Software | LLMs | Facts | Choices | Sort
ansible.builtin.set_fact:
llm_installs: >-
{{ llm_choices | selectattr('size_mb', 'le', ansible_memtotal_mb) | list }}
llm_removals: >-
{{ llm_choices | selectattr('size_mb', 'gt', ansible_memtotal_mb) | list }}
- name: Workstation | Linux | Software | LLMs | Facts | Merge | Print
ansible.builtin.debug:
var: "{{ item }}"
loop:
- llm_installs
- llm_removals
# Facts - End #
- name: Workstation | Linux | Software | LLMs | Packages | Install
ansible.builtin.package:
name:
- "{{ ollama }}"
- "{{ zeroclaw }}"
- "{{ zerocode }}"
- "{{ qwen_code }}"
state: present
- name: Workstation | Linux | Software | LLMs | Docker
ansible.builtin.package:
name:
- docker
state: present
- name: Workstation | Linux | Software | LLMs | Docker | runit
ansible.builtin.package:
name:
- docker-runit
state: present
when: ansible_pkg_mgr == "pacman"
- name: Workstation | Linux | Software | LLMs | Docker | Group
ansible.builtin.user:
name: "{{ item }}"
groups: docker
append: true
loop:
- root
- "{{ user }}"
- name: Workstation | Linux | Software | LLMs | runit
block:
# Ollama - Start #
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Run
ansible.builtin.file:
name: "{{ ollama_runit_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Run
ansible.builtin.file:
name: "{{ ollama_runit_log_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Config
ansible.builtin.file:
name: "{{ ollama_runit_log_dest }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Run
ansible.builtin.copy:
dest: "{{ ollama_runit_script }}"
content: |
#!/bin/sh -e
exec 2>&1
# Allow pulling and accessing LLMs via tools.
### Required ###
export HOME={{ user_user.home }}
### Timeouts ###
export OLLAMA_LOAD_TIMEOUT=20m # default is 5m
export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars)
### Resource Usage ###
export OLLAMA_KEEP_ALIVE=20m # default is 5m
export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs
export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed)
export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size
export OLLAMA_MAX_QUEUE=999 # default is 512 (unconfirmed)
### Prevent Data Loss ###
# OLLAMA_NOPRUNE
# Default is 0, which causes unfinished pulls to be deleted
# after 1 hour of inactivity. To manually clean partials, run:
# `rm ~/.ollama/models/blobs/*-partial*`
export OLLAMA_NOPRUNE=1
### Tokens ###
{{ ("export HF_TOKEN=" ~ hf_token) if hf_token else '' }}
### Logging ###
export OLLAMA_DEBUG=1
export OLLAMA_METRICS=1 # (check http://127.0.0.1:11434/metrics)
### Other ###
export OLLAMA_ORIGINS=*
exec chpst -u {{ user }} ollama serve
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ ollama }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log
ansible.builtin.copy:
dest: "{{ ollama_runit_log_script }}"
content: |
#!/bin/sh -e
exec svlogd -tt {{ ollama_runit_log_dest }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log Config
ansible.builtin.copy:
dest: "{{ ollama_runit_log_config }}"
content: |
s10000000 # rotate at 10 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
owner: "{{ user }}"
mode: "0755"
# Ollama - End #
# Zeroclaw - Start #
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Run
ansible.builtin.file:
name: "{{ zeroclaw_runit_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log
ansible.builtin.file:
name: "{{ zeroclaw_runit_log_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log Config
ansible.builtin.file:
name: "{{ zeroclaw_runit_log_dest }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | File
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_script }}"
content: |
#!/bin/sh -e
# Allow using the ZeroClaw frontend.
exec 2>&1
export HOME={{ user_user.home }}
export SHELL={{ bash_exec.stdout }}
exec chpst -u {{ user }} zeroclaw daemon -v --log-level info
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ zeroclaw }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_script }}"
content: |
#!/bin/sh -e
exec svlogd -tt {{ zeroclaw_runit_log_dest }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log Config
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_config }}"
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Zeroclaw - End #
# Open WebUI - Start #
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/open-webui/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /var/log
ansible.builtin.file:
name: "/var/log/open-webui"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Run
ansible.builtin.copy:
dest: /etc/sv/open-webui/run
content: |
#!/bin/sh
exec 2>&1
# Wait for Docker to start up.
while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone.
echo "RUN: Stopping..."
docker stop open-webui &&
echo "RUN: Container stopped" ||
echo "RUN: Container already stopped."
echo "RUN: Deleting..."
docker rm -f open-webui &&
echo "RUN: Container deleted" ||
echo "RUN: Container already deleted."
# Start the open-webui program.
set -e
exec docker run \
--pull=always
--name open-webui \
--network=host -e PORT={{ open_webui_port }} \
--add-host=host.docker.internal:host-gateway \
-e OLLAMA_BASE_URL=http://0.0.0.0:11434 \
{{ ("-e HF_TOKEN=" ~ hf_token) if hf_token else '' }} \
-e WEBUI_SECRET_KEY={{ open_webui_key }} \
-v open-webui:/app/backend/data \
ghcr.io/open-webui/open-webui:main
# Unsure why this was not working on Artix with runit.
# exec curl 0.0.0.0:8080/health gave a successful response.
# The environment was not accessible on the host though.
#-p {{ open_webui_port }}:8080 \
# The host.docker.internal params still seem required, even with
# giving access to the host system and trying this:
#-e OLLAMA_BASE_URL=http://localhost:11434 \
# Although I did not try it with 0.0.0.0, maybe that'd work.
# Had to change this to 0.0.0.0, below was still acting weird.
#-e OLLAMA_BASE_URL=http://host.docker.internal:11434 \
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/open-webui/finish
content: |
{{ runit_finish_sleep }}
echo "FINISH: Stopping..."
docker stop open-webui &&
echo "FINISH: Container stopped" ||
echo "FINISH: Container already stopped."
echo "FINISH: Deleting..."
docker rm -f open-webui &&
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
echo "FINISH: Done!"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log
ansible.builtin.copy:
dest: /etc/sv/open-webui/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/open-webui
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log Config
ansible.builtin.copy:
dest: /var/log/open-webui/config
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Open WebUI - End #
# Open Terminal - Start #
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/open-terminal/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /var/log
ansible.builtin.file:
name: "/var/log/open-terminal"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Run
ansible.builtin.copy:
dest: /etc/sv/open-terminal/run
content: |
#!/bin/sh
exec 2>&1
# Wait for Docker to start up.
while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone.
echo "RUN: Stopping..."
docker stop open-terminal &&
echo "RUN: Container stopped" ||
echo "RUN: Container already stopped."
echo "RUN: Deleting..."
docker rm -f open-terminal &&
echo "RUN: Container deleted" ||
echo "RUN: Container already deleted."
# Start the open-terminal program.
set -e
exec docker run \
--pull=always \
--name open-terminal \
--network=host -p {{ open_terminal_port }}:8000 \
-v open-terminal:/home/user \
-v {{ user_user.home }}/Code:/home/user/Code \
-e OPEN_TERMINAL_API_KEY={{ open_terminal_key }} \
ghcr.io/open-webui/open-terminal
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/open-terminal/finish
content: |
{{ runit_finish_sleep }}
echo "FINISH: Stopping..."
docker stop open-terminal &&
echo "FINISH: Container stopped" ||
echo "FINISH: Container already stopped."
echo "FINISH: Deleting..."
docker rm -f open-terminal &&
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
echo "FINISH: Done!"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Log
ansible.builtin.copy:
dest: /etc/sv/open-terminal/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/open-terminal
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Log Config
ansible.builtin.copy:
dest: /var/log/open-terminal/config
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Open Terminal - End #
# Swap Stopper - Start #
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Executable
ansible.builtin.copy:
dest: "{{ global_bin }}/llm_swap_stopper.sh"
src: files/scripts/llm_swap_stopper.sh
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/swap-stopper/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /var/log
ansible.builtin.file:
name: "/var/log/swap-stopper"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Run
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/run
content: |
#!/bin/sh -e
exec 2>&1
exec {{ global_bin }}/llm_swap_stopper.sh
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/swap-stopper
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log Config
ansible.builtin.copy:
dest: /var/log/swap-stopper/config
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Swap Saver - End #
# Qwen Serve - Start #
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/{{ qwen_code }}/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Dir | /var/log
ansible.builtin.file:
name: "/var/log/{{ qwen_code }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Run
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/run
content: |
#!/bin/sh -e
exec 2>&1
export QWEN_STREAM_IDLE_TIMEOUT_MS=6000000
{{ ('export QWEN_SERVER_TOKEN=' ~ qwen_code_key) if qwen_code_key else '' }}
exec chpst -u {{ user }} -C {{ user_user.home }}/Code qwen serve
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/{{ qwen_code }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log Config
ansible.builtin.copy:
dest: /var/log/{{ qwen_code }}/config
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Qwen Serve - End #
when: ansible_service_mgr == "runit"
# Services - Start #
- name: Workstation | Linux | Software | LLMs | Services | Enable
ansible.builtin.service:
name: "{{ item.name }}"
state: "{{ 'started' if item.status else 'stopped' }}"
enabled: "{{ item.status }}"
loop:
- { 'name': "{{ ollama }}" , 'status': "{{ ollama_enabled }}" }
- { 'name': "{{ zeroclaw }}" , 'status': "{{ zeroclaw_enabled }}" }
- { 'name': 'docker' , 'status': true }
- { 'name': 'open-webui' , 'status': "{{ open_webui_enabled }}" }
- { 'name': 'open-terminal' , 'status': "{{ open_terminal_enabled }}" }
- { 'name': 'swap-stopper' , 'status': true }
- { 'name': '{{ qwen_code }}', 'status': "{{ qwen_code_enabled }}" }
- name: Workstation | Linux | Software | LLMs | Services | Verify | Critical
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
timeout: 5
register: health
until: health.status == 200
retries: 12
delay: 5
loop:
- "{{ 'http://127.0.0.1:11434' if ollama_enabled else dummy_website }}"
- name: Workstation | Linux | Software | LLMs | Services | Verify | Okay to Fail
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
timeout: 5
register: health
until: health.status == 200
retries: 2
delay: 5
loop:
- "{{ 'http://127.0.0.1:42617/health' if zeroclaw_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:' ~ open_webui_port ~ '/health' if open_webui_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:' ~ open_terminal_port ~ '/health' if open_terminal_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:4170' if qwen_code_enabled else dummy_website }}"
ignore_errors: true
# Services - End #
# Models - Start #
- name: Workstation | Linux | Software | LLMs | Models | ollama | User Shell Functions
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/{{ item }}"
marker: "# {mark} ANSIBLE MANAGED BLOCK | ollama | Manual Script"
content: |
function list_llms() {
{% for llm in llm_installs %}
echo "{{ llm.model }}"
{% endfor %}
}
function pull_llms() {
echo "Manually Pulling Models"
for model in `list_llms`; do
echo -e "\n\n Starting 'ollama pull $model' \n\n"
ollama pull $model
status=$?
echo "Finished with status $status."
done
echo "Done!!"
}
alias pull-llms="pull_llms "
alias pull-models="pull_llms "
alias pull-ai="pull_llms "
function ollama_stop() {
for model in `ollama ps | tail -n +2 | awk '{print $1}'`; do
echo "Stopping ${model}..."
ollama stop $model
done
}
alias ollama-stop="ollama_stop"
alias ollama-kill="ollama_stop"
alias ollama-killall="ollama_stop"
alias ollama-kill-all="ollama_stop"
alias ollama-stop-all="ollama_stop"
function ollama_chat() {
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen3.5:4b' }}
}
alias ollama-chat="ollama_chat "
alias chat="ollama_chat "
function ollama_code() {
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen2.5-coder:7b' }}
}
alias ollama-code="ollama_code "
alias code="ollama_code "
function ollama_models() {
for model in `ollama ls | grep -v NAME | awk '{print $1;}'`; do
echo $model
ollama show $model | grep -E "architecture|context length|quantization"
done
}
alias ollama-models="ollama_models "
alias ollama-show-models="ollama_models "
alias ollama-show="ollama_models "
alias ollama-dense="ollama_models "
alias ollama-moe="ollama_models "
alias show-models="ollama_models "
alias list-models="ollama_models "
function ollama_purge {
echo "TBD: Purge all models not in llm_installs."
}
alias ollama-purge="ollama_purge "
alias ollama-clean="ollama_purge "
alias ollama-remove="ollama_purge "
alias ollama-rm="ollama_purge "
alias remove-models="ollama_purge "
alias rm-models="ollama_purge "
alias purge-models="ollama_purge "
state: present
create: false
backup: true
loop:
- .bashrc
- .zshrc
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage
block:
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Message
ansible.builtin.debug:
msg: <-
The next task may take hours, if not days. If you'd like a status
bar to know what's being worked on and its progress, I highly
recommed reloading your RC file and running `pull-models`. That
will give you a much better idea of what's going on.
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Install
ansible.builtin.shell: |
set -o pipefail
item='{{ item.model }}'
if ollama list | grep -q $item; then
echo "OKAY" &&
return
else
ollama pull $item 1>/dev/null &&
echo "INSTALLED" &&
return
fi
echo "FAILED"
loop: "{{ llm_installs }}"
register: shell_check
changed_when: "'INSTALLED' in shell_check.stdout"
failed_when: "'FAILED' in shell_check.stdout"
become: true
become_user: "{{ user }}"
when: manage_models
and llm_installs | length > 0
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Remove
ansible.builtin.shell: |
set -o pipefail
item='{{ item.model }}'
if ollama list | grep -q $item; then
ollama rm $item &&
echo "INSTALLED" &&
return
else
echo "OKAY" &&
return
fi
echo "FAILED"
loop: "{{ llm_removals }}"
register: shell_check
changed_when: "'INSTALLED' in shell_check.stdout"
failed_when: "'FAILED' in shell_check.stdout"
become: true
become_user: "{{ user }}"
when: llm_removals | length > 0
when: manage_models
- name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models
ansible.builtin.shell:
cmd: |
set -o pipefail
ollama ls | tail -n +2 | awk '{print $1}'
register: llm_models
changed_when: llm_models.rc == 0
- name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models | Result
ansible.builtin.debug:
var: llm_models
# Cheat Sheet: https://docs.zeroclaw.com/master/en/providers/configuration.html
- name: Workstation | Linux | Software | LLMs | Models | zeroclaw | Add Providers
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/.zeroclaw/config.toml"
marker: "# {mark} ANSIBLE MANAGED BLOCK | zeroclaw | Providers"
insertafter: EOF
block: |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "chat" in llm.purpose %}
[providers.models.ollama.{{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT'
}}{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}]
model = "{{ llm.model }}"
timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
context_window = {{ [llm.context] | min | int }}
max_tokens = {{ [(llm.context / 8), 16000] | min | int }}
{% endif %}
{% endif %}
{% endfor %}
state: present
create: false
backup: true
when: zeroclaw_enabled
- name: Workstation | Linux | Software | LLMs | Models | Continue | Config
ansible.builtin.copy:
dest: "{{ user_user.home }}/.continue/config.yaml"
content: |
# Maintained by `env-ansible/tasks/workstation/linux/software/llm.yml`
name: Main Config
version: 1.0.0
schema: v1
models:
# | Role | `contextLength` | `maxTokens` | Notes |
# |------|-----------------|-------------|-------|
# | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight |
# | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "code" in llm.purpose %}
- name: {{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT'
}}{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm.context] | min | int }}
maxTokens: {{ [(llm.context / 8), 16000] | min | int }}
- name: {{ llm.provider }}_AUTO{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm.context] | min | int }}
maxTokens: {{ [(llm.context / 8), 4000] | min | int }}
roles:
- autocomplete
{% endif %}
{% endif %}
{% endfor %}
mode: "0644"
when: codium_config
- name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config
ansible.builtin.copy:
dest: "{{ item }}"
content: |
{
"$version": 4,
"env": {
"OLLAMA_API_KEY": "ollama"
},
"model": {
"name": "qwen2.5-coder:3b",
"baseUrl": "http://127.0.0.1:11434/v1"
},
"modelProviders": {
"openai": [
{% for llm in llm_installs %}
{
"name": "{{ llm.provider
}}_{{ 'CODE' if "code" in llm.purpose else 'CHAT'
}}{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}"
,"baseUrl": "http://127.0.0.1:11434/v1"
,"envKey": "OLLAMA_API_KEY"
,"id": "{{ llm.model }}"
,"generationConfig": {
"timeout": 600000
,"streamIdleTimeoutMs": 600000
,"contextWindowSize": {{ [llm.context] | min | int }}
,"maxRetries": 3
}
}{{ ',' if not loop.last else '' }}
{% endfor %}
]
},
"security": {
"auth": {
"baseUrl": "http://127.0.0.1:11434/v1",
"selectedType": "openai"
}
},
"ui": {
"autoModeAcknowledged": true
}
}
mode: "0644"
loop:
- "{{ user_user.home }}/.qwen/settings.json"
# Not Needed
#- "{{ user_user.home }}/Code/.qwen/settings.json"
when: qwen_code_enabled
#"reasoning": {
# "effort": "medium"
#},
#"samplingParams": {
# "max_tokens": {{ [(llm.context / 8), 16000] | min | int }}
# ,"temperature": 0.7
# ,"top_p": 0.8
#}
- name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config | Validate
ansible.builtin.shell: |
set -o pipefail
cat {{ user_user.home }}/.qwen/settings.json | jq
register: shell_check
changed_when: shell_check.rc == 0
failed_when: shell_check.rc != 0
when: qwen_code_enabled
# Models - End #
when: ansible_pkg_mgr == "pacman"
Reference in New Issue View Git Blame Copy Permalink