me/env-ansible
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
2a7fa98825f84bc9f8019f40bec129cd94bdfb49
env-ansible/tasks/workstation/linux/software/llm.yml
T

1162 lines
50 KiB
YAML
Raw Blame History

---
# 2026-10-05 Hyperling
# Install a local LLM for use offline.
# - https://zeroclaw.net/blog/install-zeroclaw
# - https://zeroclaw.net/blog/set-up-zeroclaw-bot-ollama-telegram
#
# Packages Details
# - https://archlinux.org/packages/extra/x86_64/zeroclaw/
# - https://archlinux.org/packages/extra/x86_64/ollama/
#
# Individual Guides
# - https://github.com/zeroclaw-labs/zeroclaw
# - https://wiki.archlinux.org/title/Ollama
#
# Model Explanations
# - https://webscraft.org/blog/ollama-na-8-gb-ram-yaki-modeli-pratsyuyut-u-2026?lang=en
#
# Models
# - https://ollama.com/search
#
# References
# - https://docs.zeroclaw.com/master/en/reference/config.html
# - https://docs.ollama.com
# - https://docs.ollama.com/faq#setting-environment-variables-on-linux
# - https://modelpiper.com/blog/ollama-environment-variables
# - https://qwenlm.github.io/qwen-code-docs/en/users/configuration/settings/#environment-variables--env-files
# - https://qwenlm.github.io/qwen-code-docs/en/users/qwen-serve/
# Currently only testing this on Arch based distros (pacman).
- name: Workstation | Linux | Software | LLMs
block:
# Facts - Start #
- name: Workstation | Linux | Software | LLMs | Facts | Provisioned
ansible.builtin.debug:
var: "{{ item }}"
loop:
- ollama_enabled
- manage_models
- hf_token
- zeroclaw_enabled
- open_webui_enabled
- open_webui_port
- open_webui_key
- codium_config
- qwen_code_enabled
- qwen_code_key
- open_terminal_enabled
- open_terminal_port
- open_terminal_key
- name: Workstation | Linux | Software | LLMs | Facts 01
ansible.builtin.set_fact:
ollama: ollama
zeroclaw: zeroclaw
zerocode: zerocode
qwen_code: qwen-code
dummy_service: 'cronie'
dummy_website: 'https://hyperling.com'
# Good to keep low to prevent swap usage. Going to have a minimum of 3
# though, one for Cline, two for Continue (Agent + Autocomplete).
# 20261014: Going down to a static of 3 since we're focusing on dense.
llm_concurrency: 3 #"{{ [(ansible_memtotal_mb / 32000) | round(0, method='ceil'), 3] | max | int }}"
# Allow having multiple chats open without them just waiting each other.
# Also setting this to a minimum of 3 for using Codium with extensions.
# 20261014: Going down to 1 since only using CPUs, no GPUs. Otherwise
# the models seem to take excessively long to answer even small things.
llm_parallel: 1 #"{{ [(ansible_processor_vcpus / 2) | round(0, method='ceil'), 3] | max | int }}"
# Most LLMs take a coefficient of about 1.2 to 1.5 of their file size in
# RAM, which is usually less than guessing a usage of weight*1000.
# Example:
# - qwen3.5:9b would be 6.6GB * 1.3 = 8580, rather than 9000.
# Conversely:
# - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
llm_size_coef: 1.3
# Base the context on MB of RAM, with a minumum 64k for decent thinking.
llm_context: "{{ [(ansible_memtotal_mb * 4) | round(-3, method='ceil'), 64000] | max | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 02
ansible.builtin.set_fact:
ollama_runit_dir: "{{ runit_script_dir }}/{{ ollama }}"
ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}"
zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}"
zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}"
# Amount of output tokens allowed per response. Half of context is ok.
llm_tokens: "{{ (llm_context / 8) | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 03
ansible.builtin.set_fact:
ollama_runit_script: "{{ ollama_runit_dir }}/{{ runit_service_script }}"
ollama_runit_log_dir: "{{ ollama_runit_dir }}/{{ runit_log_dir }}"
ollama_runit_log_config: "{{ ollama_runit_log_dest }}/{{ global_log_config }}"
zeroclaw_runit_script: "{{ zeroclaw_runit_dir }}/{{ runit_service_script }}"
zeroclaw_runit_log_dir: "{{ zeroclaw_runit_dir }}/{{ runit_log_dir }}"
zeroclaw_runit_log_config: "{{ zeroclaw_runit_log_dest }}/{{ global_log_config }}"
- name: Workstation | Linux | Software | LLMs | Facts 04
ansible.builtin.set_fact:
ollama_runit_log_script: "{{ ollama_runit_log_dir }}/{{ runit_log_script }}"
zeroclaw_runit_log_script: "{{ zeroclaw_runit_log_dir }}/{{ runit_log_script }}"
- name: Workstation | Linux | Software | LLMs | Facts | LLMs
ansible.builtin.set_fact:
llm_installs: []
llm_removals:
##### QWEN #####
- { "purpose": ["chat"], "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded
##### Mistral #####
- { "purpose": ["code"], "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded
# https://ollama.com/library/mistral-small : Last Updated 2025-01-30, succeeded
- { "purpose": ["chat"], "model": "mistral-small:24b", "provider": "mistral_small30_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 32000 }
- { "purpose": ["chat"], "model": "mistral-small:22b", "provider": "mistral_small30_0220", "size_mb": "{{ llm_size_coef * 13000 }}", "context": 128000 } # Yes, the smaller one has larger context.
# https://ollama.com/library/mistral-small3.1 : Last Update 2025-04-07, succeeded
- { "purpose": ["chat"], "model": "mistral-small3.1:24b", "provider": "mistral_small31_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 }
llm_choices:
##### QWEN #####
### QWEN >3.5 - Need to try!
# Supposedly great for coding over any previous version.
# Prefer 27b (dense) over anything higher which likely uses MoE (essentially only uses 3b inside of a 35b, dumber but faster and potentially has access to more info).
# https://ollama.com/library/qwen3.8 : Last Updated 2026-09-25
# Known to overthink, suggested to prefer 3.6:27b unless setting temp to low/medium.
- { "purpose": ["code", "chat", "lang"], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
# https://ollama.com/library/qwen3.6 : Last Updated 2026-10-13 (yesterday as of this commit!!)
# Has been succeeded but dense and does not overthink so still worthy
- { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
- { "purpose": ["chat", "code", "lang"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended
# https://ollama.com/library/qwen3.5 : Last Updated 2026-10-13 (yesterday as of this commit!!)
### QWEN 3.5 - Good for generic usage and a little coding.
- { "purpose": ["chat"], "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 262144 }
- { "purpose": ["chat"], "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 262144 }
- { "purpose": ["chat"], "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 262144 }
- { "purpose": ["chat"], "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 262144 }
- { "purpose": ["chat", "code"], "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 262144 } # succeeded
- { "purpose": ["chat", "code"], "model": "qwen3.5:35b", "provider": "qwen_35_0350_moe", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 262144 }
- { "purpose": ["chat", "code"], "model": "qwen3.5:122b", "provider": "qwen_35_1220_moe", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 262144 }
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
- { "purpose": ["code", "chat"], "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300_moe", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 262144 }
- { "purpose": ["code", "chat"], "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200_moe", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 262144 }
### QWEN 2.5 Coder - Ol'reliable!
- { "purpose": ["code"], "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32768 }
- { "purpose": ["code"], "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32768 }
##### Mistral, Ministral, and Devstral - By a French company. #####
# mistral-small-4
# https://huggingface.co/unsloth/Mistral-Small-4-119B-2603-GGUF
# Supposed to be better than Qwen equivalents, it's basically mistral, mixtral, magistral, and devstral all in one.
# unsloth version is supposed to be the best quantization. UD-Q3_K_XL is supposed to take 64GB RAM, lol, while Q4_K_M would need 80+ GB.
# needs done via 'ollama create', fails with pull. https://github.com/ollama/ollama/issues/5245
- { "purpose": ["chat", "lang", "code"], "model": "hf.co/unsloth/Mistral-Small-4-119B-2603-GGUF:UD-Q3_K_XL", "provider": "mistral_small40_1190_unsloth_q3", "size_mb": "{{ llm_size_coef * 54800 }}", "context": 256000 }
# https://huggingface.co/mistralai/Mistral-Small-4-119B-2603 : 242GB unquantized, lol.
- { "purpose": ["chat", "lang", "code"], "model": "hf.co/mistralai/Mistral-Small-4-119B-2603", "provider": "mistral_small40_1190", "size_mb": "{{ llm_size_coef * 242000 }}", "context": 256000 }
# https://huggingface.co/unsloth/Mistral-Small-4-119B-2603 : 121GB, A6B MoE, hmmm!
- { "purpose": ["chat", "lang", "code"], "model": "hf.co/unsloth/Mistral-Small-4-119B-2603", "provider": "mistral_small40_1190_unsloth_a6b", "size_mb": "{{ llm_size_coef * 121000 }}", "context": 256000 }
# https://ollama.com/library/devstral-small-2 : Last Updated 2025-12-13
# Most recommended in Mistral family for agentic coding.
- { "purpose": ["code"], "model": "devstral-small-2:24b", "provider": "mistral_dev20_0240_small", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 393216 }
# https://ollama.com/library/codestral : Last Updated 2024-09-03
# Mistral's first ever coder agent, recommended for autocomplete (FIM).
- { "purpose": ["code"], "model": "codestral:22b", "provider": "mistral_code_0220", "size_mb": "{{ llm_size_coef * 12000 }}", "context": 32768 }
# https://ollama.com/library/mistral : Last Updated 2025-07-13
- { "purpose": ["chat"], "model": "mistral:7b", "provider": "mistral_gen_0070", "size_mb": "{{ llm_size_coef * 4400 }}", "context": 32768 }
# https://ollama.com/library/mistral-small3.2 : Last Updated 2025-06-20
- { "purpose": ["chat", "lang"], "model": "mistral-small3.2:24b", "provider": "mistral_small32_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 } # rated best for Croatian
# https://ollama.com/library/ministral-3 : Last Updated 2025-12-13
- { "purpose": ["chat", "lang"], "model": "ministral-3:3b", "provider": "mistral_mini30_0030", "size_mb": "{{ llm_size_coef * 3000 }}", "context": 256000 }
- { "purpose": ["chat", "lang"], "model": "ministral-3:8b", "provider": "mistral_mini30_0080", "size_mb": "{{ llm_size_coef * 6000 }}", "context": 256000 }
- { "purpose": ["chat", "lang"], "model": "ministral-3:14b", "provider": "mistral_mini30_0140", "size_mb": "{{ llm_size_coef * 9100 }}", "context": 256000 } # also rated super high for Croatian, gives more head room.
# https://ollama.com/library/mistral-nemo : Last Updated 2025-07-23
# Supposed to be a drop-in replacement for mistral:7b.
- { "purpose": ["chat", "lang"], "model": "mistral-nemo:12b", "provider": "mistral_nemo_0120", "size_mb": "{{ llm_size_coef * 7100 }}", "context": 128000 } # says 1000K in table but 128k in desc, going with desc.
##### TBD #####
### QWEN 2.5 - Older, but does not have "thinking", tempting!
#- qwen2.5:0.5b
#- qwen2.5:1.5b
#- qwen2.5:3b
#- qwen2.5:7b
#- qwen2.5:14b
#- qwen2.5:32b
#- qwen2.5:72b
### Z.ai
#- GLM-5.x
#- glm4-flash
### Others
#- deepseek-r1:1.5b
#- deepseek-r1:7b
- name: Workstation | Linux | Software | LLMs | Facts | Merge | Print
ansible.builtin.debug:
var: "{{ item }}"
loop:
- ansible_processor_vcpus
- ansible_memtotal_mb
- llm_concurrency
- llm_parallel
- llm_size_coef
- llm_context
- llm_tokens
- llm_installs
- llm_removals
- llm_choices
- name: Workstation | Linux | Software | LLMs | Facts | Choices | Sort
ansible.builtin.set_fact:
llm_installs: >-
{{ llm_choices | selectattr('size_mb', 'le', ansible_memtotal_mb) | list }}
llm_removals: >-
{{ llm_choices | selectattr('size_mb', 'gt', ansible_memtotal_mb) | list }}
- name: Workstation | Linux | Software | LLMs | Facts | Merge | Print
ansible.builtin.debug:
var: "{{ item }}"
loop:
- llm_installs
- llm_removals
# Facts - End #
- name: Workstation | Linux | Software | LLMs | Packages | Install
ansible.builtin.package:
name:
- "{{ ollama }}"
- "{{ zeroclaw }}"
- "{{ zerocode }}"
- "{{ qwen_code }}"
state: present
- name: Workstation | Linux | Software | LLMs | Docker
ansible.builtin.package:
name:
- docker
state: present
- name: Workstation | Linux | Software | LLMs | Docker | runit
ansible.builtin.package:
name:
- docker-runit
state: present
when: ansible_pkg_mgr == "pacman"
- name: Workstation | Linux | Software | LLMs | Docker | Group
ansible.builtin.user:
name: "{{ item }}"
groups: docker
append: true
loop:
- root
- "{{ user }}"
- name: Workstation | Linux | Software | LLMs | runit
block:
# Ollama - Start #
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Run
ansible.builtin.file:
name: "{{ ollama_runit_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Run
ansible.builtin.file:
name: "{{ ollama_runit_log_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Config
ansible.builtin.file:
name: "{{ ollama_runit_log_dest }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Run
ansible.builtin.copy:
dest: "{{ ollama_runit_script }}"
content: |
#!/bin/sh -e
exec 2>&1
# Allow pulling and accessing LLMs via tools.
### Required ###
export HOME={{ user_user.home }}
### Timeouts ###
export OLLAMA_LOAD_TIMEOUT=30m # default is 5m
export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars)
### Resource Usage ###
export OLLAMA_KEEP_ALIVE=2h # default is 5m. Good frontends keep the model alive as long as it needs to be, this is not a source of timeout.
export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs
export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed)
export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size
export OLLAMA_MAX_QUEUE=999 # default is 512 (unconfirmed)
export OLLAMA_NUM_THREADS={{ (ansible_processor_vcpus / 2) | int }} # not real, but maybe some day.
### Prevent Data Loss ###
# OLLAMA_NOPRUNE
# Default is 0, which causes unfinished pulls to be deleted
# after 1 hour of inactivity. To manually clean partials, run:
# `rm ~/.ollama/models/blobs/*-partial*`
export OLLAMA_NOPRUNE=1
### Tokens ###
{{ ("export HF_TOKEN=" ~ hf_token) if hf_token else '' }}
### Logging ###
#export OLLAMA_DEBUG=1
export OLLAMA_METRICS=1 # (check http://127.0.0.1:11434/metrics)
### Other ###
export OLLAMA_ORIGINS=*
exec chpst -u {{ user }} ollama serve
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ ollama }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log
ansible.builtin.copy:
dest: "{{ ollama_runit_log_script }}"
content: |
#!/bin/sh -e
exec svlogd -tt {{ ollama_runit_log_dest }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log Config
ansible.builtin.copy:
dest: "{{ ollama_runit_log_config }}"
content: |
{{ runit_log_config_10mb }}
owner: "{{ user }}"
mode: "0755"
# Ollama - End #
# Zeroclaw - Start #
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Run
ansible.builtin.file:
name: "{{ zeroclaw_runit_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log
ansible.builtin.file:
name: "{{ zeroclaw_runit_log_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log Config
ansible.builtin.file:
name: "{{ zeroclaw_runit_log_dest }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | File
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_script }}"
content: |
#!/bin/sh -e
# Allow using the ZeroClaw frontend.
exec 2>&1
export HOME={{ user_user.home }}
export SHELL={{ bash_exec.stdout }}
exec chpst -u {{ user }} zeroclaw daemon -v --log-level info
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ zeroclaw }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_script }}"
content: |
#!/bin/sh -e
exec svlogd -tt {{ zeroclaw_runit_log_dest }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log Config
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_config }}"
content: |
{{ runit_log_config_10mb }}
mode: "0755"
# Zeroclaw - End #
# Open WebUI - Start #
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/open-webui/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /var/log
ansible.builtin.file:
name: "/var/log/open-webui"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Run
ansible.builtin.copy:
dest: /etc/sv/open-webui/run
content: |
#!/bin/sh
exec 2>&1
image=ghcr.io/open-webui/open-webui:latest
# Wait for Docker to start up.
while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone.
echo "RUN: Stopping..."
docker stop open-webui &&
echo "RUN: Container stopped" ||
echo "RUN: Container already stopped."
echo "RUN: Deleting..."
docker rm -f open-webui &&
echo "RUN: Container deleted" ||
echo "RUN: Container already deleted."
# Check for any new updates in background.
echo "RUN: Updating..."
docker pull $image &
# Start the open-webui program.
set -e
exec docker run \
--name open-webui \
--network=host -e PORT={{ open_webui_port }} \
-e OLLAMA_BASE_URL=http://0.0.0.0:11434 \
{{ ("-e HF_TOKEN=" ~ hf_token) if hf_token else '' }} \
-e WEBUI_SECRET_KEY={{ open_webui_key }} \
-v open-webui:/app/backend/data \
$image
# Unsure why this was not working on Artix with runit.
# exec curl 0.0.0.0:8080/health gave a successful response.
# The environment was not accessible on the host though.
#-p {{ open_webui_port }}:8080 \
# The host.docker.internal params still seem required, even with
# giving access to the host system and trying this:
#-e OLLAMA_BASE_URL=http://localhost:11434 \
# Although I did not try it with 0.0.0.0, maybe that'd work.
# Had to change this to 0.0.0.0, below was still acting weird.
#-e OLLAMA_BASE_URL=http://host.docker.internal:11434 \
# 20261020, trying to remove this again while leaving the rest.
#--add-host=host.docker.internal:host-gateway \
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/open-webui/finish
content: |
{{ runit_finish_sleep }}
echo "FINISH: Stopping..."
docker stop open-webui &&
echo "FINISH: Container stopped" ||
echo "FINISH: Container already stopped."
echo "FINISH: Deleting..."
docker rm -f open-webui &&
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
echo "FINISH: Done!"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log
ansible.builtin.copy:
dest: /etc/sv/open-webui/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/open-webui
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log Config
ansible.builtin.copy:
dest: /var/log/open-webui/config
content: |
{{ runit_log_config_100mb }}
mode: "0755"
# Open WebUI - End #
# Open Terminal - Start #
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/open-terminal/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /var/log
ansible.builtin.file:
name: "/var/log/open-terminal"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | AI-Sandbox | Dir
ansible.builtin.file:
dest: "{{ user_user.home }}/AI-Sandbox"
state: directory
owner: "{{ user }}"
group: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | AI-Sandbox | Env
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/AI-Sandbox/{{ item }}"
marker: "# {mark} | MANAGE BY ANSIBLE | Env Vars"
content: |
if [[ -d "$HOME/bin" ]]; then
export PATH="$HOME/bin:$PATH"
fi
export ANDROID_HOME="{{ user_user.home }}/SDKs/Android/Sdk"
export GRADLE_USER_HOME={{ user_user.home }}/.gradle
export GRADLE_HOME={{ user_user.home }}/.gradle
export PATH="$ANDROID_HOME/platform-tools:$PATH"
export PATH="$ANDROID_HOME/cmdline-tools/latest/bin:$PATH"
export FLUTTER_HOME="{{ user_user.home }}/SDKs/Flutter/flutter"
export PUB_CACHE={{ user_user.home }}/.pub_cache
export PATH="$FLUTTER_HOME/bin:$PATH"
export JAVA_HOME="/{{ user_user.home }}/SDKs/Java"
export PATH="$JAVA_HOME/bin:$PATH"
create: false
backup: false
owner: "{{ user }}"
group: "{{ user }}"
mode: "0755"
loop:
- ".bashrc"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | AI-Sandbox | Defaults
ansible.builtin.set_fact:
jdk_jvm: "{{ jdk_jvm | default('/usr/lib/jvm') }}"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | AI-Sandbox | AGENTS.md
ansible.builtin.copy:
dest: "{{ user_user.home }}/AI-Sandbox/AGENTS.md"
content: |
# Environment
- You are running in a Docker container called Open Terminal. We
are communicating via Open WebUI, also being run in a Docker
container. You are the user `user` and have been granted access to
the designer's Code and SDKs via mount at `{{ user_user.home }}`.
# Rules
Never use any `git` subcommands that change anything, including
but not limited to `add`, `commit`, `pull`, `push`, `branch`,
`switch`, etc. The user is who controls the respository changes.
You may informational git subcommands such as use `git status`.
If any programs need installed in order to work on a project, we
need to document that in an install.sh script, as well as add it
to ~/PACKAGES.txt so that the user can ensure they stay installed.
install.sh scripts should be for both `apt` systems (Debian,
Ubuntu) and pacman system (Arch, Artix). using distribution
package managers is preferable over `pip` and `npm` when
possible, unless the project is actually Python or Node.js.
The user manages the `{{ user_user.home }}/SDKs` directory. Do not
modify it. Instead, ask if anything needs updated or added.
Never manually edit, modify, delete, alter, replace, etc any
files under system folders such as `/usr`. Anything under `/` is
managed by the system. If you need assistance changing anything
such as `/usr/lib/jvm`, please ask the user and halt all further
operations until you have received feedback.
It is never okay to brute force a solution if something is not
working well. it is better to ask the user for assistance or alert
them that you've encountered blockers and then cease the process
until you've received feedback.
# Preferences
You have free reign over your home directory. Feel free to
create memory files or anything else which would benefit you.
Everything we create should have both Desktop Linux and Android
in mind. Windows, MacOS, and iOS will never be compiled for, but
if we can do things in a way which makes it easy for others to
fork the project and add the support themselves, then that's best.
# Versions
These are the versions of plugins and languages we need to use.
If you know of an important upgrade from them, please let me know.
## Android / Flutter
- Gradle: 9.8.0
- AGP: 9.4.1
- Kotlin: 2.4.20
- Java: JDK 17
- Flutter (plugin loader): 1.0.0
backup: false
owner: "{{ user }}"
group: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Run
ansible.builtin.copy:
dest: /etc/sv/open-terminal/run
content: |
#!/bin/sh
exec 2>&1
image=ghcr.io/open-webui/open-terminal:latest
packages=""
# Wait for Docker to start up.
while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone.
echo "RUN: Stopping..."
docker stop open-terminal &&
echo "RUN: Container stopped" ||
echo "RUN: Container already stopped."
echo "RUN: Deleting..."
docker rm -f open-terminal &&
echo "RUN: Container deleted" ||
echo "RUN: Container already deleted."
# Check for any new updates in background.
echo "RUN: Updating..."
docker pull $image &
# This takes too long. Mount JDK from the host instead.
#if curl -sf --max-time 5 -o /dev/null https://hyperling.com; then
# # Internet is available, pull extra packages. useful for having
# # model compile Android apps, need to verify for Flutter.
# packages="default-jdk xz-utils libglu1-mesa libglu1-mesa ninja-build libnotify-bin libgtk-3-dev liblzma-dev libstdc++-12-dev libc6:amd64 libstdc++6 lib32z1 libbz2-1.0 libncurses-dev android-udev-rules"
#fi
# Start the open-terminal program.
# https://github.com/open-webui/open-terminal#configuration
set -e
exec docker run \
--name open-terminal \
--network=host \
-v {{ user_user.home }}/AI-Sandbox:/home/user \
-v {{ user_user.home }}/Code:{{ user_user.home }}/Code \
-v {{ user_user.home }}/SDKs:{{ user_user.home }}/SDKs \
-v {{ jdk_jvm }}:{{ jdk_jvm }}:ro \
-v {{ user_user.home }}/.gradle:{{ user_user.home }}/.gradle \
-v {{ user_user.home }}/.pub_cache:{{ user_user.home }}/.pub_cache \
-e OPEN_TERMINAL_FILE_BROWSER_ROOT="/home/user" \
-e OPEN_TERMINAL_API_KEY="{{ open_terminal_key }}" \
-e OPEN_TERMINAL_PACKAGES="$packages" \
$image run --port {{ open_terminal_port }}
# -p {{ open_terminal_port }}:8000
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/open-terminal/finish
content: |
{{ runit_finish_sleep }}
echo "FINISH: Stopping..."
docker stop open-terminal &&
echo "FINISH: Container stopped" ||
echo "FINISH: Container already stopped."
echo "FINISH: Deleting..."
docker rm -f open-terminal &&
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
echo "FINISH: Done!"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Log
ansible.builtin.copy:
dest: /etc/sv/open-terminal/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/open-terminal
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Log Config
ansible.builtin.copy:
dest: /var/log/open-terminal/config
content: |
{{ runit_log_config_100mb }}
mode: "0755"
# Open Terminal - End #
# Swap Stopper - Start #
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Executable
ansible.builtin.copy:
dest: "{{ global_bin }}/llm_swap_stopper.sh"
src: files/scripts/llm_swap_stopper.sh
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/swap-stopper/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /var/log
ansible.builtin.file:
name: "/var/log/swap-stopper"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Run
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/run
content: |
#!/bin/sh -e
exec 2>&1
exec {{ global_bin }}/llm_swap_stopper.sh
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/swap-stopper
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log Config
ansible.builtin.copy:
dest: /var/log/swap-stopper/config
content: |
{{ runit_log_config_1mb }}
mode: "0755"
# Swap Saver - End #
# Qwen Serve - Start #
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/{{ qwen_code }}/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Dir | /var/log
ansible.builtin.file:
name: "/var/log/{{ qwen_code }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Run
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/run
content: |
#!/bin/sh -e
exec 2>&1
export QWEN_STREAM_IDLE_TIMEOUT_MS=6000000
{{ ('export QWEN_SERVER_TOKEN=' ~ qwen_code_key) if qwen_code_key else '' }}
exec chpst -u {{ user }} -C {{ user_user.home }}/Code qwen serve
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/finish
content: |
{{ runit_finish_sleep }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log
ansible.builtin.copy:
dest: /etc/sv/{{ qwen_code }}/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/{{ qwen_code }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log Config
ansible.builtin.copy:
dest: /var/log/{{ qwen_code }}/config
content: |
{{ runit_log_config_10mb }}
mode: "0755"
# Qwen Serve - End #
when: ansible_service_mgr == "runit"
# Services - Start #
- name: Workstation | Linux | Software | LLMs | Services | Enable
ansible.builtin.service:
name: "{{ item.name }}"
state: "{{ 'started' if item.status else 'stopped' }}"
enabled: "{{ item.status }}"
loop:
- { 'name': "{{ ollama }}" , 'status': "{{ ollama_enabled }}" }
- { 'name': "{{ zeroclaw }}" , 'status': "{{ zeroclaw_enabled }}" }
- { 'name': 'docker' , 'status': true }
- { 'name': 'open-webui' , 'status': "{{ open_webui_enabled }}" }
- { 'name': 'open-terminal' , 'status': "{{ open_terminal_enabled }}" }
- { 'name': 'swap-stopper' , 'status': true }
- { 'name': '{{ qwen_code }}', 'status': "{{ qwen_code_enabled }}" }
- name: Workstation | Linux | Software | LLMs | Services | Verify | Critical
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
timeout: 5
register: health
until: health.status == 200
retries: 12
delay: 5
loop:
- "{{ 'http://127.0.0.1:11434' if ollama_enabled else dummy_website }}"
- name: Workstation | Linux | Software | LLMs | Services | Verify | Okay to Fail
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
timeout: 5
register: health
until: health.status == 200
retries: 2
delay: 5
loop:
- "{{ 'http://127.0.0.1:42617/health' if zeroclaw_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:' ~ open_webui_port ~ '/health' if open_webui_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:' ~ open_terminal_port ~ '/health' if open_terminal_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:4170' if qwen_code_enabled else dummy_website }}"
ignore_errors: true
# Services - End #
# Models - Start #
- name: Workstation | Linux | Software | LLMs | Models | ollama | User Shell Functions
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/{{ item }}"
marker: "# {mark} ANSIBLE MANAGED BLOCK | ollama | Manual Script"
content: |
function ollama_llms() {
{% for llm in llm_installs %}
echo "{{ llm.model }}"
{% endfor %}
}
function ollama_pull() {
echo "Manually Pulling Models"
for model in `ollama_llms`; do
echo -e "\n\n Starting 'ollama pull $model' \n\n"
ollama pull $model
status=$?
echo "Finished with status $status."
done
echo "Done!!"
}
alias pull-llms="ollama_pull "
alias pull-models="ollama_pull "
alias pull-ai="ollama_pull "
function ollama_chat() {
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen3.5:4b' }}
}
alias ollama-chat="ollama_chat "
alias chat="ollama_chat "
function ollama_code() {
ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen2.5-coder:7b' }}
}
alias ollama-code="ollama_code "
function ollama_purge() {
echo "TBD: Purge all models not in llm_installs."
}
alias ollama-purge="ollama_purge "
alias ollama-clean="ollama_purge "
alias ollama-remove="ollama_purge "
alias ollama-rm="ollama_purge "
alias remove-models="ollama_purge "
alias rm-models="ollama_purge "
alias purge-models="ollama_purge "
function restart_ai() {
echo "Restarting AI Software"
for sv in ollama zeroclaw open-webui open-terminal qwen-code swap-stopper; do
echo -e "\n $sv \n"
sudo sv restart $sv
done;
}
alias restart-ai="restart_ai "
alias restart-llm="restart_ai "
alias restart-llms="restart_ai "
alias restart-models="restart_ai "
alias reload-ai="restart_ai "
state: present
create: false
backup: true
loop:
- .bashrc
- .zshrc
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage
block:
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Message
ansible.builtin.debug:
msg: <-
The next task may take hours, if not days. If you'd like a status
bar to know what's being worked on and its progress, I highly
recommed reloading your RC file and running `pull-models`. That
will give you a much better idea of what's going on.
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Install
ansible.builtin.shell: |
set -o pipefail
item='{{ item.model }}'
if ollama list | grep -q $item; then
echo "OKAY" &&
return
else
ollama pull $item 1>/dev/null &&
echo "INSTALLED" &&
return
fi
echo "FAILED"
loop: "{{ llm_installs }}"
register: shell_check
changed_when: "'INSTALLED' in shell_check.stdout"
failed_when: "'FAILED' in shell_check.stdout"
become: true
become_user: "{{ user }}"
when: manage_models
and llm_installs | length > 0
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Remove
ansible.builtin.shell: |
set -o pipefail
item='{{ item.model }}'
if ollama list | grep -q $item; then
ollama rm $item &&
echo "INSTALLED" &&
return
else
echo "OKAY" &&
return
fi
echo "FAILED"
loop: "{{ llm_removals }}"
register: shell_check
changed_when: "'INSTALLED' in shell_check.stdout"
failed_when: "'FAILED' in shell_check.stdout"
become: true
become_user: "{{ user }}"
when: llm_removals | length > 0
when: manage_models
- name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models
ansible.builtin.shell:
cmd: |
set -o pipefail
ollama ls | tail -n +2 | awk '{print $1}'
register: llm_models
changed_when: llm_models.rc == 0
- name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models | Result
ansible.builtin.debug:
var: llm_models
# Cheat Sheet: https://docs.zeroclaw.com/master/en/providers/configuration.html
- name: Workstation | Linux | Software | LLMs | Models | zeroclaw | Add Providers
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/.zeroclaw/config.toml"
marker: "# {mark} ANSIBLE MANAGED BLOCK | zeroclaw | Providers"
insertafter: EOF
block: |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "chat" in llm.purpose %}
[providers.models.ollama.{{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT'
}}{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}]
model = "{{ llm.model }}"
timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
context_window = {{ [llm.context] | min | int }}
max_tokens = {{ [(llm.context / 8), 16000] | min | int }}
{% endif %}
{% endif %}
{% endfor %}
state: present
create: false
backup: true
when: zeroclaw_enabled
- name: Workstation | Linux | Software | LLMs | Models | Continue | Config
ansible.builtin.copy:
dest: "{{ user_user.home }}/.continue/config.yaml"
content: |
# Maintained by `env-ansible/tasks/workstation/linux/software/llm.yml`
name: Main Config
version: 1.0.0
schema: v1
models:
# | Role | `contextLength` | `maxTokens` | Notes |
# |------|-----------------|-------------|-------|
# | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight |
# | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "code" in llm.purpose %}
- name: {{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT'
}}{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm.context] | min | int }}
maxTokens: {{ [(llm.context / 8), 16000] | min | int }}
{% endif %}
{% endif %}
{% endfor %}
mode: "0644"
when: codium_config
#- name: {{ llm.provider }}_AUTO{{ '' if llm.model in llm_models.stdout else '-MISSING'
# }}
# provider: ollama
# model: {{ llm.model }}
# defaultCompletionOptions:
# contextLength: {{ [llm.context] | min | int }}
# maxTokens: {{ [(llm.context / 8), 4000] | min | int }}
# roles:
# - autocomplete
- name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config
ansible.builtin.copy:
dest: "{{ item }}"
content: |
{
"$version": 4,
"env": {
"OLLAMA_API_KEY": "ollama"
},
"model": {
"name": "qwen2.5-coder:3b",
"baseUrl": "http://127.0.0.1:11434/v1"
},
"modelProviders": {
"openai": [
{% for llm in llm_installs %}
{
"name": "{{ llm.provider
}}_{{ 'CODE' if "code" in llm.purpose else 'CHAT'
}}{{ '' if llm.model in llm_models.stdout else '-MISSING'
}}"
,"baseUrl": "http://127.0.0.1:11434/v1"
,"envKey": "OLLAMA_API_KEY"
,"id": "{{ llm.model }}"
,"generationConfig": {
"timeout": 600000
,"streamIdleTimeoutMs": 600000
,"contextWindowSize": {{ [llm.context] | min | int }}
,"maxRetries": 3
}
}{{ ',' if not loop.last else '' }}
{% endfor %}
]
},
"security": {
"auth": {
"baseUrl": "http://127.0.0.1:11434/v1",
"selectedType": "openai"
}
},
"ui": {
"autoModeAcknowledged": true
}
}
mode: "0644"
loop:
- "{{ user_user.home }}/.qwen/settings.json"
# Not Needed
#- "{{ user_user.home }}/Code/.qwen/settings.json"
when: qwen_code_enabled
#"reasoning": {
# "effort": "medium"
#},
#"samplingParams": {
# "max_tokens": {{ [(llm.context / 8), 16000] | min | int }}
# ,"temperature": 0.7
# ,"top_p": 0.8
#}
- name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config | Validate
ansible.builtin.shell: |
set -o pipefail
cat {{ user_user.home }}/.qwen/settings.json | jq
register: shell_check
changed_when: shell_check.rc == 0
failed_when: shell_check.rc != 0
when: qwen_code_enabled
# Models - End #
when: ansible_pkg_mgr == "pacman"
Reference in new issue View Git Blame Copy Permalink