me/env-ansible
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a3ba5e7a605e935e74d7cb56ecf64997db047879
env-ansible/tasks/workstation/linux/software/llm.yml
T

717 lines
29 KiB
YAML
Raw Blame History

---
# 2026-10-05 Hyperling
# Install a local LLM for use offline.
# - https://zeroclaw.net/blog/install-zeroclaw
# - https://zeroclaw.net/blog/set-up-zeroclaw-bot-ollama-telegram
#
# Packages Details
# - https://archlinux.org/packages/extra/x86_64/zeroclaw/
# - https://archlinux.org/packages/extra/x86_64/ollama/
#
# Individual Guides
# - https://github.com/zeroclaw-labs/zeroclaw
# - https://wiki.archlinux.org/title/Ollama
#
# Model Explanations
# - https://webscraft.org/blog/ollama-na-8-gb-ram-yaki-modeli-pratsyuyut-u-2026?lang=en
#
# Models
# - https://ollama.com/search
#
# References
# - https://docs.zeroclaw.com/master/en/reference/config.html
# - https://docs.ollama.com
# Currently only testing this on Arch based distros (pacman).
- name: Workstation | Linux | Software | LLMs
block:
# Facts - Start #
- name: Workstation | Linux | Software | LLMs | Facts | Provisioned
ansible.builtin.debug:
var: "{{ item }}"
loop:
- ollama_enabled
- manage_models
- zeroclaw_enabled
- open_webui_enabled
- open_webui_port
- open_webui_key
- hf_token
- codium_config
- name: Workstation | Linux | Software | LLMs | Facts 01
ansible.builtin.set_fact:
ollama: ollama
zeroclaw: zeroclaw
zerocode: zerocode
qwen_code: qwen-code
dummy_service: 'cronie'
dummy_website: 'https://hyperling.com'
# Good to keep low to prevent swap usage. Going to have a minimum of 3
# though, one for Cline, two for Continue (Agent + Autocomplete).
llm_concurrency: "{{ [(ansible_memtotal_mb / 32000) | round(0, method='ceil'), 3] | max | int }}"
# Allow having multiple chats open without them just waiting each other.
# Also setting this to a minimum of 3 for using Codium with extensions.
llm_parallel: "{{ [(ansible_processor_vcpus / 2) | round(0, method='ceil'), 3] | max | int }}"
# Most LLMs take a coefficient of about 1.2 to 1.5 of their file size in
# RAM, which is usually less than guessing a usage of weight*1000.
# Example:
# - qwen3.5:9b would be 6.6GB * 1.3 = 8580, rather than 9000.
# Conversely:
# - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
llm_size_coef: 1.3
# Use the MB of RAM as the context, with a maximum to prevent swapping.
#llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}"
# Let's just be consistent for now, only having 8000 on the Macbook was not usable.
llm_context: 34543
- name: Workstation | Linux | Software | LLMs | Facts 02
ansible.builtin.set_fact:
ollama_runit_dir: "{{ runit_script_dir }}/{{ ollama }}"
ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}"
zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}"
zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}"
# Amount of output tokens allowed per response. Half of context is ok.
llm_tokens: "{{ (llm_context / 4) | int }}"
- name: Workstation | Linux | Software | LLMs | Facts 03
ansible.builtin.set_fact:
ollama_runit_script: "{{ ollama_runit_dir }}/{{ runit_service_script }}"
ollama_runit_log_dir: "{{ ollama_runit_dir }}/{{ runit_log_dir }}"
ollama_runit_log_config: "{{ ollama_runit_log_dest }}/{{ global_log_config }}"
zeroclaw_runit_script: "{{ zeroclaw_runit_dir }}/{{ runit_service_script }}"
zeroclaw_runit_log_dir: "{{ zeroclaw_runit_dir }}/{{ runit_log_dir }}"
zeroclaw_runit_log_config: "{{ zeroclaw_runit_log_dest }}/{{ global_log_config }}"
- name: Workstation | Linux | Software | LLMs | Facts 04
ansible.builtin.set_fact:
ollama_runit_log_script: "{{ ollama_runit_log_dir }}/{{ runit_log_script }}"
zeroclaw_runit_log_script: "{{ zeroclaw_runit_log_dir }}/{{ runit_log_script }}"
- name: Workstation | Linux | Software | LLMs | Facts | LLMs
ansible.builtin.set_fact:
llm_installs: []
llm_removals:
- { "purpose": "chat", "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 }
- { "purpose": "chat", "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 }
- { "purpose": "chat", "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 }
llm_choices:
### QWEN >3.5 - Need to try!
- { "purpose": ["chat", code], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 }
- { "purpose": ["chat", code], "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 }
### QWEN 3.5 - Good for generic usage and a little coding.
- { "purpose": "chat", "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
- { "purpose": "chat", "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
- { "purpose": "chat", "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
- { "purpose": "chat", "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
# skip 27b and use 35b
- { "purpose": ["chat", code], "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 }
- { "purpose": ["chat", code], "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 }
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
- { "purpose": ["code", chat], "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 }
- { "purpose": ["code", chat], "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 }
### QWEN 2.5 Coder - Ol'reliable!
- { "purpose": "code", "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
- { "purpose": "code", "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
### Mistral, Ministral, and Devstral - By a French company?
- { "purpose": "code", "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 }
#- mistral:7b
#- ministral-3:3b
#- ministral-3:8b
#- ministral-3:14b
##### TBD #####
### QWEN 2.5 - Older, but does not have "thinking", tempting!
#- qwen2.5:0.5b
#- qwen2.5:1.5b
#- qwen2.5:3b
#- qwen2.5:7b
#- qwen2.5:14b
#- qwen2.5:32b
#- qwen2.5:72b
### Z.ai
#- GLM-5.x
#- glm4-flash
### Others
#- deepseek-r1:1.5b
#- deepseek-r1:7b
- name: Workstation | Linux | Software | LLMs | Facts | Merge | Print
ansible.builtin.debug:
var: "{{ item }}"
loop:
- ansible_processor_vcpus
- ansible_memtotal_mb
- llm_concurrency
- llm_parallel
- llm_size_coef
- llm_context
- llm_tokens
- llm_installs
- llm_removals
- llm_choices
- name: Workstation | Linux | Software | LLMs | Facts | Choices | Sort
ansible.builtin.set_fact:
llm_installs: >-
{{ llm_choices | selectattr('size_mb', 'le', ansible_memtotal_mb) | list }}
llm_removals: >-
{{ llm_choices | selectattr('size_mb', 'gt', ansible_memtotal_mb) | list }}
- name: Workstation | Linux | Software | LLMs | Facts | Merge | Print
ansible.builtin.debug:
var: "{{ item }}"
loop:
- llm_installs
- llm_removals
# Facts - End #
- name: Workstation | Linux | Software | LLMs | Packages | Install
ansible.builtin.package:
name:
- "{{ ollama }}"
- "{{ zeroclaw }}"
- "{{ zerocode }}"
- "{{ qwen_code }}"
state: present
- name: Workstation | Linux | Software | LLMs | runit
block:
# Ollama - Start #
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Run
ansible.builtin.file:
name: "{{ ollama_runit_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Run
ansible.builtin.file:
name: "{{ ollama_runit_log_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Config
ansible.builtin.file:
name: "{{ ollama_runit_log_dest }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Run
ansible.builtin.copy:
dest: "{{ ollama_runit_script }}"
content: |
#!/bin/sh -e
exec 2>&1
# Allow pulling and accessing LLMs via tools.
### Required ###
export HOME={{ user_user.home }}
### Timeouts ###
export OLLAMA_LOAD_TIMEOUT=20m # default is 5m
export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars)
### Resource Usage ###
export OLLAMA_KEEP_ALIVE=2h # default is 5m
export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs
export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed)
export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size
export OLLAMA_MAX_QUEUE=999 # default is 512 (unconfirmed)
### Prevent Data Loss ###
# OLLAMA_NOPRUNE
# Default is 0, which causes unfinished pulls to be deleted
# after 1 hour of inactivity. To manually clean partials, run:
# `rm ~/.ollama/models/blobs/*-partial*`
export OLLAMA_NOPRUNE=1
### Tokens ###
{{ ("export HF_TOKEN=" ~ hf_token) if hf_token else '' }}
exec chpst -u {{ user }} ollama serve
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log
ansible.builtin.copy:
dest: "{{ ollama_runit_log_script }}"
content: |
#!/bin/sh -e
exec svlogd -tt {{ ollama_runit_log_dest }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log Config
ansible.builtin.copy:
dest: "{{ ollama_runit_log_config }}"
content: |
s10000000 # rotate at 10 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
owner: "{{ user }}"
mode: "0755"
# Ollama - End #
# Zeroclaw - Start #
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Run
ansible.builtin.file:
name: "{{ zeroclaw_runit_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log
ansible.builtin.file:
name: "{{ zeroclaw_runit_log_dir }}"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log Config
ansible.builtin.file:
name: "{{ zeroclaw_runit_log_dest }}"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | File
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_script }}"
content: |
#!/bin/sh -e
# Allow using the ZeroClaw frontend.
exec 2>&1
export HOME={{ user_user.home }}
export SHELL={{ bash_exec.stdout }}
exec chpst -u {{ user }} zeroclaw daemon -v --log-level info
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_script }}"
content: |
#!/bin/sh -e
exec svlogd -tt {{ zeroclaw_runit_log_dest }}
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log Config
ansible.builtin.copy:
dest: "{{ zeroclaw_runit_log_config }}"
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Zeroclaw - End #
# Open WebUI - Start #
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Packages
ansible.builtin.package:
name:
- docker
state: present
when: open_webui_enabled
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Packages | init
ansible.builtin.package:
name:
- docker-runit
state: present
when: open_webui_enabled
and ansible_pkg_mgr == "pacman"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/open-webui/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /var/log
ansible.builtin.file:
name: "/var/log/open-webui"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Run
ansible.builtin.copy:
dest: /etc/sv/open-webui/run
content: |
#!/bin/sh
exec 2>&1
# Wait for Docker to start up.
while [ ! -S /var/run/docker.sock ]; do sleep 1; done
# Ensure the old container is gone.
echo "RUN: Stopping..."
docker stop open-webui &&
echo "RUN: Container stopped" ||
echo "RUN: Container already stopped."
echo "RUN: Deleting..."
docker rm -f open-webui &&
echo "RUN: Container deleted" ||
echo "RUN: Container already deleted."
# Start the open-webui program.
set -e
exec docker run --rm \
-p {{ open_webui_port }}:8080 \
--add-host=host.docker.internal:host-gateway \
-v open-webui:/app/backend/data \
{{ ("-e HF_TOKEN=" ~ hf_token) if hf_token else '' }} \
-e WEBUI_SECRET_KEY={{ open_webui_key }} \
-e OLLAMA_BASE_URL=http://host.docker.internal:11434 \
--name open-webui \
ghcr.io/open-webui/open-webui:main
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/open-webui/finish
content: |
#!/bin/sh
exec 2>&1
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "FINISH: Service returned with exit code: $1 and status: $2"
echo "FINISH: Stopping..."
docker stop open-webui &&
echo "FINISH: Container stopped" ||
echo "FINISH: Container already stopped."
echo "FINISH: Deleting..."
docker rm -f open-webui &&
echo "FINISH: Container deleted" ||
echo "FINISH: Container already deleted."
sleep=30
if [[ "$1" != 0 ]]; then
echo "FINISH: Sleeping for $sleep seconds due to error..."
sleep $sleep
fi
echo "FINISH: Done!"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log
ansible.builtin.copy:
dest: /etc/sv/open-webui/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/open-webui
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log Config
ansible.builtin.copy:
dest: /var/log/open-webui/config
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Open WebUI - End #
# Swap Stopper - Start #
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Executable
ansible.builtin.copy:
dest: "{{ global_bin }}/llm_swap_stopper.sh"
src: files/scripts/llm_swap_stopper.sh
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /etc/sv
ansible.builtin.file:
name: "/etc/sv/swap-stopper/log"
state: directory
owner: root
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /var/log
ansible.builtin.file:
name: "/var/log/swap-stopper"
state: directory
owner: "{{ user }}"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Run
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/run
content: |
#!/bin/sh -e
exec 2>&1
exec {{ global_bin }}/llm_swap_stopper.sh
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Finish
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/finish
content: |
#!/bin/sh -e
exec 2>&1
# $1 is the exit code, $2 is the exit status byte (signal number if killed)
echo "FINISH: Service returned with exit code: $1 and status: $2"
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log
ansible.builtin.copy:
dest: /etc/sv/swap-stopper/log/run
content: |
#!/bin/sh -e
exec svlogd -tt /var/log/swap-stopper
mode: "0755"
- name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log Config
ansible.builtin.copy:
dest: /var/log/swap-stopper/config
content: |
s1000000 # rotate at 1 MB
n5 # keep 5 old files
dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file.
mode: "0755"
# Swap Saver - End #
# Qwen Serve - Start #
# TBD
# Qwen Serve - End #
when: ansible_service_mgr == "runit"
# Services - Start #
- name: Workstation | Linux | Software | LLMs | Services | Enable
ansible.builtin.service:
name: "{{ item.name }}"
state: "{{ 'started' if item.status else 'stopped' }}"
enabled: "{{ item.status }}"
loop:
- { 'name': "{{ ollama }}" , 'status': "{{ ollama_enabled }}" }
- { 'name': "{{ zeroclaw }}", 'status': "{{ zeroclaw_enabled }}" }
- { 'name': 'docker' , 'status': true }
- { 'name': 'open-webui' , 'status': "{{ open_webui_enabled }}" }
- { 'name': 'swap-stopper' , 'status': true }
- name: Workstation | Linux | Software | LLMs | Services | Verify
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
timeout: 5
register: health
until: health.status == 200
retries: 12
delay: 5
loop:
- "{{ 'http://127.0.0.1:11434/' if ollama_enabled else dummy_website }}"
- "{{ 'http://127.0.0.1:42617/health' if zeroclaw_enabled else dummy_website }}"
# TBD: - "{{ 'http://127.0.0.1:4170' if qwen_serve_enabled else dummy_website }}"
- name: Workstation | Linux | Software | LLMs | Services | Verify | Okay to Fail
ansible.builtin.uri:
url: "{{ item }}"
status_code: 200
timeout: 5
register: health
until: health.status == 200
retries: 2
delay: 5
loop:
- "{{ 'http://127.0.0.1:3000/health' if open_webui_enabled else dummy_website }}"
ignore_errors: true
# Services - End #
# Models - Start #
- name: Workstation | Linux | Software | LLMs | Models | ollama | User Shell Functions
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/{{ item }}"
marker: "# {mark} ANSIBLE MANAGED BLOCK | ollama | Manual Script"
content: |
function pull_llms() {
set -e
echo "Manually Pulling Models"
{% for llm in llm_installs %}
echo -e "\n\n Starting {{ llm.model }} \n\n"
ollama pull {{ llm.model }}
{% endfor %}
echo "Done!!"
}
alias pull-llms="pull_llms "
alias pull-models="pull_llms "
alias pull-ai="pull_llms "
function ollama_stop {
for model in `ollama ps | tail -n +2 | awk '{print $1}'`; do
echo "Stopping ${model}..."
ollama stop $model
done
}
alias ollama-stop="ollama_stop"
alias ollama-kill="ollama_stop"
alias ollama-killall="ollama_stop"
alias ollama-kill-all="ollama_stop"
alias ollama-stop-all="ollama_stop"
state: present
create: false
backup: true
loop:
- .bashrc
- .zshrc
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage
block:
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Message
ansible.builtin.debug:
msg: <-
The next task may take hours, if not days. If you'd like a status
bar to know what's being worked on and its progress, I highly
recommed reloading your RC file and running `pull-models`. That
will give you a much better idea of what's going on.
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Install
ansible.builtin.shell: |
set -o pipefail
item='{{ item.model }}'
if ollama list | grep -q $item; then
echo "OKAY" &&
return
else
ollama pull $item 1>/dev/null &&
echo "INSTALLED" &&
return
fi
echo "FAILED"
loop: "{{ llm_installs }}"
register: shell_check
changed_when: "'INSTALLED' in shell_check.stdout"
failed_when: "'FAILED' in shell_check.stdout"
become: true
become_user: "{{ user }}"
when: manage_models
and llm_installs | length > 0
- name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Remove
ansible.builtin.shell: |
set -o pipefail
item='{{ item.model }}'
if ollama list | grep -q $item; then
ollama rm $item &&
echo "INSTALLED" &&
return
else
echo "OKAY" &&
return
fi
echo "FAILED"
loop: "{{ llm_removals }}"
register: shell_check
changed_when: "'INSTALLED' in shell_check.stdout"
failed_when: "'FAILED' in shell_check.stdout"
become: true
become_user: "{{ user }}"
when: llm_removals | length > 0
when: manage_models
- name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models
ansible.builtin.shell:
cmd: |
set -o pipefail
ollama ls | tail -n +2 | awk '{print $1}'
register: llm_models
changed_when: llm_models.rc == 0
- name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models | Result
ansible.builtin.debug:
var: llm_models
# Cheat Sheet: https://docs.zeroclaw.com/master/en/providers/configuration.html
- name: Workstation | Linux | Software | LLMs | Models | zeroclaw | Add Providers
ansible.builtin.blockinfile:
path: "{{ user_user.home }}/.zeroclaw/config.toml"
marker: "# {mark} ANSIBLE MANAGED BLOCK | zeroclaw | Providers"
insertafter: EOF
block: |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "chat" in llm.purpose %}
[providers.models.ollama.{{ llm.provider }}]
model = "{{ llm.model }}"
timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
context_window = {{ [llm_context, llm.context, 16000] | min | int }}
max_tokens = {{ [llm_tokens, (llm.context / 4), 4000] | min | int }}
{% endif %}
{% endif %}
{% endfor %}
state: present
create: false
backup: true
when: zeroclaw_enabled
- name: Workstation | Linux | Software | LLMs | Models | Continue | Config
ansible.builtin.copy:
dest: "{{ user_user.home }}/.continue/config.yaml"
content: |
# Maintained by `env-ansible/tasks/workstation/linux/software/llm.yml`
name: Main Config
version: 1.0.0
schema: v1
models:
# | Role | `contextLength` | `maxTokens` | Notes |
# |------|-----------------|-------------|-------|
# | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight |
# | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs |
{% for llm in llm_installs %}
{% if llm.model in llm_models.stdout %}
{% if "code" in llm.purpose %}
- name: {{ llm.provider }}_chat-edit
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context, 24000] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 8000] | min | int }}
- name: {{ llm.provider }}_autocomplete
provider: ollama
model: {{ llm.model }}
defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context, 12000] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }}
roles:
- autocomplete
{% endif %}
{% endif %}
{% endfor %}
mode: "0644"
when: codium_config
# Models - End #
when: ansible_pkg_mgr == "pacman"
Reference in New Issue View Git Blame Copy Permalink