--- # 2026-10-05 Hyperling # Install a local LLM for use offline. # - https://zeroclaw.net/blog/install-zeroclaw # - https://zeroclaw.net/blog/set-up-zeroclaw-bot-ollama-telegram # # Packages Details # - https://archlinux.org/packages/extra/x86_64/zeroclaw/ # - https://archlinux.org/packages/extra/x86_64/ollama/ # # Individual Guides # - https://github.com/zeroclaw-labs/zeroclaw # - https://wiki.archlinux.org/title/Ollama # # Model Explanations # - https://webscraft.org/blog/ollama-na-8-gb-ram-yaki-modeli-pratsyuyut-u-2026?lang=en # # Models # - https://ollama.com/search # # References # - https://docs.zeroclaw.com/master/en/reference/config.html # - https://docs.ollama.com # - https://docs.ollama.com/faq#setting-environment-variables-on-linux # - https://modelpiper.com/blog/ollama-environment-variables # - https://qwenlm.github.io/qwen-code-docs/en/users/configuration/settings/#environment-variables--env-files # - https://qwenlm.github.io/qwen-code-docs/en/users/qwen-serve/ # Currently only testing this on Arch based distros (pacman). - name: Workstation | Linux | Software | LLMs block: # Facts - Start # - name: Workstation | Linux | Software | LLMs | Facts | Provisioned ansible.builtin.debug: var: "{{ item }}" loop: - ollama_enabled - manage_models - hf_token - zeroclaw_enabled - open_webui_enabled - open_webui_port - open_webui_key - codium_config - qwen_code_enabled - qwen_code_key - open_terminal_enabled - open_terminal_port - open_terminal_key - name: Workstation | Linux | Software | LLMs | Facts 01 ansible.builtin.set_fact: ollama: ollama zeroclaw: zeroclaw zerocode: zerocode qwen_code: qwen-code dummy_service: 'cronie' dummy_website: 'https://hyperling.com' # Good to keep low to prevent swap usage. Going to have a minimum of 3 # though, one for Cline, two for Continue (Agent + Autocomplete). # 20261014: Going down to a static of 3 since we're focusing on dense. llm_concurrency: 3 #"{{ [(ansible_memtotal_mb / 32000) | round(0, method='ceil'), 3] | max | int }}" # Allow having multiple chats open without them just waiting each other. # Also setting this to a minimum of 3 for using Codium with extensions. # 20261014: Going down to 1 since only using CPUs, no GPUs. Otherwise # the models seem to take excessively long to answer even small things. #llm_parallel: 1 #"{{ [(ansible_processor_vcpus / 2) | round(0, method='ceil'), 3] | max | int }}" # 20261027 Increased to 3 as a maximum. # After more research, this is a more memory-based parameter. Basically # the model allocates this number of KV caches, that way when it is # running in multiple chats it does not need to drop context and reload. # Still only uses X threads for all chats, but does not waste much time. # So if set to 3, and doing 64K context, it really allocates 192k in RAM # and supposedly if only doing 1 chat the model can use it all. llm_parallel: "{{ [(ansible_memtotal_mb / 8000) | round(0, method='floor'), 3] | min | int }}" # Most LLMs take a coefficient of about 1.2 to 1.5 of their file size in # RAM, which is usually less than guessing a usage of weight*1000. # Example: # - qwen3.5:9b would be 6.6GB * 1.3 = 8580, rather than 9000. # Conversely: # - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000. llm_size_coef: 1.3 # Base the context on MB of RAM, with a minumum 64k for decent thinking. llm_context: "{{ [(ansible_memtotal_mb * 4) | round(-3, method='ceil'), 64000] | max | int }}" llm_docker_network: open-webui-net - name: Workstation | Linux | Software | LLMs | Facts 02 ansible.builtin.set_fact: ollama_runit_dir: "{{ runit_script_dir }}/{{ ollama }}" ollama_runit_log_dest: "{{ global_log }}/{{ ollama }}" zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}" zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}" # Amount of output tokens allowed per response. Half of context is ok. llm_tokens: "{{ (llm_context / 8) | int }}" - name: Workstation | Linux | Software | LLMs | Facts 03 ansible.builtin.set_fact: ollama_runit_script: "{{ ollama_runit_dir }}/{{ runit_service_script }}" ollama_runit_log_dir: "{{ ollama_runit_dir }}/{{ runit_log_dir }}" ollama_runit_log_config: "{{ ollama_runit_log_dest }}/{{ global_log_config }}" zeroclaw_runit_script: "{{ zeroclaw_runit_dir }}/{{ runit_service_script }}" zeroclaw_runit_log_dir: "{{ zeroclaw_runit_dir }}/{{ runit_log_dir }}" zeroclaw_runit_log_config: "{{ zeroclaw_runit_log_dest }}/{{ global_log_config }}" - name: Workstation | Linux | Software | LLMs | Facts 04 ansible.builtin.set_fact: ollama_runit_log_script: "{{ ollama_runit_log_dir }}/{{ runit_log_script }}" zeroclaw_runit_log_script: "{{ zeroclaw_runit_log_dir }}/{{ runit_log_script }}" - name: Workstation | Linux | Software | LLMs | Facts | LLMs ansible.builtin.set_fact: llm_installs: [] llm_removals: ##### QWEN ##### - { "purpose": ["chat"], "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded ### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it. Went okay, but better options now. - { "purpose": ["code", "chat"], "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300_moe", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 262144 } - { "purpose": ["code", "chat"], "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200_moe", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 262144 } ### QWEN 2.5 Coder - Ol'reliable! Did not seem to handle tools too well though. Defunitely not conversation. - { "purpose": ["code"], "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32768 } - { "purpose": ["code"], "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32768 } - { "purpose": ["code"], "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32768 } - { "purpose": ["code"], "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32768 } - { "purpose": ["code"], "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32768 } - { "purpose": ["code"], "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32768 } ##### Mistral ##### - { "purpose": ["code"], "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded # https://ollama.com/library/mistral-small : Last Updated 2025-01-30, succeeded - { "purpose": ["chat"], "model": "mistral-small:24b", "provider": "mistral_small30_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 32000 } - { "purpose": ["chat"], "model": "mistral-small:22b", "provider": "mistral_small30_0220", "size_mb": "{{ llm_size_coef * 13000 }}", "context": 128000 } # Yes, the smaller one has larger context. # https://ollama.com/library/mistral-small3.1 : Last Update 2025-04-07, succeeded - { "purpose": ["chat"], "model": "mistral-small3.1:24b", "provider": "mistral_small31_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 } # Supposedly ollama has it tagged at mistral-small with no params given. it's not on the site though. Let's find out! # No, this is Mistral 3, as the website says. Ask Brave is wrooong. - { "purpose": ["chat", "lang", "code"], "model": "mistral-small", "provider": "mistral_small40_1190", "size_mb": 60000 , "context": 256000 } # https://ollama.com/library/mistral-nemo : Last Updated 2025-07-23 # Supposed to be a drop-in replacement for mistral:7b. Not really any more useful than the other options. - { "purpose": ["chat", "lang"], "model": "mistral-nemo:12b", "provider": "mistral_nemo_0120", "size_mb": "{{ llm_size_coef * 7100 }}", "context": 128000 } # says 1000K in table but 128k in desc, going with desc. llm_choices: ##### QWEN ##### ### QWEN >3.5 - Need to try! # Supposedly great for coding over any previous version. # Prefer 27b (dense) over anything higher which likely uses MoE (essentially only uses 3b inside of a 35b, dumber but faster and potentially has access to more info). # Update: MoE actually runs really well on my workstation hardware, significantly faster than dense models, gives good results too. # https://ollama.com/library/qwen3.8 : Last Updated 2026-09-25 # Known to overthink, suggested to prefer 3.6:27b unless setting temp to low/medium. - { "purpose": ["code", "chat", "lang"], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 } # https://ollama.com/library/qwen3.6 : Last Updated 2026-10-13 (yesterday as of this commit!!) # Has been succeeded but dense and does not overthink so still worthy - { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 } - { "purpose": ["chat", "code", "lang"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended # https://ollama.com/library/qwen3.5 : Last Updated 2026-10-13 (yesterday as of this commit!!) ### QWEN 3.5 - Good for generic usage and a little coding. - { "purpose": ["chat"], "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 262144 } - { "purpose": ["chat"], "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 262144 } - { "purpose": ["chat"], "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 262144 } - { "purpose": ["chat"], "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 262144 } - { "purpose": ["chat", "code"], "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 262144 } # succeeded - { "purpose": ["chat", "code"], "model": "qwen3.5:35b", "provider": "qwen_35_0350_moe", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 262144 } - { "purpose": ["chat", "code"], "model": "qwen3.5:122b", "provider": "qwen_35_1220_moe", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 262144 } ##### Mistral, Ministral, and Devstral - By a French company. ##### # mistral-small-4 # https://huggingface.co/unsloth/Mistral-Small-4-119B-2603-GGUF # Supposed to be better than Qwen equivalents, it's basically mistral, mixtral, magistral, and devstral all in one. # unsloth version is supposed to be the best quantization. UD-Q3_K_XL is supposed to take 64GB RAM, lol, while Q4_K_M would need 80+ GB. # needs done via 'ollama create', fails with pull. https://github.com/ollama/ollama/issues/5245 - { "purpose": ["chat", "lang", "code"], "model": "hf.co/unsloth/Mistral-Small-4-119B-2603-GGUF:UD-Q3_K_XL", "provider": "mistral_small40_1190_unsloth_q3", "size_mb": "{{ llm_size_coef * 54800 }}", "context": 256000 } # https://huggingface.co/mistralai/Mistral-Small-4-119B-2603 : 242GB unquantized, lol. - { "purpose": ["chat", "lang", "code"], "model": "hf.co/mistralai/Mistral-Small-4-119B-2603", "provider": "mistral_small40_1190", "size_mb": "{{ llm_size_coef * 242000 }}", "context": 256000 } # https://huggingface.co/unsloth/Mistral-Small-4-119B-2603 : 121GB, A6B MoE, hmmm! - { "purpose": ["chat", "lang", "code"], "model": "hf.co/unsloth/Mistral-Small-4-119B-2603", "provider": "mistral_small40_1190_unsloth_a6b", "size_mb": "{{ llm_size_coef * 121000 }}", "context": 256000 } # https://ollama.com/library/devstral-small-2 : Last Updated 2025-12-13 # Most recommended in Mistral family for agentic coding. - { "purpose": ["code"], "model": "devstral-small-2:24b", "provider": "mistral_dev20_0240_small", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 393216 } # https://ollama.com/library/codestral : Last Updated 2024-09-03 # Mistral's first ever coder agent, recommended for autocomplete (FIM). - { "purpose": ["code"], "model": "codestral:22b", "provider": "mistral_code_0220", "size_mb": "{{ llm_size_coef * 12000 }}", "context": 32768 } # https://ollama.com/library/mistral : Last Updated 2025-07-13 - { "purpose": ["chat"], "model": "mistral:7b", "provider": "mistral_gen_0070", "size_mb": "{{ llm_size_coef * 4400 }}", "context": 32768 } # https://ollama.com/library/mistral-small3.2 : Last Updated 2025-06-20 - { "purpose": ["chat", "lang"], "model": "mistral-small3.2:24b", "provider": "mistral_small32_0240", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 128000 } # rated best for Croatian # https://ollama.com/library/ministral-3 : Last Updated 2025-12-13 - { "purpose": ["chat", "lang"], "model": "ministral-3:3b", "provider": "mistral_mini30_0030", "size_mb": "{{ llm_size_coef * 3000 }}", "context": 256000 } - { "purpose": ["chat", "lang"], "model": "ministral-3:8b", "provider": "mistral_mini30_0080", "size_mb": "{{ llm_size_coef * 6000 }}", "context": 256000 } - { "purpose": ["chat", "lang"], "model": "ministral-3:14b", "provider": "mistral_mini30_0140", "size_mb": "{{ llm_size_coef * 9100 }}", "context": 256000 } # also rated super high for Croatian, gives more head room. ##### TBD ##### ### Z.ai #- GLM-5.x #- glm4-flash ### Others #- deepseek-r1:1.5b #- deepseek-r1:7b - name: Workstation | Linux | Software | LLMs | Facts | Merge | Print ansible.builtin.debug: var: "{{ item }}" loop: - ansible_processor_vcpus - ansible_memtotal_mb - llm_concurrency - llm_parallel - llm_size_coef - llm_context - llm_tokens - llm_installs - llm_removals - llm_choices - name: Workstation | Linux | Software | LLMs | Facts | Choices | Sort ansible.builtin.set_fact: llm_installs: >- {{ llm_installs | union(llm_choices | selectattr('size_mb', 'le', ansible_memtotal_mb) | list) }} llm_removals: >- {{ llm_removals | union(llm_choices | selectattr('size_mb', 'gt', ansible_memtotal_mb) | list) }} - name: Workstation | Linux | Software | LLMs | Facts | Merge | Print ansible.builtin.debug: var: "{{ item }}" loop: - llm_installs - llm_removals # Facts - End # - name: Workstation | Linux | Software | LLMs | Packages | Install ansible.builtin.package: name: - "{{ ollama }}" - "{{ zeroclaw }}" - "{{ zerocode }}" - "{{ qwen_code }}" state: present - name: Workstation | Linux | Software | LLMs | runit block: # Ollama - Start # - name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Run ansible.builtin.file: name: "{{ ollama_runit_dir }}" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Run ansible.builtin.file: name: "{{ ollama_runit_log_dir }}" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Dir | Log Config ansible.builtin.file: name: "{{ ollama_runit_log_dest }}" state: directory owner: "{{ user }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Run ansible.builtin.copy: dest: "{{ ollama_runit_script }}" content: | #!/bin/sh -e exec 2>&1 # Allow pulling and accessing LLMs via tools. ### Required ### export HOME={{ user_user.home }} export OLLAMA_HOST=0.0.0.0 # Needed for Open WebUI ### Timeouts ### export OLLAMA_LOAD_TIMEOUT=30m # default is 5m export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars) ### Resource Usage ### export OLLAMA_KEEP_ALIVE=-1 # default is 5m. Good frontends keep the model alive as long as it needs to be, this is not a source of timeout. export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size export OLLAMA_MAX_QUEUE=999 # default is 512 (unconfirmed) export OLLAMA_NUM_THREADS={{ (ansible_processor_vcpus / 2) | int }} # not real, but maybe some day. ### Prevent Data Loss ### # OLLAMA_NOPRUNE # Default is 0, which causes unfinished pulls to be deleted # after 1 hour of inactivity. To manually clean partials, run: # `rm ~/.ollama/models/blobs/*-partial*` export OLLAMA_NOPRUNE=1 ### Tokens ### {{ ("export HF_TOKEN=" ~ hf_token) if hf_token else '' }} ### Logging ### #export OLLAMA_DEBUG=1 export OLLAMA_METRICS=1 # (check http://127.0.0.1:11434/metrics) ### Other ### export OLLAMA_ORIGINS=* exec chpst -u {{ user }} ollama serve mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Finish ansible.builtin.copy: dest: /etc/sv/{{ ollama }}/finish content: | {{ runit_finish_sleep }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log ansible.builtin.copy: dest: "{{ ollama_runit_log_script }}" content: | #!/bin/sh -e exec svlogd -tt {{ ollama_runit_log_dest }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log Config ansible.builtin.copy: dest: "{{ ollama_runit_log_config }}" content: | {{ runit_log_config_10mb }} owner: "{{ user }}" mode: "0755" # Ollama - End # # Zeroclaw - Start # - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Run ansible.builtin.file: name: "{{ zeroclaw_runit_dir }}" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log ansible.builtin.file: name: "{{ zeroclaw_runit_log_dir }}" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Dir | Log Config ansible.builtin.file: name: "{{ zeroclaw_runit_log_dest }}" state: directory owner: "{{ user }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | File ansible.builtin.copy: dest: "{{ zeroclaw_runit_script }}" content: | #!/bin/sh -e # Allow using the ZeroClaw frontend. exec 2>&1 export HOME={{ user_user.home }} export SHELL={{ bash_exec.stdout }} exec chpst -u {{ user }} zeroclaw daemon -v --log-level info mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Finish ansible.builtin.copy: dest: /etc/sv/{{ zeroclaw }}/finish content: | {{ runit_finish_sleep }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log ansible.builtin.copy: dest: "{{ zeroclaw_runit_log_script }}" content: | #!/bin/sh -e exec svlogd -tt {{ zeroclaw_runit_log_dest }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log Config ansible.builtin.copy: dest: "{{ zeroclaw_runit_log_config }}" content: | {{ runit_log_config_10mb }} mode: "0755" # Zeroclaw - End # # Open WebUI - Start # - name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /etc/sv ansible.builtin.file: name: "/etc/sv/open-webui/log" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-webui | Dir | /var/log ansible.builtin.file: name: "/var/log/open-webui" state: directory owner: "{{ user }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Run ansible.builtin.copy: dest: /etc/sv/open-webui/run content: | #!/bin/sh exec 2>&1 ## TBD / TODO ## Move this to the same compose file as Open Terminal. ## Will solve quite a few network related complications. ## Also refactor the two into their own file (open_webui.yml). ## Probably also will then condense into 1 runit service. image=ghcr.io/open-webui/open-webui:latest # Wait for Docker to start up. while [ ! -S /var/run/docker.sock ]; do sleep 1; done # Ensure the old container is gone. echo "RUN: Stopping..." docker stop open-webui && echo "RUN: Container stopped." || echo "RUN: Container already stopped." echo "RUN: Deleting..." docker rm -f open-webui && echo "RUN: Container deleted." || echo "RUN: Container already deleted." # Check for any new updates in the background. ( while true; do sleep 600 echo "RUN: Checking for updates..." docker pull $image done ) & LOOP_PID=$! # Ensure cleanup on any exit (normal or signal from runit). trap 'kill "$LOOP_PID" 2>/dev/null; wait "$LOOP_PID" 2>/dev/null' EXIT # Start the open-webui program. set -e docker run \ --name open-webui \ --network host -e PORT={{ open_webui_port }} \ -e OLLAMA_BASE_URL=http://0.0.0.0:11434 \ {{ ("-e HF_TOKEN=" ~ hf_token) if hf_token else '' }} \ -e WEBUI_SECRET_KEY={{ open_webui_key }} \ -e ENABLE_QUERIES_CACHE=True \ -v open-webui:/app/backend/data \ $image mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Finish ansible.builtin.copy: dest: /etc/sv/open-webui/finish content: | {{ runit_finish_sleep }} echo "FINISH: Stopping..." docker stop open-webui && echo "FINISH: Container stopped" || echo "FINISH: Container already stopped." echo "FINISH: Deleting..." docker rm -f open-webui && echo "FINISH: Container deleted" || echo "FINISH: Container already deleted." echo "FINISH: Done!" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log ansible.builtin.copy: dest: /etc/sv/open-webui/log/run content: | #!/bin/sh -e exec svlogd -tt /var/log/open-webui mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-webui | Script | Log Config ansible.builtin.copy: dest: /var/log/open-webui/config content: | {{ runit_log_config_100mb }} mode: "0755" # Open WebUI - End # # Open Terminal - Start # - name: Workstation | Linux | Software | LLMs | Sandbox ansible.builtin.import_tasks: tasks/workstation/linux/software/ai/sandbox.yml - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Facts ansible.builtin.set_fact: jdk_jvm: "{{ jdk_jvm | default('/usr/lib/jvm') }}" jdk_etc: "{{ jdk_etc | default('/etc/java-openjdk') }}" jdk_etc17: "{{ jdk_etc17 | default('/etc/java17-openjdk') }}" jdk_etc21: "{{ jdk_etc21 | default('/etc/java21-openjdk') }}" jdk_etc25: "{{ jdk_etc25 | default('/etc/java25-openjdk') }}" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /etc/sv ansible.builtin.file: name: "/etc/sv/open-terminal/log" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /var/log ansible.builtin.file: name: "/var/log/open-terminal" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Dir | /usr/local/etc ansible.builtin.file: name: "/usr/local/etc/open-terminal" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Compose YAML ansible.builtin.copy: dest: /usr/local/etc/open-terminal/compose.yml content: | services: open-terminal: container_name: open-terminal build: context: ./ image: hyperling/open-terminal:latest pull_policy: never network_mode: host command: run --port {{ open_terminal_port }} environment: - OPEN_TERMINAL_FILE_BROWSER_ROOT=/home/user - OPEN_TERMINAL_API_KEY={{ open_terminal_key }} - OPEN_TERMINAL_PACKAGES= - GOT_EM=true volumes: - {{ user_user.home }}/AI-Sandbox:/home/user - {{ user_user.home }}/Code:{{ user_user.home }}/Code - {{ user_user.home }}/SDKs:{{ user_user.home }}/SDKs - {{ jdk_jvm }}:{{ jdk_jvm }}:ro - {{ jdk_etc }}:{{ jdk_etc }}:ro - /etc/ssl/certs/java:/etc/ssl/certs/java:ro - /etc/ca-certificates:/etc/ca-certificates:ro - {{ jdk_etc17 }}:{{ jdk_etc17 }}:ro - {{ jdk_etc21 }}:{{ jdk_etc21 }}:ro - {{ jdk_etc25 }}:{{ jdk_etc25 }}:ro - {{ user_user.home }}/.gradle:{{ user_user.home }}/.gradle - {{ user_user.home }}/.pub-cache:{{ user_user.home }}/.pub-cache owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Dockerfile ansible.builtin.copy: dest: /usr/local/etc/open-terminal/Dockerfile content: | FROM ghcr.io/open-webui/open-terminal:latest USER root RUN echo "System" RUN apt-get update && apt-get install -y \ ca-certificates git curl bash sudo wget ansible ansible-lint RUN echo "Android" RUN apt-get update && apt-get install -y \ libc6:amd64 libncurses-dev \ libstdc++6 lib32z1 libbz2-1.0 android-udev-rules # Debian removed JDK 17, continue serving from the host. # default-jdk openjdk-17-jdk RUN echo "Flutter" RUN apt-get update && apt-get install -y \ curl git unzip xz-utils zip libglu1-mesa libglu1-mesa RUN echo "Flutter (Linux)" RUN apt-get update && apt-get install -y \ clang cmake git ninja-build pkg-config libgtk-3-dev \ liblzma-dev libstdc++-12-dev RUN echo "Flutter (Android)" RUN apt-get update && apt-get install -y \ libc6:amd64 libstdc++6 lib32z1 libbz2-1.0 RUN echo "Ansible" RUN apt-get update && apt-get install -y \ ansible ansible-lint USER user owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | env ansible.builtin.copy: dest: /usr/local/etc/open-terminal/.env content: | COMPOSE_BAKE=true owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Run ansible.builtin.copy: dest: /etc/sv/open-terminal/run content: | #!/bin/sh exec 2>&1 # Wait for Docker to start up. while [ ! -S /var/run/docker.sock ]; do sleep 1; done # Ensure the old container is stopped. echo "RUN: Stopping..." docker compose \ -f /usr/local/etc/open-terminal/compose.yml down && echo "RUN: Container stopped." || echo "RUN: Container already stopped." # Check for any new updates in the background. ( while true; do sleep 600 echo "RUN: Checking for updates..." docker compose \ -f /usr/local/etc/open-terminal/compose.yml \ build --pull done ) & LOOP_PID=$! # Ensure cleanup on any exit (normal or signal from runit). trap 'kill "$LOOP_PID" 2>/dev/null; wait "$LOOP_PID" 2>/dev/null' EXIT export GOT_EM=true # Start the open-terminal program. # https://github.com/open-webui/open-terminal#configuration set -e docker compose -f /usr/local/etc/open-terminal/compose.yml up owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Finish ansible.builtin.copy: dest: /etc/sv/open-terminal/finish content: | {{ runit_finish_sleep }} echo "FINISH: Stopping..." docker compose -f /usr/local/etc/open-terminal/compose.yml down && echo "FINISH: Container stopped" || echo "FINISH: Container already stopped." echo "FINISH: Done!" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Log ansible.builtin.copy: dest: /etc/sv/open-terminal/log/run content: | #!/bin/sh -e exec svlogd -tt /var/log/open-terminal mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Log Config ansible.builtin.copy: dest: /var/log/open-terminal/config content: | {{ runit_log_config_100mb }} mode: "0755" # Open Terminal - End # # Swap Stopper - Start # - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Executable ansible.builtin.copy: dest: "{{ global_bin }}/llm_swap_stopper.sh" src: files/scripts/llm_swap_stopper.sh mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /etc/sv ansible.builtin.file: name: "/etc/sv/swap-stopper/log" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Dir | /var/log ansible.builtin.file: name: "/var/log/swap-stopper" state: directory owner: "{{ user }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Run ansible.builtin.copy: dest: /etc/sv/swap-stopper/run content: | #!/bin/sh -e exec 2>&1 exec {{ global_bin }}/llm_swap_stopper.sh mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Finish ansible.builtin.copy: dest: /etc/sv/swap-stopper/finish content: | {{ runit_finish_sleep }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log ansible.builtin.copy: dest: /etc/sv/swap-stopper/log/run content: | #!/bin/sh -e exec svlogd -tt /var/log/swap-stopper mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | swap-stopper | Script | Log Config ansible.builtin.copy: dest: /var/log/swap-stopper/config content: | {{ runit_log_config_1mb }} mode: "0755" # Swap Saver - End # # Qwen Serve - Start # - name: Workstation | Linux | Software | LLMs | runit | qwen-code | Dir | /etc/sv ansible.builtin.file: name: "/etc/sv/{{ qwen_code }}/log" state: directory owner: root mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | qwen-code | Dir | /var/log ansible.builtin.file: name: "/var/log/{{ qwen_code }}" state: directory owner: "{{ user }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Run ansible.builtin.copy: dest: /etc/sv/{{ qwen_code }}/run content: | #!/bin/sh -e exec 2>&1 export QWEN_STREAM_IDLE_TIMEOUT_MS=6000000 {{ ('export QWEN_SERVER_TOKEN=' ~ qwen_code_key) if qwen_code_key else '' }} exec chpst -u {{ user }} -C {{ user_user.home }}/Code qwen serve mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Finish ansible.builtin.copy: dest: /etc/sv/{{ qwen_code }}/finish content: | {{ runit_finish_sleep }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log ansible.builtin.copy: dest: /etc/sv/{{ qwen_code }}/log/run content: | #!/bin/sh -e exec svlogd -tt /var/log/{{ qwen_code }} mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | qwen-code | Script | Log Config ansible.builtin.copy: dest: /var/log/{{ qwen_code }}/config content: | {{ runit_log_config_10mb }} mode: "0755" # Qwen Serve - End # when: ansible_service_mgr == "runit" # Services - Start # - name: Workstation | Linux | Software | LLMs | Services | Enable ansible.builtin.service: name: "{{ item.name }}" state: "{{ 'started' if item.status else 'stopped' }}" enabled: "{{ item.status }}" loop: - { 'name': "{{ ollama }}" , 'status': "{{ ollama_enabled }}" } - { 'name': "{{ zeroclaw }}" , 'status': "{{ zeroclaw_enabled }}" } - { 'name': 'open-webui' , 'status': "{{ open_webui_enabled }}" } - { 'name': 'open-terminal' , 'status': "{{ open_terminal_enabled }}" } - { 'name': 'swap-stopper' , 'status': true } - { 'name': '{{ qwen_code }}', 'status': "{{ qwen_code_enabled }}" } - name: Workstation | Linux | Software | LLMs | Services | Verify | Critical ansible.builtin.uri: url: "{{ item }}" status_code: 200 timeout: 5 register: health until: health.status == 200 retries: 12 delay: 5 loop: - "{{ 'http://127.0.0.1:11434' if ollama_enabled else dummy_website }}" - name: Workstation | Linux | Software | LLMs | Services | Verify | Okay to Fail ansible.builtin.uri: url: "{{ item }}" status_code: 200 timeout: 5 register: health until: health.status == 200 retries: 2 delay: 5 loop: - "{{ 'http://127.0.0.1:42617/health' if zeroclaw_enabled else dummy_website }}" - "{{ 'http://127.0.0.1:' ~ open_webui_port ~ '/health' if open_webui_enabled else dummy_website }}" - "{{ 'http://127.0.0.1:' ~ open_terminal_port ~ '/health' if open_terminal_enabled else dummy_website }}" - "{{ 'http://127.0.0.1:4170' if qwen_code_enabled else dummy_website }}" ignore_errors: true # Services - End # # Models - Start # - name: Workstation | Linux | Software | LLMs | Models | ollama | User Shell Functions ansible.builtin.blockinfile: path: "{{ user_user.home }}/{{ item }}" marker: "# {mark} ANSIBLE MANAGED BLOCK | ollama | Manual Script" content: | function llm_installs() { {% for llm in llm_installs %} echo "{{ llm.model }}" {% endfor %} } function llm_removals() { {% for llm in llm_removals %} echo "{{ llm.model }}" {% endfor %} } function ollama_pull() { echo "Manually Pulling Models" for model in `llm_installs`; do echo -e "\n\n Starting 'ollama pull $model' \n\n" ollama pull $model status=$? echo "Finished with status $status." done echo "Done!!" } alias pull-llms="ollama_pull " alias pull-models="ollama_pull " alias pull-ai="ollama_pull " function ollama_clean() { echo "Cleaning Models" for model in `llm_removals`; do echo -e "\n Removing '$model'" ollama rm $model done echo "Done!" } alias ollama-clean="ollama_clean " alias clean-ollama="ollama_clean " alias clean-llms="ollama_clean " alias clean-models="ollama_clean " alias clean-ai="ollama_clean " function ollama_chat() { ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen3.5:4b' }} } alias ollama-chat="ollama_chat " alias chat="ollama_chat " function ollama_code() { ollama run {{ 'qwen3.8:27b' if ansible_memtotal_mb > 32000 else 'qwen2.5-coder:7b' }} } alias ollama-code="ollama_code " function ollama_purge() { echo "TBD: Purge all models not in llm_installs." } alias ollama-purge="ollama_purge " alias ollama-clean="ollama_purge " alias ollama-remove="ollama_purge " alias ollama-rm="ollama_purge " alias remove-models="ollama_purge " alias rm-models="ollama_purge " alias purge-models="ollama_purge " function restart_ai() { echo "Restarting AI Software" for sv in ollama zeroclaw open-webui open-terminal qwen-code swap-stopper; do echo -e "\n $sv \n" sudo sv restart $sv done; } alias restart-ai="restart_ai " alias restart-llm="restart_ai " alias restart-llms="restart_ai " alias restart-models="restart_ai " alias reload-ai="restart_ai " state: present create: false backup: true loop: - .bashrc - .zshrc - name: Workstation | Linux | Software | LLMs | Models | ollama | Manage block: - name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Message ansible.builtin.debug: msg: <- The next task may take hours, if not days. If you'd like a status bar to know what's being worked on and its progress, I highly recommed reloading your RC file and running `pull-models`. That will give you a much better idea of what's going on. - name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Install ansible.builtin.shell: | set -o pipefail item='{{ item.model }}' if ollama list | grep -q $item; then echo "OKAY" && return else ollama pull $item 1>/dev/null && echo "INSTALLED" && return fi echo "FAILED" loop: "{{ llm_installs }}" register: shell_check changed_when: "'INSTALLED' in shell_check.stdout" failed_when: "'FAILED' in shell_check.stdout" become: true become_user: "{{ user }}" when: manage_models and llm_installs | length > 0 - name: Workstation | Linux | Software | LLMs | Models | ollama | Manage | Remove ansible.builtin.shell: | set -o pipefail item='{{ item.model }}' if ollama list | grep -q $item; then ollama rm $item && echo "INSTALLED" && return else echo "OKAY" && return fi echo "FAILED" loop: "{{ llm_removals }}" register: shell_check changed_when: "'INSTALLED' in shell_check.stdout" failed_when: "'FAILED' in shell_check.stdout" become: true become_user: "{{ user }}" when: llm_removals | length > 0 when: manage_models - name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models ansible.builtin.shell: cmd: | set -o pipefail ollama ls | tail -n +2 | awk '{print $1}' register: llm_models changed_when: llm_models.rc == 0 - name: Workstation | Linux | Software | LLMs | Models | ollama | Get Models | Result ansible.builtin.debug: var: llm_models # Cheat Sheet: https://docs.zeroclaw.com/master/en/providers/configuration.html - name: Workstation | Linux | Software | LLMs | Models | zeroclaw | Add Providers ansible.builtin.blockinfile: path: "{{ user_user.home }}/.zeroclaw/config.toml" marker: "# {mark} ANSIBLE MANAGED BLOCK | zeroclaw | Providers" insertafter: EOF block: | {% for llm in llm_installs %} {% if llm.model in llm_models.stdout %} {% if "chat" in llm.purpose %} [providers.models.ollama.{{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT' }}{{ '' if llm.model in llm_models.stdout else '-MISSING' }}] model = "{{ llm.model }}" timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though. context_window = {{ [llm.context] | min | int }} max_tokens = {{ [(llm.context / 8), 16000] | min | int }} {% endif %} {% endif %} {% endfor %} state: present create: false backup: true when: zeroclaw_enabled - name: Workstation | Linux | Software | LLMs | Models | Continue | Config ansible.builtin.copy: dest: "{{ user_user.home }}/.continue/config.yaml" content: | # Maintained by `env-ansible/tasks/workstation/linux/software/llm.yml` name: Main Config version: 1.0.0 schema: v1 models: # | Role | `contextLength` | `maxTokens` | Notes | # |------|-----------------|-------------|-------| # | Chat / Edit | 131072 (128K) | 16384 - 65536 | Qwen recommends 65,536 output for instruct models; use lower if VRAM is tight | # | Autocomplete | 32768 - 65536 | 1024 - 4096 | Keep low for speed; autocomplete doesn't need long outputs | {% for llm in llm_installs %} {% if llm.model in llm_models.stdout %} {% if "code" in llm.purpose %} - name: {{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT' }}{{ '' if llm.model in llm_models.stdout else '-MISSING' }} provider: ollama model: {{ llm.model }} defaultCompletionOptions: contextLength: {{ [llm.context] | min | int }} maxTokens: {{ [(llm.context / 8), 16000] | min | int }} {% endif %} {% endif %} {% endfor %} mode: "0644" when: codium_config #- name: {{ llm.provider }}_AUTO{{ '' if llm.model in llm_models.stdout else '-MISSING' # }} # provider: ollama # model: {{ llm.model }} # defaultCompletionOptions: # contextLength: {{ [llm.context] | min | int }} # maxTokens: {{ [(llm.context / 8), 4000] | min | int }} # roles: # - autocomplete - name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config ansible.builtin.copy: dest: "{{ item }}" content: | { "$version": 4, "env": { "OLLAMA_API_KEY": "ollama" }, "model": { "name": "qwen2.5-coder:3b", "baseUrl": "http://127.0.0.1:11434/v1" }, "modelProviders": { "openai": [ {% for llm in llm_installs %} { "name": "{{ llm.provider }}_{{ 'CODE' if "code" in llm.purpose else 'CHAT' }}{{ '' if llm.model in llm_models.stdout else '-MISSING' }}" ,"baseUrl": "http://127.0.0.1:11434/v1" ,"envKey": "OLLAMA_API_KEY" ,"id": "{{ llm.model }}" ,"generationConfig": { "timeout": 600000 ,"streamIdleTimeoutMs": 600000 ,"contextWindowSize": {{ [llm.context] | min | int }} ,"maxRetries": 3 } }{{ ',' if not loop.last else '' }} {% endfor %} ] }, "security": { "auth": { "baseUrl": "http://127.0.0.1:11434/v1", "selectedType": "openai" } }, "ui": { "autoModeAcknowledged": true } } mode: "0644" loop: - "{{ user_user.home }}/.qwen/settings.json" # Not Needed #- "{{ user_user.home }}/Code/.qwen/settings.json" when: qwen_code_enabled #"reasoning": { # "effort": "medium" #}, #"samplingParams": { # "max_tokens": {{ [(llm.context / 8), 16000] | min | int }} # ,"temperature": 0.7 # ,"top_p": 0.8 #} - name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config | Validate ansible.builtin.shell: | set -o pipefail cat {{ user_user.home }}/.qwen/settings.json | jq register: shell_check changed_when: shell_check.rc == 0 failed_when: shell_check.rc != 0 when: qwen_code_enabled # Models - End # when: ansible_pkg_mgr == "pacman"