diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index 7c7af21..77063be 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -197,25 +197,33 @@ dest: "{{ ollama_runit_script }}" content: | #!/bin/sh -e - # Allow pulling and accessing LLMs via tools. exec 2>&1 + # Allow pulling and accessing LLMs via tools. + ### Required ### export HOME={{ user_user.home }} + ### Timeouts ### export OLLAMA_LOAD_TIMEOUT=20m # default is 5m export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars) + ### Resource Usage ### export OLLAMA_KEEP_ALIVE=20m # default is 5m export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed) export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size export OLLAMA_MAX_QUEUE=999 # default is 512 (unconfirmed) + ### Prevent Data Loss ### # OLLAMA_NOPRUNE # Default is 0, which causes unfinished pulls to be deleted # after 1 hour of inactivity. To manually clean partials, run: # `rm ~/.ollama/models/blobs/*-partial*` export OLLAMA_NOPRUNE=1 + + ### Tokens ### + {{ ("export HF_TOKEN=" ~ hf_token) if hf_token else '' }} + exec chpst -u {{ user }} ollama serve mode: "0755"