No introduction found. Create it?
Install with:
helm repo add litellm oci://ghcr.io/berriai/litellm-helm
helm install litellm litellm/litellm -f values.yamlSee examples from other people.
| Name | Repo | Stars | Version | Timestamp |
|---|---|---|---|---|
| litellm | ToaHartor/maisonneux | 43 | 1.106.2 | 2 months ago |
See the most popular values for this chart:
| Key | Types |
|---|---|
| string | |
| string | |
controllers.litellm.containers.app.image.repository (11) ghcr.io/berriai/litellm | string |
controllers.litellm.containers.app.image.tag (11) v1.108.4@sha256:17921779153960f086e2660994cc398fc76ed65c531765177803b9ae0a25d97a | string |
| boolean | |
| boolean | |
| string | |
| number | |
| number | |
| number | |
| number | |
| number | |
| boolean | |
| boolean | |
| string | |
| number | |
| number | |
| number | |
| number | |
| number | |
| boolean | |
| number | |
| number | |
controllers.litellm.containers.app.probes.startup.spec.httpGet.path (3) /health/liveliness | string |
| number | |
| number | |
| number | |
| boolean | |
| string | |
| string | |
| string | |
| string | |
| string | |
controllers.litellm.containers.app.args[] (10) - --config=/app/config.yaml | string |
| string | |
| boolean | |
| string | |
| string | |
controllers.litellm.containers.app.env.REDIS_HOST (7) litellm-dragonfly | string |
| number, string | |
controllers.litellm.containers.app.env.TZ (4) ${CLUSTER_TZ} | string |
| string | |
| string | |
controllers.litellm.containers.app.env.DATABASE_URL (1) postgresql://litellm@pooler-rw.database.svc:5432/litellm?sslmode=require&sslidentity=/var/run/secrets/prisma/client-identity.p12&sslcert=/var/run/secrets/root-ca/ca.crt | string |
controllers.litellm.containers.app.env.GENERIC_AUTHORIZATION_ENDPOINT (3) https://sso.${SECRET_DOMAIN}/application/o/authorize/ | string |
controllers.litellm.containers.app.env.GENERIC_SCOPE (3) openid email profile litellm_role | string |
controllers.litellm.containers.app.env.GENERIC_TOKEN_ENDPOINT (3) https://sso.${SECRET_DOMAIN}/application/o/token/ | string |
| string | |
controllers.litellm.containers.app.env.GENERIC_USERINFO_ENDPOINT (3) https://sso.${SECRET_DOMAIN}/application/o/userinfo/ | string |
controllers.litellm.containers.app.env.PROXY_BASE_URL (3) https://litellm.${SECRET_DOMAIN} | string |
| string | |
| string | |
controllers.litellm.containers.app.env.PROMETHEUS_URL (2) http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090 | string |
| boolean | |
| string | |
| string | |
| string | |
| string | |
| string | |
| string | |
controllers.litellm.containers.app.env.GENERIC_USER_ID_ATTRIBUTE (1) preferred_username | string |
| string | |
| string | |
| string | |
| string | |
| string | |
| string | |
| number | |
| string | |
| string | |
controllers.litellm.containers.app.env.OTEL_ENDPOINT (1) http://alloy-gateway.observability.svc.cluster.local:4317 | string |
| string | |
| string | |
controllers.litellm.containers.app.env.PGSSLCERT (1) /var/run/secrets/postgresql/tls.crt | string |
controllers.litellm.containers.app.env.PGSSLKEY (1) /var/run/secrets/postgresql/tls.key | string |
controllers.litellm.containers.app.env.PGSSLROOTCERT (1) /var/run/secrets/root-ca/ca.crt | string |
| string | |
| string | |
controllers.litellm.containers.app.env.REDIS_URL (1) redis://dragonfly.database.svc.cluster.local:6379/11 | string |
| string | |
| string | |
| boolean | |
| string | |
| boolean | |
| number | |
| boolean | |
| number | |
controllers.litellm.containers.exporter.args[] (1) - --no-collector.wal | string |
| string | |
| string | |
controllers.litellm.containers.exporter.env.DATA_SOURCE_URI (1) litellm-db-rw.ai.svc.cluster.local:5432/litellm?sslmode=require | string |
| string | |
| string | |
| string | |
| string | |
controllers.litellm.containers.exporter.env.PG_EXPORTER_EXTEND_QUERY_PATH (1) /etc/pg-exporter/queries.yaml | string |
controllers.litellm.containers.exporter.image.repository (1) harbor.webgrip.dev/quay/prometheuscommunity/postgres-exporter | string |
controllers.litellm.containers.exporter.image.tag (1) v0.20.1@sha256:ac5ec343104fae0e2d84a27bb8d69b38430a11910c5382cad85d478d2bab713e | string |
| boolean | |
| boolean | |
| string | |
| number | |
| number | |
| number | |
| boolean | |
| boolean | |
| string | |
| number | |
| number | |
| number | |
| string | |
| string | |
| string | |
| boolean | |
| string | |
| boolean | |
| number | |
| string | |
controllers.litellm.containers.speaches.env.TZ (1) America/Chicago | string |
| string | |
| string | |
controllers.litellm.containers.speaches.image.repository (1) ghcr.io/speaches-ai/speaches | string |
controllers.litellm.containers.speaches.image.tag (1) 0.8.3-cpu@sha256:21e3df06d842fb7802ab470dd77c25f0e8c0d22950e8d8c6ae886e851af53ef8 | string |
| boolean | |
| boolean | |
| number | |
| string | |
| number | |
| number | |
| number | |
| boolean | |
| boolean | |
| number | |
| string | |
| number | |
| number | |
| number | |
| string | |
| string | |
| string | |
controllers.litellm.strategy (5) RollingUpdate | string |
| number | |
controllers.litellm.initContainers.copy-prisma-cache.command[] (2) - sh | string |
controllers.litellm.initContainers.copy-prisma-cache.image.repository (2) ghcr.io/berriai/litellm-non_root | string |
controllers.litellm.initContainers.copy-prisma-cache.image.tag (2) v1.108.0@sha256:08d313a00c46509c875d84f60c1448a260c478f399ade3c2abb940d358d5b724 | string |
| string | |
controllers.litellm.initContainers.init-db.image.repository (2) ghcr.io/home-operations/postgres-init | string |
controllers.litellm.initContainers.init-db.image.tag (2) 18.6.0@sha256:87627baaddb0cabed4ebe033e75038e5c7b6b081c18e86514c99067179a90719 | string |
| string | |
controllers.litellm.initContainers.01-init-db.image.repository (1) ghcr.io/home-operations/postgres-init | string |
| string | |
controllers.litellm.initContainers.gen-prisma-cert.command[] (1) - /bin/sh | string |
controllers.litellm.initContainers.gen-prisma-cert.image.repository (1) docker.io/library/alpine | string |
controllers.litellm.initContainers.gen-prisma-cert.image.tag (1) 3.24.4@sha256:f59abe656b692d901e184be3445e084786602108599d2a51d229c0f5a48c71bf | string |
| number | |
| number | |
| string | |
| string | |
| string | |
| number | |
| string | |
| string | |
| string | |
| string | |
| string | |
| number | |
controllers.litellm.pod.topologySpreadConstraints[].topologyKey (1) kubernetes.io/hostname | string |
| string | |
controllers.prune-prompts.containers.app.command[] (1) - /bin/sh | string |
| string | |
controllers.prune-prompts.containers.app.env.DATABASE_URL.valueFrom.secretKeyRef.name (1) {{ .Release.Name }}-postgres | string |
| boolean | |
controllers.prune-prompts.containers.app.env.PGAPPNAME (1) litellm-prune-prompts | string |
| string | |
controllers.prune-prompts.containers.app.image.repository (1) ghcr.io/cloudnative-pg/postgresql | string |
controllers.prune-prompts.containers.app.image.tag (1) 18.6-minimal-trixie@sha256:ec2ed3404e89fff99643901b58a2a34c46a46345fe69a3d96500b82f995c327b | string |
| string | |
| string | |
| string | |
| boolean | |
| string | |
| boolean | |
| number | |
| boolean | |
| number | |
| number | |
| string | |
| number | |
| string | |
| number | |
controllers.prune-prompts.cronjob.timeZone (1) America/Chicago | string |
| number | |
| string | |
| string | |
| number | |
| boolean | |
| number | |
| number | |
| number | |
service.app.controller (7) litellm | string |
service.port (1) ${GATEWAY_SVC_PORT} | string |
persistence.config.type (9) configMap | string |
persistence.config.name (8) litellm-configmap | string |
persistence.config.globalMounts[].path (7) /app/config.yaml | string |
| boolean | |
| string | |
persistence.config.advancedMounts.litellm.app[].path (2) /app/config.yaml | string |
| boolean | |
| string | |
persistence.config.advancedMounts.prune-prompts.app[].path (1) /app/prune-prompts.sh | string |
| boolean | |
| string | |
| string | |
persistence.cache.type (4) emptyDir | string |
| string | |
| string | |
| string | |
| string | |
| boolean | |
| string | |
persistence.data.globalMounts[].path (2) /app/.config | string |
| string | |
| string | |
| string | |
| string | |
persistence.config-file.globalMounts[].path (1) /etc/litellm/config.yaml | string |
| boolean | |
| string | |
persistence.config-file.name (1) litellm-settings | string |
persistence.config-file.type (1) configMap | string |
| string | |
| boolean | |
persistence.exporter-queries.name (1) litellm-exporter-queries | string |
| string | |
persistence.models.accessMode (1) ReadWriteOnce | string |
persistence.models.advancedMounts.litellm.speaches[].path (1) /home/ubuntu/.cache/huggingface/hub | string |
persistence.models.forceRename (1) litellm-models | string |
| string | |
persistence.models.storageClass (1) openebs-hostpath | string |
persistence.models.type (1) persistentVolumeClaim | string |
persistence.prisma-certs.advancedMounts.litellm.app[].path (1) /var/run/secrets/prisma | string |
persistence.prisma-certs.advancedMounts.litellm.gen-prisma-cert[].path (1) /var/run/secrets/prisma | string |
| string | |
persistence.runtime-patches.advancedMounts.litellm.app[].path (1) /app/.venv/lib/python3.13/site-packages/sitecustomize.py | string |
| boolean | |
| string | |
persistence.runtime-patches.name (1) litellm-runtime-patches | string |
persistence.runtime-patches.type (1) configMap | string |
| string | |
| string | |
persistence.tmpfs.type (1) emptyDir | string |
route.app.hostnames[] (8) - ${APP}.${SECRET_INTERNAL_DOMAIN} | string |
route.app.parentRefs[].name (8) envoy-internal | string |
route.app.parentRefs[].namespace (8) networking | string |
| string | |
| string, number | |
| string | |
| string | |
route.app.rules[].matches[].path.type (1) PathPrefix | string |
route.app.rules[].matches[].path.value (1) /v1 | string |
| string | |
| string | |
| string | |
route.app.annotations."gethomepage.dev/icon" (3) https://cdn.jsdelivr.net/gh/selfhst/icons@main/svg/litellm.svg | string |
| string | |
| string | |
route.app.annotations."gatus.home-operations.com/endpoint" (1) guarded: false
url: https://litellm.homeops.ca/health/liveliness
conditions: ["[STATUS] == 200"] | string |
| boolean | |
db.database (1) ${PSQL_DB_NAME} | string |
| string | |
db.secret.name (1) ${PSQL_DB_USER}-db-creds | string |
db.secret.passwordKey (1) password | string |
db.secret.usernameKey (1) username | string |
| boolean | |
| number | |
| string | |
| number | |
| boolean | |
| number | |
| string | |
| string | |
| string | |
| string | |
| string | |
masterkeySecretKey (3) masterkey | string |
masterkeySecretName (3) litellm-env | string |
proxy_config.general_settings.master_key (3) os.environ/PROXY_MASTER_KEY | string |
| string | |
proxy_config.model_list[].litellm_params.model (3) github_copilot/* | string |
proxy_config.model_list[].litellm_params.extra_headers.anthropic-beta (2) context-1m-2025-08-07 | string |
proxy_config.model_list[].litellm_params.api_base (1) http://qwen38.ai.svc.cluster.local:8080/v1 | string |
proxy_config.model_list[].litellm_params.api_key (1) sk-no-key-required | string |
| boolean | |
proxy_config.model_list[].model_info.cache_creation_input_token_cost (3) 0 | number |
proxy_config.model_list[].model_info.cache_read_input_token_cost (3) 0 | number |
| number | |
| number | |
| number | |
| number | |
| boolean | |
| boolean | |
proxy_config.model_list[].model_name (3) github_copilot/* | string |
proxy_config.litellm_settings.callbacks[] (2) - websearch_interception | string |
| boolean | |
| boolean | |
| boolean | |
| string | |
| string | |
proxy_config.router_settings.default_fallbacks[] (2) - claude-sonnet-5 | string |
proxy_config.search_tools[].litellm_params.api_key (2) os.environ/TAVILY_API_KEY | string |
| string | |
proxy_config.search_tools[].search_tool_name (2) tavily-search | string |
environmentSecrets[] (2) - litellm-env | string |
envVars.GITHUB_COPILOT_TOKEN_DIR (2) /cache/copilot | string |
| string | |
envVars.DOCS_URL (1) /docs | string |
envVars.GENERIC_AUTHORIZATION_ENDPOINT (1) https://${auth_subdomain}.${main_domain_websecure}/application/o/authorize/ | string |
envVars.GENERIC_SCOPE (1) openid profile email litellm_role | string |
envVars.GENERIC_TOKEN_ENDPOINT (1) https://${auth_subdomain}.${main_domain_websecure}/application/o/token/ | string |
envVars.GENERIC_USER_ROLE_ATTRIBUTE (1) litellm_role | string |
envVars.GENERIC_USERINFO_ENDPOINT (1) https://${auth_subdomain}.${main_domain_websecure}/application/o/userinfo/ | string |
envVars.PROXY_BASE_URL (1) https://${SUBDOMAIN}.${DOMAIN_WEBSECURE} | string |
envVars.PROXY_LOGOUT_URL (1) https://${auth_subdomain}.${main_domain_websecure}/application/o/${APP}/end-session/ | string |
envVars.REDIS_URL (1) redis://dragonfly-cluster.dragonfly.svc.cluster.local:6379/3 | string |
| string | |
image.repository (2) ghcr.io/berriai/litellm | string |
| boolean | |
volumeMounts[].mountPath (2) /cache | string |
volumeMounts[].name (2) litellm-cache | string |
volumeMounts[].subPath (2) access-token | string |
volumes[].name (2) litellm-cache | string |
volumes[].secret.items[].key (2) GITHUB_COPILOT_ACCESS_TOKEN | string |
volumes[].secret.items[].path (2) access-token | string |
volumes[].secret.secretName (2) litellm-env | string |
args[] (1) - --config | string |
configMaps.config.data."config.yaml" (1) model_list:
# --- primary: DGX Spark (spark-ab23) over the LAN ---
# One engine on the Spark since 2026-10-05: vLLM (:8000), serving
# Qwen3.6 for every `spark-*` text and vision route. (Ollama on
# :11434 served gpt-oss, then only vision, and is retired.) It is
# OpenAI-compatible at /v1 and has no auth of its own: it ignores the
# bearer token, so api_key is a placeholder, not a secret. LiteLLM
# is the authentication boundary in front of it; it is
# reachable by anything on VLAN 10.
#
# Addressed by IP, deliberately, after the name stopped resolving.
#
# This was http://spark-ab23.${SECRET_DOMAIN}:11434/v1 and that
# worked when it was set. Hours later the same lookup returned
# NODATA from the UDM (NOERROR, no A record) while the BARE
# `spark-ab23` still answered 10.0.10.237 — i.e. UniFi's automatic
# DHCP-hostname registration survived but the explicit
# derekjacobs.dev record did not. Every Hindsight call failed as
# `litellm.InternalServerError: Connection error` until this change.
#
# The bare name is not a usable substitute: with ndots:5 the
# cluster search path swallows a single-label name, and it resolves
# only as `spark-ab23.` with a trailing dot, which not every HTTP
# client normalises.
#
# Using the IP is SAFE now in a way it was not originally. The
# first version of this file used 10.0.10.237 and carried a warning
# that it was a DHCP lease which could move silently. It is now a
# fixed assignment on the UDM, so that failure mode is gone.
#
# Switch back to the name once the A record is restored and
# verified resolving FROM A POD, not just from the Spark — the
# Spark resolves names the cluster cannot.
#
# Served by vLLM on :8000 (spark-setup, roles/vllm), not Ollama,
# since 2026-09. vLLM pages its KV cache per request instead of
# preallocating 8 fixed 32k slots, so there is no slot cap to
# queue behind: measured on an idle Spark, 226 vs 153 tok/s
# aggregate at 8 concurrent, and 385 at 16 where Ollama queued
# requests 26s for a slot. It is ~25% slower for a LONE request
# (46 vs 61 tok/s) -- the trade taken deliberately.
#
# Like Ollama, vLLM runs with no --api-key: the placeholder below
# is sent and ignored, and LiteLLM stays the auth boundary.
#
# num_retries: ~4.7% of replayed reflect tool calls (3/64) came back
# 500 "unexpected tokens remaining in message header" -- gpt-oss
# intermittently emits a malformed harmony tool-call header that
# vLLM's parser rejects. The same request succeeds on retry, and
# hindsight's own retries cover only its background worker jobs, so
# without this an interactive reflect surfaces the 500. Two retries
# take it to ~1 in 10,000.
#
# ⚠️ Since 2026-10-05 the model on the Spark is Qwen3.6-35B-A3B
# (spark-setup group_vars: vllm_model), not gpt-oss-20b. The routes
# named after gpt-oss (`spark-gpt-oss-20b`, `spark-gpt-oss-20b-long`)
# were legacy aliases of `spark-main` from that day and were deleted
# on 2026-10-12; see the route comment below. Everything above about
# vLLM versus Ollama still holds; the gpt-oss harmony-header note
# does not apply to Qwen, and num_retries stays as cheap insurance.
#
# Why the model changed: replaying real hindsight calls, gpt-oss
# returned {"facts": []} on 24 of 50 retains where Qwen returned
# facts on all 50, and production showed the same (268 of 831
# successful retains in 72h stored nothing). Numbers are in
# spark-setup group_vars/all.yml.
#
# Thinking is OFF by default on the server
# (--default-chat-template-kwargs). With it on, the same retain took
# 118s instead of 13s. ⚠️ `reasoning_effort` OVERRIDES that default:
# any value but "none" turns thinking on for that request ("low"
# produced 508 reasoning tokens for a one-sentence answer, "none"
# produced 0). So a consumer tuned for gpt-oss with
# reasoning_effort: low is asking Qwen to think. An explicit
# `chat_template_kwargs: {"enable_thinking": false}` wins over both.
#
# --- the routes: spark-main, spark-main-vision ---
# Model-neutral on purpose. The first routes here were named after
# the models the Spark ran (spark-gpt-oss-20b, spark-gpt-oss-20b-long,
# spark-qwen25-vl-7b), which is what naming a route after its model
# gets you after one swap: three stale names aliasing one backend.
# These say what the route is FOR, so the next model change touches
# spark-setup and this file and no consumer.
#
# The three old names were kept as aliases from 2026-10-05 and
# deleted on 2026-10-12. Every consumer in git and mealie (admin UI)
# had moved on 2026-10-05; the Hermes agent, the suspected holdout,
# turned out to talk to Anthropic and Nous directly rather than
# through LiteLLM; and LiteLLM_DailyUserSpend grouped by model_group
# showed no request on any of the three after 2026-10-05 (the last
# were karakeep and hindsight mid-move that day). The virtual keys
# that still listed the old names had them stripped the same day.
#
# The -long name existed because reflect once needed a separate
# one-slot 98k ollama on :11435; each vLLM replica serves the
# full 131,072-token window to any request (an ~87k-token prompt
# measured 18s to first token alongside short traffic), so one
# route covers reflect too.
#
# spark-main text. Falls back to the hosted copy, then to
# Haiku, when BOTH replicas are down
# (router_settings.fallbacks).
# spark-main-vision the same model, for requests carrying
# images. NO fallback, so a photo never leaves
# the house: that is the whole reason it is a
# separate name.
#
# Both are thinking-off by default like every Spark route; see the
# reasoning_effort warning above.
#
# TWO deployments per Spark route since 2026-10-26: spark-ab23
# (10.0.10.237) and gx10-7ac6 (10.0.10.117) each run their own
# vLLM replica of the same model, image and flags (spark-setup
# `vllm_serving`; the GX10 reads the weights from spark-ab23
# over NFS). LiteLLM's default simple-shuffle spreads requests
# across a model group's deployments. On an error, the
# deployment's own num_retries go to the SAME box; then
# `enable_weighted_failover` (router_settings) re-picks the other
# deployment of the group, and only when both have failed do the
# route's fallbacks fire. Without that flag a failed deployment
# goes straight to the fallbacks (Nous, then Haiku: paid) while
# the other box sits healthy -- the LiteLLM default, caught in
# the #2308 review.
#
# A replica that HANGS (accepts, never answers) is not an error,
# so failover does not see it: requests that land there wait out
# LiteLLM's default timeout. A per-deployment timeout would fix
# that, but thinking-on hindsight page refreshes legitimately run
# for minutes (32k-token cap; hindsight itself waits 600s), so it
# needs a measured ceiling first, not a guess.
#
# ⚠️ This covers a restart or a dead ENGINE on spark-ab23, not a
# spark-ab23 HOST outage: the GX10 loads its weights from
# spark-ab23 over NFS, so with that host down the GX10 replica
# keeps serving only until it restarts, and a hard NFS mount can
# hang its file reads meanwhile.
#
# Each replica keeps its own prefix cache (--enable-prefix-caching).
# A shared system prompt costs one cold miss per replica, but a
# multi-turn call (hindsight reflect re-sends the whole growing
# conversation each tool turn) lands on the cold replica about
# half the time and pays the full prefill again. NOT measured yet:
# spark-ab23 alone served 28% of prompt tokens from cache
# (4.64M of 16.39M, 2026-10-26). Compare
# spark:vllm_cached_prompt_tokens / spark:vllm_prompt_tokens a
# day after this change; if it drops well below that, look at
# prefix-aware routing before pinning hindsight-main to one box.
- model_name: spark-main
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.237:8000/v1
api_key: unused-no-auth
num_retries: 2
- model_name: spark-main # gx10-7ac6 replica, see spark-main
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.117:8000/v1
api_key: unused-no-auth
num_retries: 2
# hindsight's own name for the SAME Spark backend. It exists so the
# fallback chain CAN differ: `router_settings.fallbacks` is per
# route name and all-or-nothing, and the deployed LiteLLM does NOT
# check a fallback target against the calling key's model list (the
# hindsight key lists only spark-main and still reached Nous 123
# times on 2026-10-24; #2280 review). The paid Anthropic stage was
# attached to this name alone at first; later the same day
# spark-main got it too while Nous is out of credit (~2026-11-05),
# so today the two chains are identical. The name stays: it is
# cheap, and dropping Haiku from spark-main again once Nous has
# credit is a one-line change that leaves hindsight's chain alone.
# The "one more rename" the privacy note describes.
- model_name: hindsight-main
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.237:8000/v1
api_key: unused-no-auth
num_retries: 2
- model_name: hindsight-main # gx10-7ac6 replica, see spark-main
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.117:8000/v1
api_key: unused-no-auth
num_retries: 2
# The SAME Spark backend a third time, as the fallback TARGET for
# the Anthropic routes (router_settings.fallbacks). It must be a
# name with no fallbacks of its own: the two Spark chains above end
# in claude-haiku-5-5, and LiteLLM nests fallbacks (run_async_fallback
# recurses up to max_fallbacks=5), so pointing claude-haiku-5-5 at
# hindsight-main would, with the Spark down and the Anthropic budget
# spent, bounce haiku -> hindsight-main -> nous -> haiku -> ... five
# deep before failing. Nothing calls this name directly and no key
# lists it; fallback targets are not key-filtered.
- model_name: spark-direct
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.237:8000/v1
api_key: unused-no-auth
num_retries: 2
- model_name: spark-direct # gx10-7ac6 replica, see spark-main
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.117:8000/v1
api_key: unused-no-auth
num_retries: 2
# Vision: Qwen3.6 reads images itself. Checked side by side with the
# retired qwen2.5vl on five images: same or better tags, and a
# rendered receipt transcribed exactly. Images are capped at ~1 MP
# and FOUR PER REQUEST on the server (spark-setup vllm_extra_args:
# --limit-mm-per-prompt, --mm-processor-kwargs); a request with more
# gets a 400. homebox-companion's multi-photo detection is the
# consumer to check against that cap. vLLM fetches an http(s) image
# URL as well as taking a base64 `data:` URI; consumers send bytes.
#
# Ollama is RETIRED on the Spark (stopped and disabled, 2026-10-05;
# spark-setup ollama_enabled). The binary and qwen2.5vl:7b are still
# on disk, so the rollback to a separate vision model is two steps:
# ollama_enabled: true with `--tags inference` there, then point this
# at model: openai/qwen2.5vl:7b, api_base: http://10.0.10.237:11434/v1.
# qwen2.5vl is DENSE, measured 20 GiB resident at OLLAMA_NUM_PARALLEL=8
# (1.3-2.7s warm per image): see the memory hazard below before
# bringing it back. Ollama does NOT fetch remote image URLs, only the
# base64 `data:` URI shape worked there.
- model_name: spark-main-vision
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.237:8000/v1
api_key: unused-no-auth
num_retries: 2
- model_name: spark-main-vision # gx10-7ac6 replica, see spark-main
litellm_params:
model: openai/qwen3.6-35b-a3b
api_base: http://10.0.10.117:8000/v1
api_key: unused-no-auth
num_retries: 2
# --- audio (spark-setup `audio` role) ---
# Two plain Docker containers on the Spark, NOT Ollama, so they use
# their own ports and the `mode` below tells LiteLLM which OpenAI
# audio route each one serves. Like Ollama they have no auth of their
# own; LiteLLM is the boundary. Round trip verified on the box: TTS ->
# WAV -> STT returned the synthesised sentence.
#
# Speech-to-text: parakeet.cpp (NVIDIA Parakeet TDT 0.6B v3,
# multilingual). An 11 s clip transcribed word-for-word in ~0.3 s warm.
# This is the model Mealie's "audio" provider slot should point at.
# The `openai/` prefix + api_base makes LiteLLM speak the OpenAI wire
# format to it; the part after `openai/` is only what LiteLLM sends as
# `model`, which parakeet accepts and ignores.
- model_name: spark-stt
litellm_params:
model: openai/tdt-0.6b-v3
api_base: http://10.0.10.237:8081/v1
api_key: unused-no-auth
model_info:
mode: audio_transcription
# Text-to-speech: Qwen3-TTS 1.7B voice-clone. The `voice` field selects
# an entry in voices.json on the Spark (a reference clip + its exact
# transcript), which is how character voices are added -- see the
# spark-setup README. ~7 s warm for a ~5 s sentence, so this is fine
# for notifications and read-aloud, not for real-time conversation.
#
# ⚠️ Streams the WAV, so the header's length field is bogus (a 4.8 s
# clip reports ~89,000 s). Players cope; strict parsers may not. Ask for
# response_format=mp3 (encoded after generation, so the header is right).
- model_name: spark-tts
litellm_params:
model: openai/tts-1
api_base: http://10.0.10.237:8082/v1
api_key: unused-no-auth
model_info:
mode: audio_speech
# --- embeddings + reranking (spark-setup PR #28, `embeddings` role) ---
# Qwen3-Embedding-0.6B (1024-d, L2-normalised, 8192-token input) and
# Qwen3-Reranker-0.6B, one copy per box like the text routes, so
# LiteLLM's default shuffle spreads load across both. Consumer: the
# mail-archiver semantic-search indexer (t_8f3a8cc8). No auth on the
# engines; LiteLLM is the boundary, as everywhere here.
#
# ⚠️ NO RATE CAP LIVES ON THESE ROUTES. Uncapped, a backfill halves
# Qwen3.6 decode on the box it hits (measured ~65-73 emb/s per box
# at ~500 tokens; ~35 emb/s, about 1M tokens/min, costs Qwen3.6
# ~24%). A per-deployment `tpm` here would NOT help: router_settings
# uses the default simple-shuffle, where tpm/rpm only weight the
# random pick and reject nothing (caught in the #2470 review).
# The cap is `tpm_limit` on the indexer's virtual key (enforced via
# coordination_redis), about 2M tokens/min across both boxes, so
# set it when creating the key. Bulk consumers must ALSO throttle
# themselves or run off-hours.
#
# ⚠️ The reranker needs the Qwen3 prompt template (instruction +
# <Query>/<Document> wrapping) in the client's strings. The route
# through LiteLLM has not been exercised yet: rerun the 3-document
# check after this lands.
- model_name: spark-embed
litellm_params:
model: openai/qwen3-embedding-0.6b
api_base: http://10.0.10.237:8010/v1
api_key: unused-no-auth
num_retries: 2
model_info:
mode: embedding
- model_name: spark-embed # gx10-7ac6 replica, see above
litellm_params:
model: openai/qwen3-embedding-0.6b
api_base: http://10.0.10.117:8010/v1
api_key: unused-no-auth
num_retries: 2
model_info:
mode: embedding
- model_name: spark-rerank
litellm_params:
model: hosted_vllm/qwen3-reranker-0.6b
api_base: http://10.0.10.237:8011
api_key: unused-no-auth
model_info:
mode: rerank
- model_name: spark-rerank # gx10-7ac6 replica, see above
litellm_params:
model: hosted_vllm/qwen3-reranker-0.6b
api_base: http://10.0.10.117:8011
api_key: unused-no-auth
model_info:
mode: rerank
# --- removed 2026-09-29: spark-qwen25-14b (qwen2.5:14b-instruct) ---
# A dense non-reasoning alternative to gpt-oss that nothing used: it
# was reflect's escape hatch until reflect needed a ~92k window, and it
# was never faster (26 tok/s vs gpt-oss's 63 -- dense loses to MoE on
# a bandwidth-bound box). Removed for the memory hazard below.
#
# ⚠️ MEMORY HAZARD, still true for anything served by Ollama. It runs
# with OLLAMA_KEEP_ALIVE=-1, so ANY model touched through this proxy
# stays resident forever, and its KV cache is slots x context per
# model: one evaluation request against the dense 14B pinned 73 GiB
# at 4 slots and took the box to 103 of 121 GiB, with hindsight
# operations timing out into PERMANENT failures. With vLLM now
# holding a fixed ~36 GiB, the same load would reach earlyoom, whose
# largest preferred target is vLLM. Before adding a dense model here,
# size slots x context x its per-token KV against what is free.
# --- fallback: the SAME model, hosted (Nous Portal) ---
# Used only when the Spark fails a request after its retries; see
# router_settings.fallbacks below. It is the same Qwen3.6-35B-A3B
# the Spark serves, so a fallback answer comes from the same model
# and prompt behaviour, not a stand-in.
#
# Why it exists: with one backend, a Spark restart, image bump or
# engine crash failed every hindsight call in flight, and hindsight
# retries a job a bounded number of times before failing it for
# good. The engine restarted itself five times on 2026-10-05 alone.
#
# ⚠️ PRIVACY. A request that falls back LEAVES THE HOUSE: Nous
# resells OpenRouter capacity (its usage block reports
# `cost_details.source: openrouter_list`), so the prompt goes to
# whichever upstream provider serves it. That applies to EVERY
# consumer of the text routes (spark-main and hindsight-main), not
# only hindsight:
# hindsight session transcripts, commit diffs, knowledge pages
# karakeep bookmark text being summarised/tagged
# mealie recipe text (its settings live in its database)
# n8n, node-red whatever a flow sends (none use it today)
# hermes-agent Wisp's prompts and tool output (the `spark`
# alias; the one consumer with real traffic)
# and since 2026-10-24 BOTH text routes have a second stage to
# Anthropic (claude-haiku-5-5) when Nous fails too, so any of the
# above can reach Anthropic as well as Nous's upstream.
# homebox-companion is not affected: it uses the vision route for
# everything, and vision does not fall back. This is the price of
# not losing the memory, taken deliberately; the only off switch is
# the fallbacks block, and it is all-or-nothing per route name.
# karakeep and mealie share `spark-main`; hindsight has its own name
# (`hindsight-main`, since 2026-10-24) so its chain can differ when
# it needs to. Keeping karakeep or mealie local means the same
# rename for it: its own route name with no fallbacks entry, or one
# that falls back only as far as it should.
#
# `reasoning: {enabled: false}` is NOT optional. Hosted, the model
# thinks by default: a one-word reply spent its whole 200-token cap
# reasoning and returned empty content, and retains ran 26-142s with
# one of three failing the schema. Off, 16 of 17 replayed retains
# were schema-valid at 8s p50 (hosted, at 4 concurrent -- not
# comparable to the Spark's 13s at 8). (The Spark's own switch,
# chat_template_kwargs, is ignored here; `reasoning` is OpenRouter's.)
#
# Cost, measured: 17 retains for $0.026, so ~$0.0015 a call, on the
# Nous subscription's included credits ($0.05 / $0.70 per million
# tokens). All of hindsight for a month would be roughly $25; as a
# fallback it is cents.
#
# Known gap: 1 of 17 came back as a malformed object despite
# strict json_schema, so upstream enforcement is not as tight as
# vLLM's grammar. Hindsight validates and retries, so it costs a
# retry, not a bad memory.
#
# Listed in exactly ONE virtual key, `gatus`, for the "Nous Portal
# (LiteLLM fallback)" probe in gatus's resources/config.yaml, which
# sends a one-token completion here every 30 minutes so a lapsed
# key or spent credits page BEFORE the Spark goes down. Every other
# consumer reaches it only as a fallback, never by name. Renaming
# this route means updating that key's models and the probe body.
- model_name: nous-qwen36-fallback
litellm_params:
model: openai/qwen/qwen3.6-35b-a3b
api_base: https://inference-api.nousresearch.com/v1
api_key: os.environ/NOUS_API_KEY
extra_body:
reasoning:
enabled: false
# --- hosted, by name: GLM-5.3-Flash on Nous Portal ---
# Not a fallback and not on the Spark: a 320B/18B-active MoE that
# needs two Sparks (sparkrun's recipe is TP=2 at 98 GB each), reached
# on Nous's credits instead. Named after the model because choosing
# this route IS choosing the model; `spark-main` stays the neutral
# default. Same key and privacy posture as the fallback above: every
# prompt leaves the house, via whichever upstream Nous routes to.
#
# Verified 2026-10-10 with the key: plain reply, strict json_schema
# (nested object, enum, nullable), and a tool call all worked; text,
# image and video in; $0.12 / $0.40 per million tokens. There is a
# `z-ai/glm-5.3-flash:US` at about 2x that if US-only hosting ever
# matters.
#
# Reasoning CANNOT be disabled on this model ("Reasoning is mandatory
# for this endpoint", HTTP 400 on reasoning.enabled=false), so the
# cheap mode is reasoning_effort: low -- 0 reasoning tokens for a
# one-word reply against 93 at the default. Set here as the route's
# default through extra_body, and a caller's own reasoning_effort DOES
# override it -- measured through this proxy on 2026-10-10: the same
# prompt gave 0 reasoning tokens on the route default and 70 with a
# top-level reasoning_effort: high. (A review guessed the opposite
# because the OpenAI client merges extra_body last; in practice the
# caller's value wins.)
#
# Not in any virtual key yet: add it per consumer with /key/update.
- model_name: nous-glm53-flash
litellm_params:
model: openai/z-ai/glm-5.3-flash
api_base: https://inference-api.nousresearch.com/v1
api_key: os.environ/NOUS_API_KEY
extra_body:
reasoning_effort: low
# The full GLM-5.3 (753B, text only -- no images), same key, same
# posture, for work that wants the bigger model: $0.91 / $2.86 per
# million tokens, ~8x the Flash. Verified 2026-10-10 like the Flash:
# plain reply, strict json_schema, tool call. Nothing pinned here:
# on a one-word reply it spent 0 reasoning tokens even at the
# default, and this route exists for the harder calls where its
# reasoning is the point. Not in any virtual key yet.
- model_name: nous-glm53
litellm_params:
model: openai/z-ai/glm-5.3
api_base: https://inference-api.nousresearch.com/v1
api_key: os.environ/NOUS_API_KEY
# Claude Opus 5.5 at Anthropic, for ONE consumer: hindsight's reflect
# stage, as a controlled comparison. (The 2026-10-24 refresh and
# consolidation sweep ran on a second name for the same model,
# `claude-opus-5-5-sweep`, which also dropped `reasoning_effort`;
# removed with the sweep, #2282 has it.)
# The set-2 re-gate rerun of 2026-10-23
# (Evals/2026-10-17-hindsight-regate-set2.md in the vault)
# passed the novelty bar (16/20) but asserted 9 wrong facts; some are
# stale records in the bank, some are synthesis errors. Running only
# reflect on Claude, with the bank untouched, separates the two: a
# wrong fact that survives here is the bank's, one that vanishes was
# the Spark model's. Paid for by the account's monthly API credit.
#
# Why through LiteLLM and not hindsight's own `anthropic` provider:
# one place for the key and spend, and `drop_params` strips whatever
# the OpenAI-shaped caller sends that Anthropic rejects. LiteLLM
# sends its own `anthropic-version`, so the 2026-10-21 API version's
# refusal of `temperature` does not apply. No reasoning_effort pinned:
# Opus 5.5 thinks adaptively on its own, and the Qwen rule that "any
# effort but none switches thinking on" is not this model's.
#
# Only in the `hindsight` virtual key, added with /key/update together
# with a max_budget on that key so a runaway loop cannot spend the
# credit. Remove the route, or keep it, once the comparison is graded.
#
# Measured 2026-10-24 against the API with LiteLLM's default
# anthropic-version: `temperature: 0.9` (hindsight's reflect default,
# DEFAULT_LLM_TEMPERATURE_REFLECT) returns 400 "`temperature` is
# deprecated for this model"; `temperature: 1` returns 200 WITH
# thinking tokens, so thinking is on without any thinking parameter.
# `drop_params` does not catch it (LiteLLM lists temperature as
# supported for Anthropic), hence the explicit list. max_tokens
# matches the Spark's 32k cap so a long synthesis is not cut at
# LiteLLM's small Anthropic default and misread as a worse answer.
- model_name: claude-opus-5-5
litellm_params:
model: anthropic/claude-opus-5-5
api_key: os.environ/ANTHROPIC_API_KEY
additional_drop_params:
- temperature
- top_p
- top_k
max_tokens: 32768
num_retries: 2
# Claude Haiku 5.5: the LAST stage of both text routes' fallback
# chains below (`hindsight-main` from the start, `spark-main` since
# later on 2026-10-24 while Nous is out of credit), nothing else.
# What reaches it is Hindsight's background retain/consolidation/
# refresh (bulk fact extraction, the job Qwen3.6-35B does on the
# Spark) and the other consumers' light traffic during a Spark
# outage, so the cheap tier is the right one: $0.10 in / $0.50 out
# per million tokens on the pricing page as of 2026-10 (prompts up
# to 100k), a fortieth of Opus. Added 2026-10-24 after the TensorFold test
# window showed the Nous stage failing every request with
# "Insufficient available credits" (balance $0.04 until 2026-11-05)
# while Gatus's one-token probe on it kept passing: 123 retains
# failed in an hour with no fallback. Same Anthropic key and the
# same parameter hygiene as the Opus route, plus `reasoning_effort`
# dropped: hindsight sends its per-stage effort (`none` for the
# refresh chain) in the body and a fallback replays it, and LiteLLM
# would turn it into Anthropic thinking parameters this model may
# reject. Hindsight's spend here is bounded by its key's max_budget
# per 30 days (budget_duration 30d, set 2026-10-24; $25, raised to
# $95 the same day for the accuracy sweep, resets 2026-11-01). That
# cap is on the KEY, so once hit it also blocks hindsight's free
# Spark route until the window resets; the duration is what makes
# that a pause rather than a permanent lockout. The spark-main
# consumers' keys carry no budget, so their fallback traffic is
# uncapped -- cents at their volumes, see the fallbacks comment.
- model_name: claude-haiku-5-5
litellm_params:
model: anthropic/claude-haiku-5-5
api_key: os.environ/ANTHROPIC_API_KEY
additional_drop_params:
- temperature
- top_p
- top_k
- reasoning_effort
max_tokens: 32768
num_retries: 2
# Pinned so the key budget counts this route even if the deployed
# LiteLLM's cost map predates the model (a $0 price would make the
# key's cap a no-op here). Anthropic list price as of 2026-10 for
# prompts up to 100k tokens: $0.10 / $0.50 per million.
model_info:
input_cost_per_token: 0.0000001
output_cost_per_token: 0.0000005
# Cache prices too (write 1.25x, read 0.1x of input): a pinned
# model_info replaces the cost map's entry wholesale, so without
# them LiteLLM priced cache writes at the plain input rate on
# every pinned route (the Sonnet re-extract calls of 2026-10-26
# came out ~3 % under). The provider budget below is only as
# good as this arithmetic.
cache_creation_input_token_cost: 0.000000125
cache_read_input_token_cost: 0.00000001
# GLM-5.3-Flash on TensorFold across both GB10s (spark-ab23 head,
# gx10-7ac6 worker; recipe ~/GLM-5.3-Flash-EXL3-2x-DGX-Sparks-TensorFold
# on the spark, served on :8888 as `GLM-5.3-Flash-EXL3`). The third
# leg of the re-gate reflect comparison, after the Spark's Qwen3.6
# and claude-opus-5-5 above: same 12 questions, same grading. It is
# UP ONLY DURING A TEST WINDOW -- it needs ~110 GiB free per box, so
# production vLLM (spark-main) is stopped while it runs, and the
# route answers connection-refused the rest of the time. No fallback
# on purpose: a reflect that silently went to Nous would be graded
# as TensorFold. max_tokens matches the other two legs; thinking is
# the engine default, as it was for both of them.
- model_name: tensorfold-glm53-flash
litellm_params:
model: openai/GLM-5.3-Flash-EXL3
api_base: http://10.0.10.237:8888/v1
api_key: unused-no-auth
max_tokens: 32768
num_retries: 2
litellm_settings:
drop_params: true
# Prometheus metrics (/metrics on the app port; the internal
# hostname's UI route sits behind Authentik, /v1 behind LiteLLM's
# keys, and the in-cluster scrape needs neither). Off by default,
# and until 2026-10-26 nothing exported a litellm_* series, so the
# only spend record was the database. Wanted for one gauge first:
# litellm_provider_remaining_budget_metric{api_provider="anthropic"},
# which prometheusrule.yaml turns into "credit is running low" and
# "Anthropic is failed over to the Spark" alerts. Request and spend
# counters per model/key come along for free; servicemonitor.yaml.
# `success_callback`, not `callbacks`: in v1.106.2 the proxy mounts
# /metrics only while loading the success_callback key
# (proxy/proxy_server.py:6736, "mounting metrics endpoint"); the
# `callbacks` key registers the logger but leaves /metrics at the
# catch-all 401, which is what the first roll of #2305 showed.
success_callback:
- prometheus
# v1.106.2 also guards the mounted /metrics with the proxy's key
# auth by default (litellm/__init__.py:191,
# proxy/auth/user_api_key_auth.py `_route_requires_auth_despite_public`):
# without this the scrape gets 401 "Unauthorized access to metrics
# endpoint", and a virtual key is not enough either. The scrape is
# in-cluster and the UI route is Authentik-gated, so no auth
# here, like hindsight's /metrics. Second roll of #2305/#2307.
require_auth_for_metrics_endpoint: false
# Anthropic prompt caching, injected here for every `anthropic/`
# route. Nothing upstream asks for it: hindsight reaches these
# routes through its OpenAI-shaped provider (its own `anthropic`
# provider does mark cache_control, but we route through LiteLLM
# for the single key and `drop_params`, see the Opus route), and
# that provider sends no cache_control. LiteLLM forwards markers
# only when the caller sends them or injection is configured (this
# flag, or per-route cache_control_injection_points), so every Claude
# call ran uncached: LiteLLM_DailyUserSpend for 2026-10-24 shows
# 669 claude-opus-5-5 requests, 10.6M prompt tokens, $54.85, and
# cache_read_input_tokens = cache_creation_input_tokens = 0, which
# is what Anthropic's 2026-10-26 "low cache hit rate" notice
# (est. 54 % of direct spend avoidable) was about.
#
# The traffic caches well: each hindsight stage has a fixed system
# prompt of 2-4k tokens (9-14k chars; retain 8.9k, consolidation
# 13.3k, refresh 14.2k), far above the 512-token minimum of the
# 5.5 models, and refresh/reflect are agent loops that re-send a
# growing conversation (12.5 messages per call on average on
# 2026-10-26). Calls arrive 1 s apart at the median, none more
# than 5 min apart, so the default 5-minute TTL is right; the
# 1h TTL (`anthropic_prompt_caching_ttl`) would only double the
# write price. The flag sets one breakpoint on the system prompt
# and one on the trailing turn, which is the loop pattern; on a
# two-message retain call the trailing breakpoint writes the
# unique document at the 1.25x write price with no later read,
# a few percent of that call, accepted. Spark/Nous routes are
# untouched (anthropic/ and bedrock/ only). Verify with
# cache_read_input_tokens in LiteLLM_DailyUserSpend and
# hindsight.llm_requests.cached_tokens, both 0 before this.
enable_anthropic_prompt_caching: true
# Response cache in the shared dragonfly, OPT-IN ONLY (default_off).
# A request is cached only when the caller sends
# `cache: {"use-cache": true}`; everything else, including all of
# hindsight's retain/consolidation/reflect traffic, is never cached.
# That is deliberate: those prompts are generated from evolving memory
# with sampling on, so a blanket cache would return a stale or
# identical "creative" answer. Turning it on per request is safe;
# turning it on globally would be a behaviour change nobody asked for.
#
# `namespace` prefixes every key so this tenant is separable inside a
# dragonfly shared with immich, tracearr, thanos and others. It
# runs --cluster_mode=emulated, where SELECT-based db isolation is not
# reliable, so a key prefix is the isolation mechanism (same as
# tracearr's REDIS_PREFIX).
cache: true
cache_params:
type: redis
host: dragonfly.database.svc.cluster.local
port: 6379
namespace: litellm.cache
mode: default_off
# Shared state that must NOT be per-process: tpm/rpm limits, each virtual
# key's max_parallel_requests, spend counters. Without this every worker
# enforces its own copy, so a key capped at 2 parallel requests can
# actually run 2 per worker, defeating the Spark slot budget the caps
# exist to enforce, and spend can overshoot. This is what the UI's
# "No Redis configured" banner is about; it clears once it is set.
general_settings:
coordination_redis:
host: dragonfly.database.svc.cluster.local
port: 6379
# Keep only the last hour of spend-log rows. This is a workaround for
# a collision, not a storage decision.
#
# LiteLLM keys each SpendLogs row on the provider's response id and
# silently drops a row whose id already exists (its own code notes
# "SpendLogs does not allow duplicate request_id"). Ollama makes ids
# like `chatcmpl-335` from a range of about 1,000, so once the table
# holds ~1,000 rows almost every new call collides and is not logged.
# Measured 2026-09-23: 1,645 rows with ids spanning 0-998, and 7 of 8
# probe calls returned an id already in the table and produced no row,
# so store_prompts_in_spend_logs was capturing ~1% of traffic. It was
# NOT caused by running two replicas: the same loss showed on the
# single pod before that change.
#
# A short window keeps the table small enough that collisions are
# rare (roughly 40 rows/hour here, so a few percent). The interval
# defaults to "1d", which would let a day of rows build up between
# cleanups and defeat the point, so it is set explicitly. Daily spend
# aggregates and key/team totals live in other tables and are
# unaffected. It also stops stored prompts, mental models included,
# from accumulating in Postgres indefinitely, which the unset default
# would do.
#
# Not a fix: prompt history is only ever the last hour, and a burst
# can still collide. Hindsight's own /llm-requests log has unique ids
# and is the reliable record for its calls.
maximum_spend_logs_retention_period: "1h"
maximum_spend_logs_retention_interval: "30m"
# Admin UI: no login with UI_USERNAME/UI_PASSWORD or, when those are
# unset (they are here), with the master key as a shared static
# secret. The UI's own banner asked for this once a real admin
# exists; it does: `bluevulpine` signs in through Authentik (the
# SSO env block on the container) and has been `proxy_admin` in
# LiteLLM_UserTable since 2026-10-03, re-applied from the
# `litellm_role` claim on every login. In v1.106.2
# (proxy/auth/login_utils.py, `_admin_credentials_match` and
# `is_env_credential_login_enabled`) this gates ONLY the
# username/password form: the SSO callback and the master key's
# API use (/key/generate, /provider/budgets, …) are untouched,
# so nothing operational changes. The stricter neighbour,
# `disable_password_login_when_sso_enabled`, would also refuse
# password logins for DB users; not needed, there are none.
# Lockout path if Authentik is down: none through the UI, by
# design; the master key still works against the API.
disable_env_credential_login: true
# The router's own Redis: cooldowns and usage-based routing state. Kept
# as a separate setting because LiteLLM treats it as a separate client.
router_settings:
redis_host: dragonfly.database.svc.cluster.local
redis_port: 6379
# Retry a failed request on ANOTHER deployment of the same model
# group before any fallback (LiteLLM >= v1.86, simple-shuffle
# only, async paths -- the proxy's). It matters only for the
# Spark routes, which have two deployments each (spark-ab23 and
# the gx10-7ac6 replica); every other group has one, so a
# failure there still goes straight to its fallbacks as before.
enable_weighted_failover: true
# Text routes only. A fallback fires after the route's own
# num_retries are spent -- on the Spark routes, after BOTH
# replicas' (enable_weighted_failover above) -- on errors that
# come back FAST: connection refused, 5xx. That covers the Spark
# being down, restarting or its engine having crashed. It does
# NOT cover an engine that accepts the connection and then hangs:
# no route sets a timeout, so LiteLLM's 600s default applies per
# attempt (with weighted failover a hung box can be followed by
# attempts on the other one), and hindsight's own 600s gives up
# first. Slow is not a fallback either.
#
# Vision (spark-main-vision) is deliberately NOT here: its callers
# are interactive (karakeep tagging, homebox photos), a failed tag
# is retried by the app, and photos are a different thing to send
# out than text. Add it only on purpose.
#
# ⚠️ A fallback pointing at a dead credential is worse than none:
# LiteLLM spends a failed round-trip and then buries the real cause
# in a nested error ("Connection error.. Error doing the fallback:
# AuthenticationError.."), so a Spark outage reads as an auth
# problem somewhere else. That is what an exhausted Azure member did
# here in 2026-09.
#
# Per route name, all-or-nothing, and NOT filtered by the calling
# key's model list in the deployed version (see hindsight-main).
# Both text routes get two stages, tried in order (LiteLLM:
# "fallbacks are done in-order"): Nous first because its credits
# are already paid for ($0.05/$0.70 per million); when they are
# spent it fails fast ("Insufficient available credits", seen
# 2026-10-24 at a $0.04 balance) and the request moves on to
# Haiku, which bills the Anthropic account.
#
# spark-main gained the Haiku stage on 2026-10-24 (Derek's call)
# because Nous was empty and stays so until ~2026-11-05: during
# the GB10 reboot that morning every spark-main consumer failed
# outright while hindsight-main already had Haiku behind it. The
# consumers here (n8n, node-red, karakeep, mealie, hermes-agent;
# homebox-companion's key lists spark-main but it only ever calls
# the vision route) carry no key budget, so an outage bills
# Haiku uncapped -- acceptable at $0.10/$0.50 per million for
# their volumes (hermes-agent is the only one with real traffic:
# $0.05 lifetime on the Spark's API-equivalent price). The
# `claude-haiku-5-5` route does not need to be on their keys:
# fallback targets are not key-filtered in this version.
#
# The Anthropic routes fall back to the Spark (Derek, 2026-10-26)
# so that an exhausted Anthropic budget degrades reflect and retain
# instead of failing them: when provider_budget_config below trips,
# every anthropic/ deployment is skipped, a route with no other
# deployment raises "No deployments available - crossed budget",
# a plain ValueError from router_strategy/budget_limiter.py; the
# route's num_retries may re-attempt it (no cost: nothing is
# called) and async_function_with_fallbacks' broad `except
# Exception` then hands it to this list like any other failure.
# The Haiku stage inside the two Spark chains
# above drops out the same way. The target is `spark-direct`, not
# hindsight-main, so the chains cannot loop (see that route). This
# also fires on ordinary Anthropic 429/529s, so a reflect can land
# on Qwen quietly; LiteLLMAnthropicBudgetExhausted names the budget
# case, and a graded comparison window should remove these two
# lines for its duration, as the TensorFold leg did.
fallbacks:
- spark-main:
- nous-qwen36-fallback
- claude-haiku-5-5
- hindsight-main:
- nous-qwen36-fallback
- claude-haiku-5-5
- claude-opus-5-5:
- spark-direct
- claude-haiku-5-5:
- spark-direct
# Fail-safe for the Anthropic monthly credit ($100 grant, issued
# on the 26th, expiring on the 29th of the next month; the older
# grant appears to drain first, so in effect each covers the 29th
# to the 29th). Until 2026-10-26 the only guard was the
# hindsight key's max_budget, enforced before routing, so tripping
# it blocks every route on that key, the free Spark one included,
# and the spark-main consumers' uncapped keys could still spend
# through Haiku. This counts every anthropic/ call from every key
# (Redis key `provider_spend:anthropic:1mo` in the router's
# dragonfly, the usage object priced by LiteLLM's cost map or the
# model_info pins above) and, once over the limit, removes the
# anthropic/ deployments from routing so the fallbacks above carry
# the traffic to the Spark. $99, not $100: overshooting the credit
# by cents has in the past forced a $5 top-up to clear the balance
# (Derek asked for $99.50); the extra half dollar covers calls in
# flight when the limit trips, since the check is before the call
# and the charge after it, and up to three Opus reflect calls
# plus, with the Spark down, 14 hindsight calls on Haiku can be
# running. Not counted here, because it never touches LiteLLM:
# direct use of the account key. The claude-code-review GitHub
# workflow is NOT such a case: it runs on the Claude Code OAuth
# token (the subscription), not the API key.
#
# The window starts at the FIRST anthropic/ call after the keys
# below are gone; "1mo" = same day next month, not a calendar
# anchor. First deployed 2026-10-26, it ran to 11-26, three days
# short of the grant, and in that gap LiteLLM would have seen a
# fresh $99 while the real credit was nearly spent. The 26th looked
# right at first because the grant is dated then, but on 10-29 the
# console showed the new grant untouched while LiteLLM had counted
# $15.83 since the 26th. Re-anchored 2026-10-30 04:35 UTC (a day
# late, which errs safe at the far end), so the window now ends
# 2026-11-30; re-anchor on the 29th whenever it drifts. The window
# lives in TWO Redis keys, `provider_spend:anthropic:1mo` and
# `provider_budget_start_time:anthropic`, plus each replica's
# in-memory copy that a replica fills from Redis on first use and
# that a `DEL` does not clear. The order that is safe under both
# hazards (a live replica syncing old spend back; a fresh replica
# caching the old start time before the keys are gone): delete
# both keys, `kubectl rollout restart deploy/litellm`, delete both
# keys again once the new pods are ready (on 10-30 a new pod had
# already recreated them mid-rollout), then make one small Haiku
# call so the window starts now rather than at hindsight's next
# call, and check that `budget_reset_at` in GET /provider/budgets
# reads the re-anchor day a month on (the 29th when done on time)
# on every replica. That check is the real gate.
# Deleting the spend key alone zeroes the spend but keeps the old
# window end (#2305 review). Spend from before a re-anchor is not
# counted, so the hindsight key's $95 cap stays as the guard for
# the allotment in progress until that reset.
provider_budget_config:
anthropic:
budget_limit: 99
time_period: 1mo | string |
configMaps.litellm-config.data."config.yaml" (1) model_list:
# OpenRouter cloud models ride the same proxy/key — one
# endpoint for local + cloud. Key comes from hermes-secret.
- model_name: deepseek-v4.1-flash
litellm_params:
model: openrouter/deepseek/deepseek-v4.1-flash
api_key: os.environ/OPENROUTER_API_KEY
# Vision aliases for photo intake. homebox-companion calls this
# proxy through the LiteLLM *library* with an OpenAI-compatible
# api_base, so it addresses these as `openai/<alias>` and the
# proxy resolves the alias. Both read image input; both were
# measured on a real walkthrough (49 frames @1fps) before this
# was added — see the comparison in REFACTOR_PLANS.md.
# qwen3.8-omni-flash — companion's default. Also takes audio
# and video, which is why the walkthrough-intake route uses it:
# one 49s clip cost ~26.7k tokens and returned 20 items, more
# cheaply than 49 still frames. Slower per photo (19.3s avg,
# 96.7s worst) than luna (9.0s/18.1s), so swap in Settings if
# the photo path feels sluggish.
# gpt-5.6-luna — companion's own documented default, and the
# faster of the two on single photos.
# Image barcodes are read by both, with occasional bad digits:
# validate the ISBN-13 check digit before trusting one.
- model_name: qwen3.8-omni-flash
litellm_params:
model: openrouter/qwen/qwen3.8-omni-flash
api_key: os.environ/OPENROUTER_API_KEY
- model_name: gpt-5.6-luna
litellm_params:
model: openrouter/openai/gpt-5.6-luna
api_key: os.environ/OPENROUTER_API_KEY
# Dan Manners' shared LiteLLM gateway — another LiteLLM proxy at
# https://llm.cloud.danmanners.com/v1. Only the models backed by
# hardware on his network are aliased here: they cost nothing and
# keep document text off a cloud provider.
# nvidia-llama -> nvidia-qwen3-32b-nvfp4 (vLLM, 151k ctx)
# intel-llama -> llama (vLLM, 229k ctx)
# gptoss-llama -> gpt-oss-20b-f16 (114k ctx)
# His catalog also carries cloud-backed models (zai/glm-*, and the
# misleadingly-named media-analysis/decision/embed-text which
# resolve to OpenRouter/OpenAI) — deliberately not proxied. Outbound
# key: 1P item `llm.cloud.danmanners.com` / field `credential`.
- model_name: nvidia-llama
litellm_params:
model: openai/nvidia-llama
api_base: https://llm.cloud.danmanners.com/v1
api_key: os.environ/DANMANNERS_LLM_API_KEY
- model_name: intel-llama
litellm_params:
model: openai/intel-llama
api_base: https://llm.cloud.danmanners.com/v1
api_key: os.environ/DANMANNERS_LLM_API_KEY
- model_name: gptoss-llama
litellm_params:
model: openai/gptoss-llama
api_base: https://llm.cloud.danmanners.com/v1
api_key: os.environ/DANMANNERS_LLM_API_KEY
# Audio via the in-pod speaches sidecar. The request's `voice`
# param passes straight through to piper (e.g.
# en_US-glados-medium to match the wyoming-piper voice).
- model_name: whisper
litellm_params:
model: openai/Systran/faster-whisper-small
api_base: http://127.0.0.1:8000/v1
api_key: speaches # required by the openai provider; unused
model_info:
mode: audio_transcription
- model_name: piper
litellm_params:
# speaches addresses TTS by voice repo, and the model has to
# be installed once (`POST /v1/models/speaches-ai%2Fpiper-...`)
# before it serves — it then lives on the models PVC, so this
# survives pod restarts. Catalog: huggingface.co/speaches-ai/
# piper-*. GLaDOS is not in it, so HA keeps wyoming-piper for
# that voice. Clients must still send a `voice` field: speaches
# requires the key even though the voice repo picks the voice.
model: openai/speaches-ai/piper-en_US-amy-medium
api_base: http://127.0.0.1:8000/v1
api_key: speaches # required by the openai provider; unused
model_info:
mode: audio_speech
litellm_settings:
# Don't 400 on params a local backend doesn't implement.
drop_params: true
num_retries: 2
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
| string |
configMaps.litellm-config.forceRename (1) litellm-config | string |
extraEnvVars[].name (1) GENERIC_CLIENT_ID | string |
extraEnvVars[].valueFrom.secretKeyRef.key (1) clientID | string |
extraEnvVars[].valueFrom.secretKeyRef.name (1) ${APP}-oidc-authentik-application | string |
fullnameOverride (1) litellm | string |
ingress.enabled (1) false | boolean |
redis.enabled (1) false | boolean |
| number | |
| string | |
| string | |
| string | |
| boolean | |
| string | |
| string | |
| string | |
| boolean |