From 3fa633370ddc2aafc18b5d28da1b349e2ac61cba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:46:56 +0000 Subject: [PATCH 01/25] chore(e2e): port the compat-matrix cron publisher to tests/e2e/claude_code Ports the daily cron VM publisher from the unmerged tests/claude_code checkout so the automation runs the e2e suite from litellm_internal_staging. Adds find_regressions to matrix_builder for the green to red auto-merge gate, pins the cron venv to Python 3.12, and ships the systemd units, env template, and runbook alongside --- .../_builder_unit_tests/__init__.py | 0 .../test_matrix_builder.py | 148 ++++ tests/e2e/claude_code/cron_vm/README.md | 187 +++++ tests/e2e/claude_code/cron_vm/build_matrix.py | 52 ++ .../claude_code/cron_vm/check_regressions.py | 80 +++ .../cron_vm/litellm-compat-matrix.env.example | 59 ++ .../cron_vm/litellm-compat-matrix.service | 101 +++ .../cron_vm/litellm-compat-matrix.timer | 25 + tests/e2e/claude_code/cron_vm/run_daily.sh | 645 ++++++++++++++++++ tests/e2e/claude_code/matrix_builder.py | 80 +++ 10 files changed, 1377 insertions(+) create mode 100644 tests/e2e/claude_code/_builder_unit_tests/__init__.py create mode 100644 tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py create mode 100644 tests/e2e/claude_code/cron_vm/README.md create mode 100644 tests/e2e/claude_code/cron_vm/build_matrix.py create mode 100644 tests/e2e/claude_code/cron_vm/check_regressions.py create mode 100644 tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example create mode 100644 tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service create mode 100644 tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer create mode 100755 tests/e2e/claude_code/cron_vm/run_daily.sh diff --git a/tests/e2e/claude_code/_builder_unit_tests/__init__.py b/tests/e2e/claude_code/_builder_unit_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py new file mode 100644 index 00000000000..16cb87032b9 --- /dev/null +++ b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -0,0 +1,148 @@ +"""Unit tests for `find_regressions`, the green→red detector that gates +auto-merge on the daily compat-matrix docs PR (see `cron_vm/`). + +Markerless harness tests: they exercise publisher plumbing, not a product +feature, so they run without a proxy and carry no `e2e` marker. +""" + +from __future__ import annotations + +from typing import Mapping, Union + +from claude_code.matrix_builder import find_regressions + +_CellSpec = Union[str, Mapping[str, str]] + + +def _matrix( + cells: Mapping[tuple[str, str], _CellSpec], + *, + names: Mapping[str, str] | None = None, +) -> dict[str, object]: + """Build a minimal matrix dict from a {(feature_id, provider): status} + or {(feature_id, provider): cell_dict} mapping.""" + names = names or {} + features: dict[str, dict[str, dict[str, str]]] = {} + for (feature_id, provider), value in cells.items(): + cell = {"status": value} if isinstance(value, str) else dict(value) + features.setdefault(feature_id, {})[provider] = cell + return { + "features": [ + { + "id": feature_id, + "name": names.get(feature_id, feature_id.upper()), + "providers": providers, + } + for feature_id, providers in features.items() + ] + } + + +def test_find_regressions_flags_pass_to_fail() -> None: + old = _matrix({("vision", "anthropic"): "pass"}) + new = _matrix( + {("vision", "anthropic"): {"status": "fail", "error": "credit balance too low"}} + ) + regressions = find_regressions(old, new) + assert len(regressions) == 1 + r = regressions[0] + assert r["feature_id"] == "vision" + assert r["provider"] == "anthropic" + assert r["old_status"] == "pass" + assert r["new_status"] == "fail" + assert r["error"] == "credit balance too low" + + +def test_find_regressions_ignores_red_to_red() -> None: + """An already-failing cell that stays failing is NOT a regression — a + provider that's independently broken (e.g. out of credits) must not + block the daily auto-merge forever.""" + old = _matrix({("vision", "anthropic"): "fail"}) + new = _matrix({("vision", "anthropic"): "fail"}) + assert find_regressions(old, new) == [] + + +def test_find_regressions_ignores_improvements_and_steady_green() -> None: + old = _matrix( + { + ("vision", "anthropic"): "fail", # red -> green + ("tool_use", "azure"): "pass", # green -> green + } + ) + new = _matrix( + { + ("vision", "anthropic"): "pass", + ("tool_use", "azure"): "pass", + } + ) + assert find_regressions(old, new) == [] + + +def test_find_regressions_ignores_green_to_grey() -> None: + """green→not_tested / green→not_applicable are degradations but not + *red* regressions; we deliberately don't block on them.""" + old = _matrix( + { + ("vision", "azure"): "pass", + ("tool_use", "azure"): "pass", + } + ) + new = _matrix( + { + ("vision", "azure"): "not_tested", + ("tool_use", "azure"): {"status": "not_applicable", "reason": "skip"}, + } + ) + assert find_regressions(old, new) == [] + + +def test_find_regressions_ignores_new_cells_without_baseline() -> None: + """A cell only present in the new matrix (new feature/provider) has no + baseline, so a fail there can't be a regression.""" + old = _matrix({("vision", "anthropic"): "pass"}) + new = _matrix( + { + ("vision", "anthropic"): "pass", + ("brand_new_feature", "anthropic"): "fail", + } + ) + assert find_regressions(old, new) == [] + + +def test_find_regressions_matches_by_id_not_name() -> None: + """Renaming a feature's display name must not hide a regression: cells + are matched on the stable id.""" + old = _matrix({("thinking", "anthropic"): "pass"}, names={"thinking": "Old Name"}) + new = _matrix( + {("thinking", "anthropic"): "fail"}, names={"thinking": "Totally New Name"} + ) + regressions = find_regressions(old, new) + assert len(regressions) == 1 + assert regressions[0]["feature_id"] == "thinking" + assert regressions[0]["feature_name"] == "Totally New Name" + + +def test_find_regressions_reports_multiple_sorted() -> None: + old = _matrix( + { + ("vision", "anthropic"): "pass", + ("tool_use", "anthropic"): "pass", + ("vision", "azure"): "pass", + } + ) + new = _matrix( + { + ("vision", "anthropic"): "fail", + ("tool_use", "anthropic"): "fail", + ("vision", "azure"): "pass", # stays green + } + ) + regressions = find_regressions(old, new) + keys = [(r["feature_id"], r["provider"]) for r in regressions] + assert keys == [("tool_use", "anthropic"), ("vision", "anthropic")] + + +def test_find_regressions_empty_old_matrix_is_safe() -> None: + """No baseline at all (first publish) yields no regressions.""" + new = _matrix({("vision", "anthropic"): "fail"}) + assert find_regressions({}, new) == [] diff --git a/tests/e2e/claude_code/cron_vm/README.md b/tests/e2e/claude_code/cron_vm/README.md new file mode 100644 index 00000000000..c1a4eaa2169 --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/README.md @@ -0,0 +1,187 @@ +# Cron VM setup for the Claude Code compatibility-matrix populator + +The populator runs daily on a dedicated GCP VM +(`litellm-compatibility-matrix-populator`) rather than as a GitHub +Action. Trade-offs: + +- ✅ Real VM means we can `gh auth login` against an account that's + already a collaborator on `BerriAI/litellm-docs`, instead of + provisioning a GitHub App with `pull-requests: write`. +- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`) + is reused across runs, so each daily run does a fast `git checkout` + + incremental `uv sync` rather than a fresh clone + cold sync. +- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`. +- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers + from short outages, but a multi-day outage means the matrix goes + stale until the VM is back. +- ⚠️ Provider credentials live on the VM filesystem + (`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat + the VM as an environment with comparable blast radius to a CI runner. + +This directory used to live at `tests/claude_code/cron_vm/` (paired with +the standalone `tests/claude_code/` suite); it now runs the maintained +`tests/e2e/claude_code/` suite instead. The pytest env interface changed +accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY` +(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure +column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously +`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and +`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`. + +## Layout + +| File | Purpose | +| --- | --- | +| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. | +| `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. | +| `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. | +| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. | +| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. | +| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. | + +## What `run_daily.sh` does + +1. **Resolves the latest LiteLLM final release tag** (newest bare + `vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the + GitHub Releases API (`curl | jq`). +2. **Reads the local Claude Code CLI version** via `claude --version`. + The cron does not auto-upgrade the CLI — operators do that + out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`. +3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`: + `git fetch --tags --force`, `git reset --hard`, + `git clean -fdx -e .venv -e .uv-bin`, `git checkout --force `. + The `.venv` is preserved across runs so `uv sync --frozen` is + incremental. Then **shims the test suite**: `tests/e2e/` in the + worktree is rebuilt from the dev checkout — the `claude_code/` suite + plus the five shared transport helpers it imports (`proxy_client.py`, + `e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the + cron always runs *today's* tests against the latest stable proxy. The + tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`, + whose imports the stable venv doesn't install) is deliberately not + used. +4. **Boots the proxy** as a `setsid` background process on port `4100` + (so it can't collide with a developer's `:4000`), then polls + `/health/liveliness` until it's up. +5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL` + pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest + hook writes the per-test results artifact. Test failures become + `fail` cells in the JSON, not script errors. +6. **Builds `compatibility-matrix.json`** by handing the artifact + + manifest to `build_matrix.py`. +7. **Opens or updates a docs PR**: `gh repo clone` of `litellm-docs` + into a tempdir, deterministic head branch + (`compat-matrix/--`), + `--force` push **directly to `BerriAI/litellm-docs`** (the + `mateo-berri` token has write access, so this is a same-repo branch, + not a fork), `gh pr create`. A re-run on the same day fast-forwards + the existing branch and `gh pr create` no-ops ("a pull request for + branch ... already exists" is treated as success). These PRs are no + longer gated on a second human review. +8. **Gates auto-merge on a regression check**: before enabling + auto-merge, `check_regressions.py` diffs the new matrix against the + one currently on `main`. Auto-merge (`gh pr merge --auto --squash`) + is only enabled when **no cell flipped green→red** — i.e. every + transition is red→green, green→green, or red→red. A pre-existing red + cell (e.g. a provider that's out of API credits) is `red→red` and + does **not** block; only a `pass`→`fail` flip does. When a regression + is detected the PR is still opened/updated (with a warning banner + naming the offending cells) but auto-merge is left **off** — and any + auto-merge a prior same-day run enabled is explicitly disabled — so a + human reviews before it lands on the public table. The check fails + *closed*: if it errors, auto-merge is withheld. +9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every + other open `compat-matrix/*` PR on the docs repo is closed (and its + bot-owned branch deleted), so at most one compat-matrix PR is ever + open — the newest. + +## One-time VM setup + +Run as `mateo` on the cron VM: + +```bash +# 1. Toolchain +sudo apt-get update +sudo apt-get install -y git nodejs npm jq curl +curl -LsSf https://astral.sh/uv/install.sh | sh +sudo apt-get install -y gh # or follow https://cli.github.com/ + +# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this +# line out-of-band when you want a fresh CLI to be tested) +sudo npm install -g @anthropic-ai/claude-code@latest + +# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the +# source of the .service / .timer files. The cron itself runs out +# of the separate worktree at ~/litellm-cron-worktree/. +mkdir -p ~/litellm +git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm +git -C ~/litellm/litellm checkout litellm_internal_staging + +# 4. gh auth — must be a collaborator on BerriAI/litellm-docs. +gh auth login # follow prompts; pick HTTPS + token paste flow + +# 5. Provider credentials. +sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \ + /etc/litellm-compat-matrix.env +sudoedit /etc/litellm-compat-matrix.env # fill in real values +sudo chmod 0600 /etc/litellm-compat-matrix.env + +# 6. systemd units. +sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ +sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now litellm-compat-matrix.timer +``` + +## Operating it + +```bash +# When does it run next? +systemctl list-timers litellm-compat-matrix.timer + +# Trigger a real run right now (PRs to litellm-docs). +sudo systemctl start litellm-compat-matrix.service + +# Trigger a run that does NOT open a PR (good for first-time validation). +SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh + +# Narrow to one cell while debugging. +SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \ + ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh + +# Watch the most recent run. +journalctl -u litellm-compat-matrix.service -f + +# Read older runs. +journalctl -u litellm-compat-matrix.service --since '2 days ago' + +# Disable until further notice (e.g. while debugging). +sudo systemctl disable --now litellm-compat-matrix.timer +``` + +## Gotchas + +- **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The + e2e suite uses PEP 695 `type` aliases, which the VM's system Python + (3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython + into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against + it. The first run after a version bump is a cold venv rebuild. +- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd + into the same VM with their own `:4000` proxy doesn't collide with a + cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env` + if you need to. +- **`uv sync --frozen` requires the resolved tag to be tagged on + GitHub.** If the latest stable release was made but not pushed as a + git tag, the `git checkout` step fails. Push the tag, then rerun. +- **`GITHUB_TOKEN` rotation is your problem.** The cron does not + refresh the token; if `mateo-berri`'s PAT in + `/etc/litellm-compat-matrix.env` expires, the run fails at the + `git push`/`gh pr create` step with a 401 ("Bad credentials" / + "Authentication failed"). Mint a fresh PAT and update the env file. + The token needs write access to `BerriAI/litellm-docs` (classic + `repo` scope, or fine-grained Contents:RW + Pull requests:RW). +- **First run after upgrading the Claude Code CLI is the riskiest one.** + If the new CLI changes its wire format the matrix run can produce + systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI + upgrade before letting the next scheduled fire happen. +- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory + is ~1 GB. Plan for at least 5 GB free on the VM, otherwise + `uv sync` will fail mid-run and leave you with a half-installed venv. diff --git a/tests/e2e/claude_code/cron_vm/build_matrix.py b/tests/e2e/claude_code/cron_vm/build_matrix.py new file mode 100644 index 00000000000..3d4fa767a1b --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/build_matrix.py @@ -0,0 +1,52 @@ +"""Tiny CLI wrapper around `claude_code.matrix_builder.build_from_paths`. + +Exists only so `run_daily.sh` can hand the version metadata + paths into +the matrix builder without re-implementing it in bash. All real logic +lives in `matrix_builder.py`. + +The suite imports its own modules with `tests/e2e/` on sys.path (that is +how pytest resolves them: `tests/e2e/` has no `__init__.py`, while +`claude_code/` does), so this script bootstraps the same root — two +levels up from this file — before importing. +""" + +from __future__ import annotations + +import argparse +import datetime +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from claude_code.matrix_builder import ( + build_from_paths, +) # noqa: E402 # needs the sys.path bootstrap above + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--litellm-version", required=True) + parser.add_argument("--claude-code-version", required=True) + args = parser.parse_args() + + generated_at = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + build_from_paths( + manifest_path=args.manifest, + results_path=args.results, + litellm_version=args.litellm_version, + claude_code_version=args.claude_code_version, + generated_at=generated_at, + output_path=args.output, + ) + print(f"wrote {args.output}") # noqa: T201 # CLI output read by run_daily.sh + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/e2e/claude_code/cron_vm/check_regressions.py b/tests/e2e/claude_code/cron_vm/check_regressions.py new file mode 100644 index 00000000000..5899e417ade --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/check_regressions.py @@ -0,0 +1,80 @@ +"""CLI: detect green→red regressions between the published matrix and a +freshly built one, so `run_daily.sh` can decide whether to enable +auto-merge on the daily docs PR. + +All real logic lives in `claude_code.matrix_builder.find_regressions`; +this file only does the I/O and maps the result onto an exit code the +bash caller can branch on. + +Exit codes (the bash gate depends on these exact values): + + 0 no green→red regressions -> safe to auto-merge + 3 one or more green→red regressions -> do NOT auto-merge (human review) + 2 argparse/usage error (argparse default) + +The `--old` file is allowed to be missing: on the first-ever publish there +is no baseline to regress against, so we exit 0. + +Imports resolve with `tests/e2e/` on sys.path, mirroring build_matrix.py. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from claude_code.matrix_builder import ( + find_regressions, +) # noqa: E402 # needs the sys.path bootstrap above + +REGRESSION_EXIT = 3 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--old", + type=Path, + required=True, + help="currently published matrix JSON (may be absent on first publish)", + ) + parser.add_argument( + "--new", + type=Path, + required=True, + help="freshly built matrix JSON", + ) + args = parser.parse_args() + + if not args.old.exists(): + print( # noqa: T201 # CLI output read by run_daily.sh + "no published matrix to compare against " + "(first publish); treating as no regressions" + ) + return 0 + + old_matrix = json.loads(args.old.read_text()) + new_matrix = json.loads(args.new.read_text()) + + regressions = find_regressions(old_matrix, new_matrix) + if not regressions: + print("no green->red regressions detected") # noqa: T201 # CLI output + return 0 + + print( # noqa: T201 # CLI output read by run_daily.sh + f"detected {len(regressions)} green->red regression(s):" + ) + for r in regressions: + line = f" - {r['feature_name']} [{r['provider']}]: pass -> fail" + if r["error"]: + line += f" ({r['error'][:160]})" + print(line) # noqa: T201 # CLI output read by run_daily.sh + return REGRESSION_EXIT + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example new file mode 100644 index 00000000000..008cbb748cb --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -0,0 +1,59 @@ +# Environment file consumed by `litellm-compat-matrix.service`. +# +# Install at `/etc/litellm-compat-matrix.env` and chmod 0600. +# `EnvironmentFile=-` in the unit means the service is allowed to start +# even if this file is missing, but the populator will fail at the +# first provider request without these credentials. + +# Anthropic +ANTHROPIC_API_KEY= + +# Bedrock (invoke + converse columns; also bedrock_mantle when enabled). +# Use Anthropic's Bedrock API-key passthrough (long-lived bearer token). +# No AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY required for the matrix -- +# both the LiteLLM invoke and converse routes pick up +# AWS_BEARER_TOKEN_BEDROCK when present. +AWS_BEARER_TOKEN_BEDROCK= +AWS_REGION_NAME=us-east-1 + +# Vertex AI (vertex_ai + vertex_ai_gpt columns). +# On the GCP VM, the default service-account ADC from the metadata server +# is used -- no JSON key file is needed. If you ever need to run outside +# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json. +VERTEXAI_PROJECT= +VERTEXAI_LOCATION=global + +# Azure AI Foundry (azure column — Claude models on Foundry) +AZURE_AI_API_KEY= +AZURE_AI_API_BASE= + +# OpenAI (openai GPT column) +OPENAI_API_KEY= + +# Azure OpenAI (azure_openai GPT column) +AZURE_API_BASE= +AZURE_API_KEY= + +# REQUIRED for publishing: PAT for the `mateo-berri` user, who has write +# access on BerriAI/litellm-docs. Used to (a) resolve the latest stable +# release, (b) push the daily compat-matrix branch directly to +# BerriAI/litellm-docs, (c) open the same-repo PR, and (d) enable +# squash auto-merge on it. Scopes: classic `repo` + `workflow`, or +# fine-grained on BerriAI/litellm-docs with Contents:RW + Pull +# requests:RW + Workflows:RW. +# Skip by setting SKIP_PUBLISH=1 (publishes nothing; only writes the +# matrix JSON locally). +GITHUB_TOKEN= + +# Optional: the bedrock_mantle column is opt-in because the AWS account +# needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the +# mantle cells are skipped and recorded as not_tested rather than fail. +# COMPAT_MANTLE_CELLS=1 + +# Optional overrides; defaults are sensible for the cron VM. +# PROXY_PORT=4100 +# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree +# DOCS_REPO=BerriAI/litellm-docs +# DOCS_BRANCH=main +# DOCS_TARGET_PATH=src/data/compatibility-matrix.json +# AUTO_MERGE_METHOD=squash diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service new file mode 100644 index 00000000000..9753d208135 --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service @@ -0,0 +1,101 @@ +# systemd service for the Claude Code compatibility-matrix populator. +# +# Triggered by `litellm-compat-matrix.timer`; not started directly. The +# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics +# describe "run once per day" cleanly — there's no long-lived daemon to +# supervise; each invocation runs the populator end-to-end and exits. +# +# Install +# ------- +# +# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ +# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/ +# sudo systemctl daemon-reload +# sudo systemctl enable --now litellm-compat-matrix.timer +# +# Paths are hard-coded to /home/mateo rather than using systemd's %h +# specifier. Why: in *system* units (this one), %h is expanded at +# parse time against the *manager's* home -- which is /root for PID 1 +# -- and *not* against the User= directive. That mismatch makes +# ReadWritePaths point at /root/.cache (which doesn't exist), causing +# the namespace setup to fail with status=226/NAMESPACE before the +# script ever runs. The runtime user (`User=mateo`) must: +# +# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the +# publisher module is importable; +# * have a uv venv at `~/litellm/litellm/.venv` (created by +# `uv sync --frozen` inside that checkout once); +# * have `gh` already authenticated against an account with +# `pull-requests: write` on `BerriAI/litellm-docs`; +# * have provider credentials exported in `/etc/litellm-compat-matrix.env` +# (see `litellm-compat-matrix.env.example` in this directory). + +[Unit] +Description=Claude Code compatibility-matrix populator (oneshot) +Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=mateo +Group=mateo + +# Provider credentials + any gh/PROXY_PORT overrides live here. Format +# is the standard `KEY=value` one line per env var. +EnvironmentFile=-/etc/litellm-compat-matrix.env + +# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin). +# `uv` and `claude` are installed under the runtime user's `~/.local/bin` +# so we have to prepend it explicitly; otherwise run_daily.sh fails at +# the up-front command-presence check. +Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be +# explicit so anything that reads $HOME (e.g. uv's cache lookup, the +# claude CLI's per-session dir) sees the right value even if a future +# refactor flips DynamicUser= or PrivateUsers= on. +Environment=HOME=/home/mateo + +WorkingDirectory=/home/mateo/litellm/litellm + +ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh + +# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new +# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM, +# plus the full feature x provider grid of pytest cells hitting several +# cloud providers. +TimeoutStartSec=90min + +# A failed run shouldn't restart automatically — the next timer fire is +# the right retry. Reruns of the same day's matrix are idempotent. +Restart=no + +# Security hardening: the populator only reads the litellm checkout and +# the env-file; everything else it writes lives in either the worktree +# (managed) or `/tmp` (cleaned up by tempfile). +# +# ReadWritePaths whitelist: +# * litellm-cron-worktree - the long-lived stable-tag checkout + +# its `.venv` (`uv sync` rewrites every +# run) + `.uv-bin` (pinned `uv` binary +# cache). +# * .cache - uv's wheel cache (~/.cache/uv) so we +# don't redownload pinned deps each run. +# * .claude - `claude` CLI's per-session state under +# `~/.claude/projects//`; created +# on every `claude --print` invocation. +# * .config/gh - `gh` CLI host config; technically not +# needed when we pass GH_TOKEN inline, +# but cheap to whitelist and prevents +# future regressions if a code path +# ever falls back to the host config. +# * /tmp - mktemp -d workdir + proxy logs. +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer new file mode 100644 index 00000000000..ee22538c6ed --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer @@ -0,0 +1,25 @@ +# Daily timer for the compatibility-matrix populator. +# +# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so +# operators in US/EU timezones see fresh PRs at the start of their work +# day. +# +# `Persistent=true` causes a missed run (VM was off / suspended) to +# fire the next time the timer is started, which is the property we +# want for a once-a-day job: the matrix should refresh as soon as the +# VM is reachable again, not wait another 24h. +# +# `RandomizedDelaySec=10min` smears load if multiple matrix-style +# pipelines are ever colocated on the same VM in the future. + +[Unit] +Description=Run the Claude Code compatibility-matrix populator daily + +[Timer] +OnCalendar=*-*-* 06:00:00 UTC +Persistent=true +RandomizedDelaySec=10min +Unit=litellm-compat-matrix.service + +[Install] +WantedBy=timers.target diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh new file mode 100755 index 00000000000..40b3245f6ab --- /dev/null +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -0,0 +1,645 @@ +#!/usr/bin/env bash +# Daily Claude Code compatibility-matrix populator. +# +# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the +# systemd timer in this directory. The flow is: +# +# 1. Resolve the latest LiteLLM final release tag from the GitHub +# Releases API. +# 2. Update a long-lived worktree at $WORKTREE to that tag and `uv sync` it. +# 3. Boot the proxy as a background subprocess on $PROXY_PORT (default +# 4100; a separate port from the human-tended :4000 proxy). +# 4. Run `pytest tests/e2e/claude_code/` against the proxy. Test +# failures become `fail` cells in the JSON, not script errors. +# 5. Hand the per-test results artifact + manifest to a small Python +# CLI (`build_matrix.py`) that wraps the existing +# `matrix_builder.build_from_paths` to produce the published +# compatibility-matrix.json. +# 6. `gh repo clone` litellm-docs, write the JSON to a deterministic +# branch (`compat-matrix/--`), commit, +# push the branch straight to BerriAI/litellm-docs (mateo-berri has +# write access), `gh pr create`, then — *only if no cell regressed +# green→red versus the currently-published matrix* — enable squash +# auto-merge so the PR merges itself once required checks pass. A +# green→red regression leaves auto-merge off for human review; an +# already-red cell (red→red) does not block. +# 7. Sweep stale compat-matrix PRs: once today's PR exists, close any +# other open `compat-matrix/*` PR (and delete its bot-owned branch) +# so at most ONE compat-matrix PR is ever open — the newest. A +# gate-withheld PR that nobody triages is superseded by the next +# day's run rather than accumulating in the queue. +# +# Same-day reruns land on the same branch so they update the existing PR +# rather than spawning a new one. If the JSON is byte-identical to the +# docs branch, we skip the push entirely. +# +# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm. +# Required state: a litellm checkout at $LITELLM_REPO (this file lives in +# it), $WORKTREE is created on first run, gh is already authenticated. +# +# Override any default by setting the matching env var; see the systemd +# unit for the production wiring. + +set -Eeuo pipefail + +LITELLM_REPO="${LITELLM_REPO:-${HOME}/litellm/litellm}" +WORKTREE="${LITELLM_WORKTREE:-${HOME}/litellm-cron-worktree}" +PROXY_PORT="${PROXY_PORT:-4100}" +PROXY_API_KEY="${PROXY_API_KEY:-sk-cron-matrix}" +DOCS_REPO="${DOCS_REPO:-BerriAI/litellm-docs}" +DOCS_BRANCH="${DOCS_BRANCH:-main}" +DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}" +SKIP_PUBLISH="${SKIP_PUBLISH:-0}" +PYTEST_K="${PYTEST_K:-}" +# The e2e suite uses PEP 695 `type` aliases, so the venv needs Python +# >= 3.12 (also what repo CI runs) even when the VM's system python is +# older. uv fetches a managed CPython of this version on first use -- +# checksum-verified against the manifest baked into the pinned uv +# binary -- and installs it under ${WORKTREE}/.uv-python (see +# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the +# systemd sandbox lets us write to. +CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}" +# Merge method for auto-merge. BerriAI/litellm-docs only allows squash +# merges (merge-commit and rebase are disabled at the repo level), so +# `squash` is the only valid value here unless that changes upstream. +AUTO_MERGE_METHOD="${AUTO_MERGE_METHOD:-squash}" + +POPULATOR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKDIR="$(mktemp -d -t litellm-compat-matrix.XXXXXX)" +PROXY_PID_FILE="${WORKDIR}/proxy.pid" + +# Cleanup is intentionally aggressive: it can run on normal exit, on a +# signal received by the script, or after a partial failure where the +# proxy is up but ${PROXY_PID_FILE} is stale. We try four things in +# order and stop as soon as the proxy port is free: +# +# 1. SIGTERM the pid recorded in proxy.pid. +# 2. SIGKILL anything from `pgrep -f "litellm.*--port ${PROXY_PORT}"` +# that survived. This catches the common case where the recorded +# pid was the sh wrapper, not the long-lived python child. +# 3. ss -K on the port (kernel kills sockets but not processes; +# mostly useful for catching lingering CLOSE_WAITs). +# 4. wipe ${WORKDIR}. +cleanup() { + local rc=$? + set +e + local proxy_pid + if [[ -f "${PROXY_PID_FILE}" ]]; then + proxy_pid="$(cat "${PROXY_PID_FILE}")" + if [[ -n "${proxy_pid}" ]]; then + kill -TERM "-${proxy_pid}" 2>/dev/null || kill -TERM "${proxy_pid}" 2>/dev/null || true + for _ in 1 2 3 4 5; do + kill -0 "${proxy_pid}" 2>/dev/null || break + sleep 1 + done + fi + fi + # Belt-and-braces: any python or uv talking to ${PROXY_PORT} that + # survived the SIGTERM gets SIGKILL'd by name. + pgrep -f "litellm.*--port[ =]?${PROXY_PORT}([^0-9]|$)" 2>/dev/null \ + | xargs -r kill -KILL 2>/dev/null || true + pgrep -f "${WORKTREE}/.uv-bin/uv.*run litellm" 2>/dev/null \ + | xargs -r kill -KILL 2>/dev/null || true + rm -rf "${WORKDIR}" + exit "${rc}" +} +trap cleanup EXIT INT TERM + +log() { printf '==> %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +for cmd in git uv gh jq curl claude; do + command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}" +done + +# Publishing pushes the branch straight to BerriAI/litellm-docs and opens +# the PR as mateo-berri, who has write access on the docs repo. The same +# ${GITHUB_TOKEN} is reused for release-listing above, so require it up +# front -- failing 30 minutes into a run because the env file is missing +# one line is a waste of CI quota. +if [[ "${SKIP_PUBLISH}" != "1" ]]; then + [[ -n "${GITHUB_TOKEN:-}" ]] \ + || die "GITHUB_TOKEN (mateo-berri, write access to ${DOCS_REPO}) required to push the branch and open the PR (or set SKIP_PUBLISH=1)" +fi + +# --------------------------------------------------------------------------- +# 1. Resolve versions +# --------------------------------------------------------------------------- + +# Newest PEP 440 *final* release on BerriAI/litellm. LiteLLM moved off +# the legacy `vX.Y.Z-stable` tag convention to PEP 440: a final/stable +# release is now a bare `vX.Y.Z` tag, while pre-releases carry a +# `-rc.N` / `-dev.N` segment (and the old `…-stable` / `…-stable.patch.N` +# tags are legacy and frozen at v1.83.x). We therefore select the newest +# tag with no pre-release segment -- matching `^v[0-9]+\.[0-9]+\.[0-9]+$` +# -- and skip drafts. The numeric version_key sort handles 1.10 > 1.9. +# +# Paginate through the releases endpoint instead of grabbing only page 1 +# (default page_size=30). LiteLLM ships multiple pre-releases per day, so +# it's common to need to walk past 30+ entries before hitting the most +# recent final release. We cap at 5 pages (500 releases) which is +# conservatively beyond the worst observed gap. +GH_AUTH_HEADER=() +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + GH_AUTH_HEADER=(-H "Authorization: Bearer ${GITHUB_TOKEN}") +fi +RELEASES_JSON="${WORKDIR}/releases.json" +echo "[]" >"${RELEASES_JSON}" +for page in 1 2 3 4 5; do + PAGE_JSON="${WORKDIR}/releases.page${page}.json" + curl -fsS \ + -H 'Accept: application/vnd.github+json' \ + -H 'User-Agent: litellm-compat-matrix' \ + "${GH_AUTH_HEADER[@]}" \ + "https://api.github.com/repos/BerriAI/litellm/releases?per_page=100&page=${page}" \ + >"${PAGE_JSON}" + jq -s '.[0] + .[1]' "${RELEASES_JSON}" "${PAGE_JSON}" >"${RELEASES_JSON}.merged" + mv "${RELEASES_JSON}.merged" "${RELEASES_JSON}" + # Stop early once we've seen at least one final release tag — no point + # paging further for a daily script that only needs the newest. + if jq -e '[.[] | select((.draft // false) == false) | .tag_name // "" | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+$"))] | length > 0' "${PAGE_JSON}" >/dev/null; then + break + fi + # No more pages? GitHub returns an empty array past the last page. + if [[ "$(jq 'length' "${PAGE_JSON}")" == "0" ]]; then + break + fi +done +LITELLM_VERSION="$( + jq -r ' + [ .[] + | select((.draft // false) == false) + | .tag_name // empty + | select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+$")) + ] + | sort_by( + capture("^v(?[0-9]+)\\.(?[0-9]+)\\.(?[0-9]+)$") + | [(.a|tonumber), (.b|tonumber), (.c|tonumber)] + ) + | last // empty + ' "${RELEASES_JSON}" +)" +[[ -n "${LITELLM_VERSION}" ]] || die "could not resolve latest PEP 440 final release (vX.Y.Z) in 5 pages of releases" +log "resolved litellm: ${LITELLM_VERSION}" + +CLAUDE_CODE_VERSION="$(claude --version 2>/dev/null | awk '{print $1}')" +[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not read 'claude --version'" +log "local claude code: ${CLAUDE_CODE_VERSION}" + +# --------------------------------------------------------------------------- +# 2. Update the worktree to that tag +# --------------------------------------------------------------------------- + +if [[ ! -d "${WORKTREE}/.git" ]]; then + log "first run: cloning litellm into ${WORKTREE}" + mkdir -p "$(dirname "${WORKTREE}")" + git clone https://github.com/BerriAI/litellm.git "${WORKTREE}" +fi + +log "updating worktree to ${LITELLM_VERSION}" +git -C "${WORKTREE}" fetch --tags --force +git -C "${WORKTREE}" reset --hard +# Keep the venv, the .uv-bin cache, and the .uv-python managed +# interpreter around — uv sync will reconcile the venv on every run, +# and we don't want to re-download the pinned uv binary or the managed +# CPython each time. Drop everything else (including any prior +# tests/e2e/ shim) so each run starts clean before the shim below +# rewrites it from the dev checkout. +git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python +git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}" + +# Always rebuild tests/e2e/ in the worktree from the dev checkout, +# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two +# reasons: +# +# * The matrix populator's job is to exercise *today's* tests against +# the latest stable proxy. The dev checkout carries the most recent +# test fixes that haven't yet rolled into a stable release, and we +# want every cron run to pick those up the moment they land on +# ${LITELLM_REPO}, not whenever the next stable release happens. +# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose +# top-level conftest.py imports modules (e2e_db, lifecycle, +# otel_client, ...) that the stable venv does not install. Copying +# the whole tree would make pytest collection blow up on those +# imports. +# +# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY +# the claude_code suite plus the shared transport helpers it imports. +# pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while +# claude_code/ does), which is what resolves both the `claude_code.*` +# and the bare `proxy_client` / `e2e_http` imports inside the suite. +E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py) +if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then + die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code" +fi +for helper in "${E2E_HELPER_FILES[@]}"; do + [[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \ + || die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}" +done +log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)" +rm -rf "${WORKTREE}/tests/e2e" +mkdir -p "${WORKTREE}/tests/e2e" +cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/" +for helper in "${E2E_HELPER_FILES[@]}"; do + cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/" +done + +# litellm pins an exact uv version in pyproject.toml's [tool.uv] +# `required-version` field, so a system uv that's newer or older +# refuses to sync. We pin our own local copy at the version the +# checked-out tag asks for, cached under .uv-bin/ inside the worktree +# so subsequent runs skip the download. +PINNED_UV_VERSION="$( + awk -F'"' ' + /^required-version[[:space:]]*=/ { + # Field 2 is the value between the quotes, e.g. ">=0.10.9" or + # "0.10.9". Strip any leading specifier prefix so we end up with + # the bare version string, which is what /releases/download// + # expects. + v = $2 + sub(/^[[:space:]=<>!~]+/, "", v) + if (v != "") { print v; exit } + } + ' "${WORKTREE}/pyproject.toml" +)" +if [[ -z "${PINNED_UV_VERSION}" ]]; then + log "no uv version pin in pyproject.toml; using system uv" + WORKTREE_UV="$(command -v uv)" +else + WORKTREE_UV="${WORKTREE}/.uv-bin/uv-${PINNED_UV_VERSION}" + if [[ ! -x "${WORKTREE_UV}" ]]; then + log "downloading uv ${PINNED_UV_VERSION} for the worktree" + mkdir -p "${WORKTREE}/.uv-bin" + UV_TARBALL_NAME="uv-x86_64-unknown-linux-gnu.tar.gz" + UV_DOWNLOAD_URL="https://github.com/astral-sh/uv/releases/download/${PINNED_UV_VERSION}/${UV_TARBALL_NAME}" + UV_TMPDIR="$(mktemp -d -t uv-download.XXXXXX)" + # Download the tarball and Astral's official .sha256 sidecar to disk + # and verify the digest before extracting/executing anything. This + # closes the supply-chain trust gap of piping a remote binary + # straight into `tar -xzO ... > file ; chmod +x` (see CLAUDE.md + # "CI Supply-Chain Safety"). + curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}" "${UV_DOWNLOAD_URL}" + curl -fsSL --output "${UV_TMPDIR}/${UV_TARBALL_NAME}.sha256" "${UV_DOWNLOAD_URL}.sha256" + (cd "${UV_TMPDIR}" && sha256sum -c "${UV_TARBALL_NAME}.sha256") \ + || { rm -rf "${UV_TMPDIR}"; die "uv ${PINNED_UV_VERSION} sha256 mismatch — refusing to install"; } + tar -xzf "${UV_TMPDIR}/${UV_TARBALL_NAME}" -C "${UV_TMPDIR}" "uv-x86_64-unknown-linux-gnu/uv" + mv "${UV_TMPDIR}/uv-x86_64-unknown-linux-gnu/uv" "${WORKTREE_UV}.tmp" + chmod +x "${WORKTREE_UV}.tmp" + mv "${WORKTREE_UV}.tmp" "${WORKTREE_UV}" + rm -rf "${UV_TMPDIR}" + fi +fi +# `--extra proxy` pulls fastapi/uvicorn/etc. so `uv run litellm` can +# actually serve. `--group proxy-dev` brings in pytest and the rest of +# what tests/e2e/claude_code/ needs. `--python` pins the venv to +# ${CRON_PYTHON_VERSION}; the first run after a version bump recreates +# the venv from scratch (a one-time cold sync). +export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python" +log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})" +(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}") + +PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml" +[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)" + +# --------------------------------------------------------------------------- +# 3. Boot the proxy +# --------------------------------------------------------------------------- + +log "starting proxy on 127.0.0.1:${PROXY_PORT}" +# Bind the proxy to loopback only. The populator proxy is talked to +# exclusively by the pytest run on the same host (the health check and +# the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`), +# so there's no reason to expose it on the VM's external interfaces. +# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with +# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would +# allow anything that can reach :${PROXY_PORT} on the VM to authenticate +# and burn upstream provider credentials. +# +# `setsid` puts the proxy in its own session+pgroup so cleanup() can +# SIGTERM the whole tree by passing the pgid as a negative pid. We +# write that pid to a file so cleanup() doesn't need to remember a +# variable that might be stale by the time the trap fires. +setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c ' + echo "$$" > "$0" + cd "$1" + exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4" +' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \ + >"${WORKDIR}/proxy.log" 2>&1 & +disown + +HEALTH_URL="http://127.0.0.1:${PROXY_PORT}/health/liveliness" +for _ in $(seq 1 45); do + if curl -fsS "${HEALTH_URL}" >/dev/null 2>&1; then + break + fi + sleep 2 +done +curl -fsS "${HEALTH_URL}" >/dev/null \ + || { tail -50 "${WORKDIR}/proxy.log" >&2; die "proxy did not become healthy"; } + +# --------------------------------------------------------------------------- +# 4. Run pytest +# --------------------------------------------------------------------------- + +RESULTS_JSON="${WORKDIR}/compat-results.json" +# The `_*_unit_tests` ignore is defensive: those harness-only trees are +# markerless (they run without a proxy) and don't feed matrix cells, so +# the cron skips them if/when they land in the suite. +PYTEST_ARGS=( + tests/e2e/claude_code/ + "--ignore-glob=*_unit_tests*" +) +if [[ -n "${PYTEST_K}" ]]; then + log "PYTEST_K set; narrowing to: ${PYTEST_K}" + PYTEST_ARGS+=(-k "${PYTEST_K}") +fi + +log "running pytest" +set +e +( + cd "${WORKTREE}" \ + && LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \ + LITELLM_MASTER_KEY="${PROXY_API_KEY}" \ + COMPAT_RESULTS_PATH="${RESULTS_JSON}" \ + "${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}" +) +PYTEST_EXIT=$? +set -e +log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)" +[[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}" + +# --------------------------------------------------------------------------- +# 5. Build the matrix JSON +# --------------------------------------------------------------------------- + +MATRIX_JSON="${WORKDIR}/compatibility-matrix.json" +log "building ${MATRIX_JSON}" +( + cd "${WORKTREE}" \ + && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \ + --manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \ + --results "${RESULTS_JSON}" \ + --output "${MATRIX_JSON}" \ + --litellm-version "${LITELLM_VERSION}" \ + --claude-code-version "${CLAUDE_CODE_VERSION}" +) + +# --------------------------------------------------------------------------- +# 6. Open a docs-repo PR +# --------------------------------------------------------------------------- + +if [[ "${SKIP_PUBLISH}" == "1" ]]; then + cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json" + log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json" + exit 0 +fi + +DATE_UTC="$(date -u +%Y-%m-%d)" +BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}" +DOCS_CLONE="${WORKDIR}/litellm-docs" + +log "cloning ${DOCS_REPO}@${DOCS_BRANCH}" +gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}" + +cd "${DOCS_CLONE}" +git config user.email "litellm-bot@berri.ai" +git config user.name "litellm-compat-matrix-bot" +git checkout -b "${BRANCH_NAME}" + +# Snapshot the currently-published matrix *before* we overwrite it, so the +# auto-merge gate below can diff old→new cell statuses. On the first-ever +# publish the file won't exist yet; we leave ${PUBLISHED_MATRIX} pointing +# at a path that doesn't exist and let check_regressions.py treat that as +# "no baseline → no regressions". +PUBLISHED_MATRIX="${WORKDIR}/published-matrix.json" +if [[ -f "${DOCS_TARGET_PATH}" ]]; then + cp "${DOCS_TARGET_PATH}" "${PUBLISHED_MATRIX}" +fi + +mkdir -p "$(dirname "${DOCS_TARGET_PATH}")" +cp "${MATRIX_JSON}" "${DOCS_TARGET_PATH}" +git add "${DOCS_TARGET_PATH}" + +if git diff --cached --quiet; then + log "matrix JSON unchanged from ${DOCS_BRANCH}; skipping PR" + exit 0 +fi + +# --- Auto-merge regression gate -------------------------------------------- +# Only auto-merge when the new matrix is improvement-or-equal: every cell +# transition is red→green, green→green, or red→red. If any cell flips +# green→red (a `pass` that became `fail`), we still open/refresh the PR but +# leave auto-merge OFF so a human reviews the regression before it lands on +# the public docs table. A pre-existing red cell (e.g. Anthropic out of API +# credits) is red→red and does NOT block, so the daily PR keeps flowing. +log "checking for green->red regressions vs the published matrix" +set +e +REGRESSION_REPORT="$( + cd "${WORKTREE}" \ + && "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \ + --old "${PUBLISHED_MATRIX}" \ + --new "${MATRIX_JSON}" +)" +REGRESSION_EXIT=$? +set -e +printf '%s\n' "${REGRESSION_REPORT}" | sed 's/^/ /' >&2 +# Exit 0 = clean. Exit 3 = green→red regression(s) found. Any other code +# means the checker itself errored; fail *closed* (withhold auto-merge) so a +# bug in the gate can never silently auto-merge a regression. +if [[ ${REGRESSION_EXIT} -eq 0 ]]; then + ALLOW_AUTOMERGE=1 +elif [[ ${REGRESSION_EXIT} -eq 3 ]]; then + ALLOW_AUTOMERGE=0 + log "WARN: green->red regression(s) detected; auto-merge will be left OFF for review" +else + ALLOW_AUTOMERGE=0 + log "WARN: regression check errored (exit ${REGRESSION_EXIT}); withholding auto-merge to be safe" +fi + +GENERATED_AT="$(jq -r '.generated_at' "${MATRIX_JSON}")" +COMMIT_MSG="$(cat </dev/null || true +git remote add publish "${PUBLISH_PUSH_URL}" +git push --force --set-upstream publish "${BRANCH_NAME}" +git remote remove publish +unset PUBLISH_PUSH_URL + +# Per-feature status table for the PR body. Reviewers triage from this. +PR_FEATURE_TABLE="$(jq -r ' + .features[] as $f + | "- **\($f.name)**: " + + ([ .providers[] as $p + | "\($p)=\($f.providers[$p].status // "not_tested")" + ] | join(", ")) +' "${MATRIX_JSON}")" + +# When the gate withheld auto-merge, call it out at the top of the PR body +# (with the offending cells) so a reviewer knows this PR needs a human and +# why. On the clean path this section is empty. Note `$(...)` strips the +# trailing newline, so the body below puts explicit blank lines *around* +# the placeholder rather than relying on the heredoc's own spacing. +if [[ "${ALLOW_AUTOMERGE}" != "1" ]]; then + PR_REGRESSION_SECTION="$(cat < [!WARNING] +> **Auto-merge disabled:** one or more cells regressed green→red versus the +> currently-published matrix. Review the diff before merging. + +\`\`\` +${REGRESSION_REPORT} +\`\`\` +EOF +)" +else + PR_REGRESSION_SECTION="" +fi + +PR_TITLE="chore(compat-matrix): refresh for ${LITELLM_VERSION} + claude-code ${CLAUDE_CODE_VERSION}" +PR_BODY="$(cat < ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)" +# GH_TOKEN is mateo-berri's write-scoped token, the same identity used +# for release-listing above. The branch lives on ${DOCS_REPO} itself, so +# --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`. +set +e +PR_OUT="$( + GH_TOKEN="${GITHUB_TOKEN}" gh pr create \ + --repo "${DOCS_REPO}" \ + --base "${DOCS_BRANCH}" \ + --head "${BRANCH_NAME}" \ + --title "${PR_TITLE}" \ + --body "${PR_BODY}" 2>&1 +)" +PR_EXIT=$? +set -e +echo "${PR_OUT}" + +if [[ ${PR_EXIT} -ne 0 ]]; then + if grep -q "a pull request for branch.*already exists" <<<"${PR_OUT}"; then + log "PR already exists for ${BRANCH_NAME}; updated branch in place" + else + die "gh pr create failed (exit ${PR_EXIT})" + fi +fi + +# Enable auto-merge so the PR merges itself once the docs repo's required +# checks pass -- we no longer gate these bot PRs on a second human +# approval. mateo-berri authors and merges them directly. The repo only +# permits squash merges and has auto-merge enabled at the repo level +# (${AUTO_MERGE_METHOD} defaults to squash accordingly). +# +# This only fires when the regression gate above is satisfied +# (${ALLOW_AUTOMERGE}==1): a green→red regression — or a gate error — +# leaves auto-merge OFF so a human triages the PR. +# +# `gh pr merge --auto` is idempotent: re-enabling auto-merge on a PR that +# already has it set is a no-op, so same-day reruns stay clean. It's +# non-fatal: if auto-merge can't be enabled (e.g. the PR is already in a +# clean/mergeable state with nothing left to wait on, or branch +# protection isn't configured), the matrix JSON has still landed on the +# PR and the worst case is a manual merge click. +if [[ "${ALLOW_AUTOMERGE}" == "1" ]]; then + log "enabling ${AUTO_MERGE_METHOD} auto-merge on ${BRANCH_NAME}" + set +e + GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \ + "${BRANCH_NAME}" \ + --repo "${DOCS_REPO}" \ + --auto \ + "--${AUTO_MERGE_METHOD}" 2>&1 | sed 's/^/ /' + AUTOMERGE_EXIT=${PIPESTATUS[0]} + set -e + if [[ ${AUTOMERGE_EXIT} -ne 0 ]]; then + log "WARN: gh pr merge --auto exited ${AUTOMERGE_EXIT} (non-fatal)" + fi +else + # Regression (or gate error): make sure auto-merge is OFF. A same-day + # rerun may have enabled it on an earlier, clean pass, so explicitly + # disable rather than just skipping. Non-fatal: if it was never enabled, + # `--disable-auto` is a harmless no-op/error we swallow. + log "leaving ${BRANCH_NAME} for manual review; disabling any prior auto-merge" + set +e + GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \ + "${BRANCH_NAME}" \ + --repo "${DOCS_REPO}" \ + --disable-auto 2>&1 | sed 's/^/ /' + set -e +fi + +# --- Stale-PR sweep ---------------------------------------------------------- +# Keep at most ONE compat-matrix PR open: today's. Any other open +# `compat-matrix/*` PR is a leftover from a day whose regression gate +# withheld auto-merge and nobody triaged it; the PR we just opened or +# refreshed above carries strictly fresher results, so the old one is +# pure queue noise. Closing is non-destructive — the PR record and its +# regression report stay browsable; only the bot-owned branch is +# deleted. This runs only after today's PR exists (a `die` above skips +# it), so a failed publish can never close the queue down to zero. +# +# Non-fatal: a sweep failure (rate limit, transient API error) leaves +# stale PRs for the next run to retry; it must not fail the pipeline. +log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})" +set +e +STALE_PRS="$( + GH_TOKEN="${GITHUB_TOKEN}" gh pr list \ + --repo "${DOCS_REPO}" \ + --state open \ + --limit 100 \ + --json number,headRefName \ + --jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"' +)" +while IFS=$'\t' read -r stale_pr stale_head; do + [[ -z "${stale_pr}" ]] && continue + [[ "${stale_head}" == "${BRANCH_NAME}" ]] && continue + GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \ + --repo "${DOCS_REPO}" \ + --delete-branch \ + --comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /' + if [[ ${PIPESTATUS[0]} -eq 0 ]]; then + log "closed stale compat-matrix PR #${stale_pr} (${stale_head})" + else + log "WARN: could not close stale compat-matrix PR #${stale_pr} (non-fatal)" + fi +done <<<"${STALE_PRS}" +set -e + +log "done" diff --git a/tests/e2e/claude_code/matrix_builder.py b/tests/e2e/claude_code/matrix_builder.py index d9a13d17ea4..d6fdd658f2a 100644 --- a/tests/e2e/claude_code/matrix_builder.py +++ b/tests/e2e/claude_code/matrix_builder.py @@ -174,6 +174,86 @@ def _aggregate_cell(results: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: return {"status": "not_tested"} +def _index_cells(matrix: Mapping[str, Any]) -> dict[tuple[str, str], dict[str, Any]]: + """Map ``(feature_id, provider) -> cell dict`` for a built matrix. + + Cells are keyed by the *stable* feature ``id`` (not the display + ``name``, which can be reworded without changing the underlying row) + and the provider key, so two matrices built at different times line up + even if feature names drift. + """ + out: dict[tuple[str, str], dict[str, Any]] = {} + for feature in matrix.get("features", []) or []: + if not isinstance(feature, Mapping): + continue + feature_id = feature.get("id") + if not feature_id: + continue + providers = feature.get("providers", {}) or {} + if not isinstance(providers, Mapping): + continue + for provider, cell in providers.items(): + if isinstance(cell, Mapping): + out[(feature_id, provider)] = dict(cell) + return out + + +def find_regressions( + old_matrix: Mapping[str, Any], + new_matrix: Mapping[str, Any], +) -> list[dict[str, str]]: + """Return the cells that flipped green→red (``pass`` → ``fail``). + + A *regression* is defined strictly: a cell that was ``pass`` in + ``old_matrix`` and is ``fail`` in ``new_matrix``. Every other + transition is intentionally *not* a regression: + + * ``red → green`` / ``green → green`` — the happy path. + * ``red → red`` — a cell that is *already* failing for an unrelated + reason (e.g. Anthropic out of API credits) must not block + publishing, otherwise the daily PR would never auto-merge until + that independent issue is fixed. + * ``green → not_tested`` / ``green → not_applicable`` — a cell going + grey is a degradation but not a *red* regression; treating a + skipped/flaky run as a hard block would create false positives. + + Cells present only in ``new_matrix`` (a newly added feature or + provider) have no baseline and therefore cannot be regressions. + + Each returned item is a flat str→str mapping so callers (the cron's + ``check_regressions.py``) can render it without further lookups: + ``feature_id``, ``feature_name``, ``provider``, ``old_status``, + ``new_status``, ``error``. + """ + old_cells = _index_cells(old_matrix) + feature_names = { + f.get("id"): str(f.get("name", f.get("id"))) + for f in new_matrix.get("features", []) or [] + if isinstance(f, Mapping) and f.get("id") + } + + regressions: list[dict[str, str]] = [] + for (feature_id, provider), new_cell in sorted( + _index_cells(new_matrix).items(), key=lambda kv: (kv[0][0], kv[0][1]) + ): + if new_cell.get("status") != "fail": + continue + old_cell = old_cells.get((feature_id, provider)) + if old_cell is None or old_cell.get("status") != "pass": + continue + regressions.append( + { + "feature_id": str(feature_id), + "feature_name": feature_names.get(feature_id, str(feature_id)), + "provider": str(provider), + "old_status": "pass", + "new_status": "fail", + "error": str(new_cell.get("error", "")), + } + ) + return regressions + + def build_from_paths( *, manifest_path: Path, From 5818848413057b4d253cb20d62189a87266600e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:43:59 +0000 Subject: [PATCH 02/25] docs(e2e): document the openai gpt opt-in flag in the cron env example --- .../claude_code/cron_vm/litellm-compat-matrix.env.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example index 008cbb748cb..e1e98b98120 100644 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -50,6 +50,11 @@ GITHUB_TOKEN= # mantle cells are skipped and recorded as not_tested rather than fail. # COMPAT_MANTLE_CELLS=1 +# Optional: the openai column is likewise opt-in; its cells hit CLI +# timeouts under the concurrent stage suite, but the serial cron can +# usually run them. Skipped cells are recorded as not_tested. +# COMPAT_OPENAI_GPT_CELLS=1 + # Optional overrides; defaults are sensible for the cron VM. # PROXY_PORT=4100 # LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree From 123561527b1044a7c101045474519d6b4b8f34bb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:54:25 +0000 Subject: [PATCH 03/25] fix(e2e): fail closed on partial pytest runs and unverified auto-merge disable --- tests/e2e/claude_code/cron_vm/run_daily.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index 40b3245f6ab..9d9a7abe367 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -366,6 +366,10 @@ set +e PYTEST_EXIT=$? set -e log "pytest exit code: ${PYTEST_EXIT} (failures become 'fail' cells, not script errors)" +# 0=green, 1=test failures (fail cells); >=2 = interrupted/internal/usage/no +# tests, i.e. a partial run whose missing cells would publish as not_tested. +[[ ${PYTEST_EXIT} -le 1 ]] \ + || die "pytest exited abnormally (${PYTEST_EXIT}); refusing to publish a partial matrix" [[ -f "${RESULTS_JSON}" ]] || die "pytest did not produce ${RESULTS_JSON}" # --------------------------------------------------------------------------- @@ -594,8 +598,10 @@ if [[ "${ALLOW_AUTOMERGE}" == "1" ]]; then else # Regression (or gate error): make sure auto-merge is OFF. A same-day # rerun may have enabled it on an earlier, clean pass, so explicitly - # disable rather than just skipping. Non-fatal: if it was never enabled, - # `--disable-auto` is a harmless no-op/error we swallow. + # disable rather than just skipping. The disable call itself is allowed + # to error (`--disable-auto` fails harmlessly when auto-merge was never + # enabled), but the read-back below is authoritative: a regressed matrix + # must never be left armed to merge, so a still-armed PR is fatal. log "leaving ${BRANCH_NAME} for manual review; disabling any prior auto-merge" set +e GH_TOKEN="${GITHUB_TOKEN}" gh pr merge \ @@ -603,6 +609,15 @@ else --repo "${DOCS_REPO}" \ --disable-auto 2>&1 | sed 's/^/ /' set -e + AUTOMERGE_ARMED="$( + GH_TOKEN="${GITHUB_TOKEN}" gh pr view \ + "${BRANCH_NAME}" \ + --repo "${DOCS_REPO}" \ + --json autoMergeRequest \ + --jq '.autoMergeRequest.enabledAt // empty' + )" || die "could not read back the auto-merge state on ${BRANCH_NAME}" + [[ -z "${AUTOMERGE_ARMED}" ]] \ + || die "auto-merge still armed on ${BRANCH_NAME} (enabled ${AUTOMERGE_ARMED}) after --disable-auto" fi # --- Stale-PR sweep ---------------------------------------------------------- From 08bea8d0dd210f33e8a24f9acca655026d682fe6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:57:25 +0000 Subject: [PATCH 04/25] fix(compat-matrix): keep publish token out of the job-wide process env The mateo-berri PAT now arrives via systemd LoadCredential as a file instead of the EnvironmentFile, so pytest, the proxy, and the model-driven claude CLI never inherit it and a same-UID /proc read cannot lift it. run_daily.sh reads the credential when present, still accepts an exported GITHUB_TOKEN for manual runs, and dies up front when publishing is enabled with neither. Full CLI sandboxing is tracked in LIT-5420 --- tests/e2e/claude_code/cron_vm/README.md | 20 +++++++++++----- .../cron_vm/litellm-compat-matrix.env.example | 24 +++++++++++-------- .../cron_vm/litellm-compat-matrix.service | 14 ++++++++++- tests/e2e/claude_code/cron_vm/run_daily.sh | 22 +++++++++++++---- 4 files changed, 58 insertions(+), 22 deletions(-) diff --git a/tests/e2e/claude_code/cron_vm/README.md b/tests/e2e/claude_code/cron_vm/README.md index c1a4eaa2169..f120c30605b 100644 --- a/tests/e2e/claude_code/cron_vm/README.md +++ b/tests/e2e/claude_code/cron_vm/README.md @@ -118,11 +118,16 @@ git -C ~/litellm/litellm checkout litellm_internal_staging # 4. gh auth — must be a collaborator on BerriAI/litellm-docs. gh auth login # follow prompts; pick HTTPS + token paste flow -# 5. Provider credentials. +# 5. Provider credentials + the publish token. sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \ /etc/litellm-compat-matrix.env sudoedit /etc/litellm-compat-matrix.env # fill in real values sudo chmod 0600 /etc/litellm-compat-matrix.env +# The mateo-berri PAT lives in its own file, mapped into the service via +# systemd LoadCredential so it stays out of the test processes' env +# (see the env.example comment for why). +sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token +sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT # 6. systemd units. sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/ @@ -171,13 +176,16 @@ sudo systemctl disable --now litellm-compat-matrix.timer - **`uv sync --frozen` requires the resolved tag to be tagged on GitHub.** If the latest stable release was made but not pushed as a git tag, the `git checkout` step fails. Push the tag, then rerun. -- **`GITHUB_TOKEN` rotation is your problem.** The cron does not +- **Publish-token rotation is your problem.** The cron does not refresh the token; if `mateo-berri`'s PAT in - `/etc/litellm-compat-matrix.env` expires, the run fails at the - `git push`/`gh pr create` step with a 401 ("Bad credentials" / - "Authentication failed"). Mint a fresh PAT and update the env file. + `/etc/litellm-compat-matrix-github-token` expires, the run fails at + the `git push`/`gh pr create` step with a 401 ("Bad credentials" / + "Authentication failed"). Mint a fresh PAT and update that file. The token needs write access to `BerriAI/litellm-docs` (classic - `repo` scope, or fine-grained Contents:RW + Pull requests:RW). + `repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is + delivered via systemd `LoadCredential`, not the env file, so pytest, + the proxy, and the claude CLI never inherit it; manual runs export + `GITHUB_TOKEN` instead. - **First run after upgrading the Claude Code CLI is the riskiest one.** If the new CLI changes its wire format the matrix run can produce systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example index e1e98b98120..d15561e96cd 100644 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example @@ -34,16 +34,20 @@ OPENAI_API_KEY= AZURE_API_BASE= AZURE_API_KEY= -# REQUIRED for publishing: PAT for the `mateo-berri` user, who has write -# access on BerriAI/litellm-docs. Used to (a) resolve the latest stable -# release, (b) push the daily compat-matrix branch directly to -# BerriAI/litellm-docs, (c) open the same-repo PR, and (d) enable -# squash auto-merge on it. Scopes: classic `repo` + `workflow`, or -# fine-grained on BerriAI/litellm-docs with Contents:RW + Pull -# requests:RW + Workflows:RW. -# Skip by setting SKIP_PUBLISH=1 (publishes nothing; only writes the -# matrix JSON locally). -GITHUB_TOKEN= +# The publish PAT (mateo-berri, write access on BerriAI/litellm-docs) +# deliberately does NOT live in this file. Everything here lands in the +# process environment of pytest, the proxy, and the model-driven claude +# CLI, where any same-UID reader can lift it from /proc//environ. +# Instead, install the token at /etc/litellm-compat-matrix-github-token +# (chmod 0600, single line); the service maps it in via systemd +# LoadCredential and run_daily.sh keeps it out of every child process +# env. Used to (a) resolve the latest stable release, (b) push the +# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open +# the same-repo PR, and (d) enable squash auto-merge on it. Scopes: +# classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs +# with Contents:RW + Pull requests:RW + Workflows:RW. +# Manual runs export GITHUB_TOKEN instead, or skip publishing entirely +# with SKIP_PUBLISH=1 (only writes the matrix JSON locally). # Optional: the bedrock_mantle column is opt-in because the AWS account # needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the diff --git a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service index 9753d208135..6c74b3b04bb 100644 --- a/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service +++ b/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service @@ -28,7 +28,10 @@ # * have `gh` already authenticated against an account with # `pull-requests: write` on `BerriAI/litellm-docs`; # * have provider credentials exported in `/etc/litellm-compat-matrix.env` -# (see `litellm-compat-matrix.env.example` in this directory). +# (see `litellm-compat-matrix.env.example` in this directory); +# * have the mateo-berri publish PAT at +# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single +# line), delivered via `LoadCredential=` below. [Unit] Description=Claude Code compatibility-matrix populator (oneshot) @@ -45,6 +48,15 @@ Group=mateo # is the standard `KEY=value` one line per env var. EnvironmentFile=-/etc/litellm-compat-matrix.env +# The mateo-berri publish PAT is mapped in via the credential store, NOT +# the EnvironmentFile, so it never lands in the process environment that +# pytest, the proxy, and the model-driven claude CLI inherit (any +# same-UID process can read /proc//environ). run_daily.sh reads +# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call. +# Unlike EnvironmentFile= above, this is deliberately NOT optional: a +# missing token file fails the unit at start instead of 30 minutes in. +LoadCredential=github-token:/etc/litellm-compat-matrix-github-token + # systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin). # `uv` and `claude` are installed under the runtime user's `~/.local/bin` # so we have to prepend it explicitly; otherwise run_daily.sh fails at diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index 9d9a7abe367..00d3e66e5bc 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -113,13 +113,25 @@ for cmd in git uv gh jq curl claude; do done # Publishing pushes the branch straight to BerriAI/litellm-docs and opens -# the PR as mateo-berri, who has write access on the docs repo. The same -# ${GITHUB_TOKEN} is reused for release-listing above, so require it up -# front -- failing 30 minutes into a run because the env file is missing -# one line is a waste of CI quota. +# the PR as mateo-berri, who has write access on the docs repo. Under +# systemd the PAT arrives as a file via LoadCredential=, NOT via the +# EnvironmentFile: several suite cells let the model-driven claude CLI +# read arbitrary files as this user, and /proc//environ of the +# script, pytest, and the proxy would hand an env-borne token to any +# same-UID reader. Kept as an unexported shell variable and passed per +# invocation (GH_TOKEN=... / curl header / push URL), it never enters a +# child's environment. Manual runs may export GITHUB_TOKEN instead. +# Require it up front -- failing 30 minutes into a run is a waste of CI +# quota. +if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then + GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")" + log "publish token source: systemd credential store" +elif [[ -n "${GITHUB_TOKEN:-}" ]]; then + log "publish token source: process environment" +fi if [[ "${SKIP_PUBLISH}" != "1" ]]; then [[ -n "${GITHUB_TOKEN:-}" ]] \ - || die "GITHUB_TOKEN (mateo-berri, write access to ${DOCS_REPO}) required to push the branch and open the PR (or set SKIP_PUBLISH=1)" + || die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)" fi # --------------------------------------------------------------------------- From e368eeac496a0a8b6d16599910321b59db611a84 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:11:29 +0000 Subject: [PATCH 05/25] feat(ui): warn in the Admin UI when no Redis is configured Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../health_endpoints/_health_endpoints.py | 23 ++++++++ .../health_endpoints/test_health_endpoints.py | 55 ++++++++++++++++++ .../useHealthReadinessDetails.ts | 1 + .../src/app/(dashboard)/layout.test.tsx | 4 ++ .../src/app/(dashboard)/layout.tsx | 3 + .../components/NoRedisWarningBanner.test.tsx | 57 +++++++++++++++++++ .../src/components/NoRedisWarningBanner.tsx | 40 +++++++++++++ 7 files changed, 183 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/NoRedisWarningBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/NoRedisWarningBanner.tsx diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 52b7faeac07..29f849ccebb 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -50,6 +50,7 @@ from litellm.router_utils.clientside_credential_handler import ( _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path clientside_credential_keys, ) +from litellm.secret_managers.main import get_secret_bool #### Health ENDPOINTS #### @@ -1447,6 +1448,25 @@ def callback_name(callback): return str(callback) +DISABLE_NO_REDIS_WARNING_ENV_VAR: Final = "LITELLM_DISABLE_NO_REDIS_WARNING" + + +def _show_no_redis_warning() -> bool: + """ + Whether the UI should warn that no coordination Redis is configured. + + Redis is what makes rate limits, budgets, router state, and cache + invalidation consistent across workers, so a proxy running without it is + only safe as a single worker. Operators who know that can silence the + warning with LITELLM_DISABLE_NO_REDIS_WARNING=true. + """ + from litellm.proxy.proxy_server import redis_usage_cache + + if redis_usage_cache is not None: + return False + return get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is not True + + async def _get_health_readiness_details( response: Response | None = None, ) -> dict[str, Any]: @@ -1487,6 +1507,7 @@ async def _get_health_readiness_details( # check log level log_level_name: Final = logging.getLevelName(verbose_logger.getEffectiveLevel()) is_detailed_debug: Final = verbose_logger.isEnabledFor(logging.DEBUG) + show_no_redis_warning: Final = _show_no_redis_warning() # check DB if prisma_client is not None: # if db passed in, check if it's connected @@ -1506,6 +1527,7 @@ async def _get_health_readiness_details( "use_aiohttp_transport": AsyncHTTPHandler._should_use_aiohttp_transport(), "log_level": log_level_name, "is_detailed_debug": is_detailed_debug, + "show_no_redis_warning": show_no_redis_warning, } else: return { @@ -1517,6 +1539,7 @@ async def _get_health_readiness_details( "use_aiohttp_transport": AsyncHTTPHandler._should_use_aiohttp_transport(), "log_level": log_level_name, "is_detailed_debug": is_detailed_debug, + "show_no_redis_warning": show_no_redis_warning, } except Exception as e: raise HTTPException(status_code=503, detail=f"Service Unhealthy ({e})") diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index f74aafd9df1..c2a43502c8a 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -21,6 +21,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, + _show_no_redis_warning, get_callback_identifier, health_license_endpoint, health_services_endpoint, @@ -2457,3 +2458,57 @@ class TestConfigBaseForHealthCheck: ) assert base["litellm_credential_name"] == "OpenAI-prod" assert base["api_key"] == "sk-configured" + + +class TestNoRedisWarning: + """`show_no_redis_warning` drives the Admin UI's default-on "no Redis" banner.""" + + def test_warns_when_no_coordination_redis_is_configured(self, monkeypatch): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with patch("litellm.proxy.proxy_server.redis_usage_cache", None): + assert _show_no_redis_warning() is True + + def test_stays_quiet_when_a_coordination_redis_is_configured(self, monkeypatch): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()): + assert _show_no_redis_warning() is False + + @pytest.mark.parametrize("value", ["true", "True"]) + def test_env_var_suppresses_the_warning(self, monkeypatch, value): + monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", value) + with patch("litellm.proxy.proxy_server.redis_usage_cache", None): + assert _show_no_redis_warning() is False + + def test_env_var_set_false_keeps_the_warning(self, monkeypatch): + monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", "false") + with patch("litellm.proxy.proxy_server.redis_usage_cache", None): + assert _show_no_redis_warning() is True + + @pytest.mark.asyncio + @pytest.mark.parametrize("has_prisma_client", [True, False]) + async def test_readiness_details_carries_the_flag(self, monkeypatch, has_prisma_client): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + prisma_client = MagicMock() if has_prisma_client else None + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch.object( + _health_endpoints_module, + "_db_health_readiness_check", + AsyncMock(return_value={"status": "connected"}), + ), + ): + details = await _health_endpoints_module._get_health_readiness_details() + assert details["show_no_redis_warning"] is True + + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()), + patch.object( + _health_endpoints_module, + "_db_health_readiness_check", + AsyncMock(return_value={"status": "connected"}), + ), + ): + details = await _health_endpoints_module._get_health_readiness_details() + assert details["show_no_redis_warning"] is False diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts index 3b79e5c7643..307fa9e1691 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails.ts @@ -13,6 +13,7 @@ export interface HealthReadinessDetailsResponse { use_aiohttp_transport?: boolean; log_level?: string; is_detailed_debug?: boolean; + show_no_redis_warning?: boolean; } const fetchHealthReadinessDetails = async (accessToken: string): Promise => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index 7973855ebd4..340077e9569 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -25,6 +25,10 @@ vi.mock("@/components/DebugWarningBanner", () => ({ DebugWarningBanner: () => null, })); +vi.mock("@/components/NoRedisWarningBanner", () => ({ + NoRedisWarningBanner: () => null, +})); + vi.mock("@/components/LicenseExpiryBanner", () => ({ LicenseExpiryBanner: () => null, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index fb3a4db58f7..5f9e2bc846f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -9,6 +9,7 @@ import { useAuth } from "@/contexts/AuthContext"; import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; import { useRouter, useSearchParams, usePathname } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; +import { NoRedisWarningBanner } from "@/components/NoRedisWarningBanner"; import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner"; import { UserBanner } from "@/components/UserBanner"; import { MIGRATED_PAGES, migratedHref, legacyPageHref, legacyKeyForPathname } from "@/utils/migratedPages"; @@ -120,6 +121,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
+
@@ -143,6 +145,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) {
+
{children}
diff --git a/ui/litellm-dashboard/src/components/NoRedisWarningBanner.test.tsx b/ui/litellm-dashboard/src/components/NoRedisWarningBanner.test.tsx new file mode 100644 index 00000000000..8afde8eec94 --- /dev/null +++ b/ui/litellm-dashboard/src/components/NoRedisWarningBanner.test.tsx @@ -0,0 +1,57 @@ +import { renderWithProviders, screen } from "../../tests/test-utils"; +import { vi } from "vitest"; +import { NoRedisWarningBanner } from "./NoRedisWarningBanner"; +import type { HealthReadinessDetailsResponse } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; +import type { UseQueryResult } from "@tanstack/react-query"; + +vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails", () => ({ + useHealthReadinessDetails: vi.fn(), +})); + +import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; + +const mockDetails = (data: Partial | undefined) => { + vi.mocked(useHealthReadinessDetails).mockReturnValue({ data } as UseQueryResult); +}; + +describe("NoRedisWarningBanner", () => { + it("should warn that Redis is recommended when the proxy reports no Redis", () => { + mockDetails({ status: "healthy", show_no_redis_warning: true }); + renderWithProviders(); + expect(screen.getByRole("alert")).toBeInTheDocument(); + expect(screen.getByText(/No Redis configured\. Redis is highly recommended/i)).toBeInTheDocument(); + }); + + it("should link to the docs page listing what breaks without Redis", () => { + mockDetails({ status: "healthy", show_no_redis_warning: true }); + renderWithProviders(); + expect(screen.getByRole("link", { name: /does not work without Redis/i })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/proxy/redis_requirements", + ); + }); + + it("should name the env var that suppresses it", () => { + mockDetails({ status: "healthy", show_no_redis_warning: true }); + renderWithProviders(); + expect(screen.getByText("LITELLM_DISABLE_NO_REDIS_WARNING=true")).toBeInTheDocument(); + }); + + it("should render nothing when the proxy reports the warning is not needed", () => { + mockDetails({ status: "healthy", show_no_redis_warning: false }); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should render nothing when readiness details are unavailable", () => { + mockDetails(undefined); + const { container } = renderWithProviders(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should pass the access token to the readiness hook", () => { + mockDetails(undefined); + renderWithProviders(); + expect(useHealthReadinessDetails).toHaveBeenCalledWith("my-token"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/NoRedisWarningBanner.tsx b/ui/litellm-dashboard/src/components/NoRedisWarningBanner.tsx new file mode 100644 index 00000000000..93c0f55486d --- /dev/null +++ b/ui/litellm-dashboard/src/components/NoRedisWarningBanner.tsx @@ -0,0 +1,40 @@ +"use client"; + +import React from "react"; +import { TriangleAlert } from "lucide-react"; +import { useHealthReadinessDetails } from "@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails"; + +const REDIS_DOCS_URL = "https://docs.litellm.ai/docs/proxy/redis_requirements"; + +interface NoRedisWarningBannerProps { + accessToken: string | null; +} + +export const NoRedisWarningBanner: React.FC = ({ accessToken }) => { + const { data: healthData } = useHealthReadinessDetails(accessToken); + + if (!healthData?.show_no_redis_warning) { + return null; + } + + return ( + + ); +}; From 48fa4a0f06c2ed5f6d31cfae58e8184102142aa3 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 02:40:14 +0000 Subject: [PATCH 06/25] fix(ui): treat router redis as configured for the no-redis banner Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../skills/testing-admin-ui-banners/SKILL.md | 75 +++++++++++++++++++ .../health_endpoints/_health_endpoints.py | 12 ++- .../health_endpoints/test_health_endpoints.py | 44 +++++++++-- 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 .agents/skills/testing-admin-ui-banners/SKILL.md diff --git a/.agents/skills/testing-admin-ui-banners/SKILL.md b/.agents/skills/testing-admin-ui-banners/SKILL.md new file mode 100644 index 00000000000..615229d4a52 --- /dev/null +++ b/.agents/skills/testing-admin-ui-banners/SKILL.md @@ -0,0 +1,75 @@ +--- +name: testing-admin-ui-banners +description: How to run the LiteLLM Admin UI dev server against a live proxy (including a second BEFORE/base worktree) to test dashboard shell banners and /health/readiness/details driven UI state. +--- + +# Testing Admin UI dashboard banners against a live proxy + +## Bring up AFTER (branch under test) + +``` +sudo service postgresql start +cd && (setsid uv run --no-sync litellm --config litellm/proxy/dev_config.yaml --detailed_debug --port 4000 > /tmp/proxy.log 2>&1 < /dev/null &) +cd ui/litellm-dashboard && (npm run dev > /tmp/ui_dev.log 2>&1 &) # port 3000 +``` + +Proxy startup takes ~45-60s before `/health/readiness/details` answers. Log in at +http://localhost:3000/ (it redirects to the proxy's login page) with `admin` / +the `general_settings.master_key` from `litellm/proxy/dev_config.yaml` (`sk-1234` by default). +In dev (`NODE_ENV=development`) the UI defaults its API base to `http://localhost:4000`, so no +extra env var is needed for the main dev server. + +Launcher gotcha: if an `exec` shell call runs longer than ~10s it gets backgrounded and can take +the freshly spawned proxy with it. Keep the launch command short (`setsid ... & ; sleep 6`) and +poll readiness in a separate call. + +## Bring up BEFORE (base commit) side by side + +``` +git worktree add /home/ubuntu/repos/litellm-base +cp -al /ui/litellm-dashboard/node_modules /home/ubuntu/repos/litellm-base/ui/litellm-dashboard/node_modules +``` + +Do NOT symlink `node_modules` into a worktree: Turbopack panics with +"Symlink [project]/node_modules is invalid, it points out of the filesystem root". A hardlink copy +(`cp -al`) works and is fast. + +Run the base proxy with the main venv but the base source tree, and point the base UI at it: + +``` +cd /home/ubuntu/repos/litellm-base && PYTHONPATH=$PWD /.venv/bin/python -m litellm.proxy.proxy_cli --config /litellm/proxy/dev_config.yaml --detailed_debug --port 4001 +cd /home/ubuntu/repos/litellm-base/ui/litellm-dashboard && NEXT_PUBLIC_BASE_URL=http://localhost:4001 npm run dev -- --port 3001 +``` + +`PYTHONPATH` wins over the editable install, so the base proxy really runs base code (verify with +`python -c "import litellm; print(litellm.__file__)"`). + +## Banner-specific notes + +Dashboard shell banners (`DebugWarningBanner`, `NoRedisWarningBanner`, `LicenseExpiryBanner`) all +read `useHealthReadinessDetails`, which has `staleTime: 5 min` and `retry: false`. After restarting +the proxy with different env, hard-reload the page (ctrl+shift+r) or the cached readiness payload +keeps the old banner state. Running the proxy with `--detailed_debug` always shows the yellow debug +banner, which is a handy control: if it is present but the banner under test is not, the readiness +call succeeded and the banner condition really is false. + +Coordination Redis (`litellm.proxy.proxy_server.redis_usage_cache`, which drives +`show_no_redis_warning`) is NOT populated by `REDIS_HOST`/`REDIS_PORT` alone: the env fallback only +runs inside `_init_cache`, which requires a cache block in the config. To get a real coordination +Redis, run `docker run -d -p 6379:6379 redis:7` and add to the config: + +``` +litellm_settings: + cache: true + cache_params: + type: redis + host: localhost + port: 6379 +``` + +`general_settings.coordination_redis` is the other supported path. + +## Devin Secrets Needed + +None for banner/UI-state testing; the proxy boots with the bundled dev config and a local Postgres. +Provider keys are only needed when a test actually issues LLM requests. diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 29f849ccebb..e814ec42d26 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1453,17 +1453,23 @@ DISABLE_NO_REDIS_WARNING_ENV_VAR: Final = "LITELLM_DISABLE_NO_REDIS_WARNING" def _show_no_redis_warning() -> bool: """ - Whether the UI should warn that no coordination Redis is configured. + Whether the UI should warn that no Redis is configured. Redis is what makes rate limits, budgets, router state, and cache invalidation consistent across workers, so a proxy running without it is - only safe as a single worker. Operators who know that can silence the + only safe as a single worker. Both places a Redis can land count: the + coordination cache (from a Redis response cache, general_settings. + coordination_redis, or the REDIS_* env fallback) and the router's own + Redis (router_settings.redis_host), which backs cooldowns and usage-based + routing on its own. Operators who know they run one worker can silence the warning with LITELLM_DISABLE_NO_REDIS_WARNING=true. """ - from litellm.proxy.proxy_server import redis_usage_cache + from litellm.proxy.proxy_server import llm_router, redis_usage_cache if redis_usage_cache is not None: return False + if llm_router is not None and llm_router.cache.redis_cache is not None: + return False return get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is not True diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index c2a43502c8a..e2705bd5fec 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -2463,25 +2463,58 @@ class TestConfigBaseForHealthCheck: class TestNoRedisWarning: """`show_no_redis_warning` drives the Admin UI's default-on "no Redis" banner.""" - def test_warns_when_no_coordination_redis_is_configured(self, monkeypatch): + @staticmethod + def _router(redis_cache): + return SimpleNamespace(cache=SimpleNamespace(redis_cache=redis_cache)) + + def test_warns_when_no_redis_is_configured(self, monkeypatch): monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) - with patch("litellm.proxy.proxy_server.redis_usage_cache", None): + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + ): + assert _show_no_redis_warning() is True + + def test_warns_when_there_is_no_router_at_all(self, monkeypatch): + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", None), + ): assert _show_no_redis_warning() is True def test_stays_quiet_when_a_coordination_redis_is_configured(self, monkeypatch): monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) - with patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()): + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", MagicMock()), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + ): + assert _show_no_redis_warning() is False + + def test_stays_quiet_when_only_the_router_has_redis(self, monkeypatch): + """router_settings.redis_host alone backs cooldowns and usage-based routing.""" + monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False) + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(MagicMock())), + ): assert _show_no_redis_warning() is False @pytest.mark.parametrize("value", ["true", "True"]) def test_env_var_suppresses_the_warning(self, monkeypatch, value): monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", value) - with patch("litellm.proxy.proxy_server.redis_usage_cache", None): + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + ): assert _show_no_redis_warning() is False def test_env_var_set_false_keeps_the_warning(self, monkeypatch): monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", "false") - with patch("litellm.proxy.proxy_server.redis_usage_cache", None): + with ( + patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), + ): assert _show_no_redis_warning() is True @pytest.mark.asyncio @@ -2492,6 +2525,7 @@ class TestNoRedisWarning: with ( patch("litellm.proxy.proxy_server.prisma_client", prisma_client), patch("litellm.proxy.proxy_server.redis_usage_cache", None), + patch("litellm.proxy.proxy_server.llm_router", self._router(None)), patch.object( _health_endpoints_module, "_db_health_readiness_check", From e06d1036d298a2425518a52604a2381610061460 Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 21:36:24 +0000 Subject: [PATCH 07/25] chore: drop the admin ui banner testing skill Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../skills/testing-admin-ui-banners/SKILL.md | 75 ------------------- 1 file changed, 75 deletions(-) delete mode 100644 .agents/skills/testing-admin-ui-banners/SKILL.md diff --git a/.agents/skills/testing-admin-ui-banners/SKILL.md b/.agents/skills/testing-admin-ui-banners/SKILL.md deleted file mode 100644 index 615229d4a52..00000000000 --- a/.agents/skills/testing-admin-ui-banners/SKILL.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -name: testing-admin-ui-banners -description: How to run the LiteLLM Admin UI dev server against a live proxy (including a second BEFORE/base worktree) to test dashboard shell banners and /health/readiness/details driven UI state. ---- - -# Testing Admin UI dashboard banners against a live proxy - -## Bring up AFTER (branch under test) - -``` -sudo service postgresql start -cd && (setsid uv run --no-sync litellm --config litellm/proxy/dev_config.yaml --detailed_debug --port 4000 > /tmp/proxy.log 2>&1 < /dev/null &) -cd ui/litellm-dashboard && (npm run dev > /tmp/ui_dev.log 2>&1 &) # port 3000 -``` - -Proxy startup takes ~45-60s before `/health/readiness/details` answers. Log in at -http://localhost:3000/ (it redirects to the proxy's login page) with `admin` / -the `general_settings.master_key` from `litellm/proxy/dev_config.yaml` (`sk-1234` by default). -In dev (`NODE_ENV=development`) the UI defaults its API base to `http://localhost:4000`, so no -extra env var is needed for the main dev server. - -Launcher gotcha: if an `exec` shell call runs longer than ~10s it gets backgrounded and can take -the freshly spawned proxy with it. Keep the launch command short (`setsid ... & ; sleep 6`) and -poll readiness in a separate call. - -## Bring up BEFORE (base commit) side by side - -``` -git worktree add /home/ubuntu/repos/litellm-base -cp -al /ui/litellm-dashboard/node_modules /home/ubuntu/repos/litellm-base/ui/litellm-dashboard/node_modules -``` - -Do NOT symlink `node_modules` into a worktree: Turbopack panics with -"Symlink [project]/node_modules is invalid, it points out of the filesystem root". A hardlink copy -(`cp -al`) works and is fast. - -Run the base proxy with the main venv but the base source tree, and point the base UI at it: - -``` -cd /home/ubuntu/repos/litellm-base && PYTHONPATH=$PWD /.venv/bin/python -m litellm.proxy.proxy_cli --config /litellm/proxy/dev_config.yaml --detailed_debug --port 4001 -cd /home/ubuntu/repos/litellm-base/ui/litellm-dashboard && NEXT_PUBLIC_BASE_URL=http://localhost:4001 npm run dev -- --port 3001 -``` - -`PYTHONPATH` wins over the editable install, so the base proxy really runs base code (verify with -`python -c "import litellm; print(litellm.__file__)"`). - -## Banner-specific notes - -Dashboard shell banners (`DebugWarningBanner`, `NoRedisWarningBanner`, `LicenseExpiryBanner`) all -read `useHealthReadinessDetails`, which has `staleTime: 5 min` and `retry: false`. After restarting -the proxy with different env, hard-reload the page (ctrl+shift+r) or the cached readiness payload -keeps the old banner state. Running the proxy with `--detailed_debug` always shows the yellow debug -banner, which is a handy control: if it is present but the banner under test is not, the readiness -call succeeded and the banner condition really is false. - -Coordination Redis (`litellm.proxy.proxy_server.redis_usage_cache`, which drives -`show_no_redis_warning`) is NOT populated by `REDIS_HOST`/`REDIS_PORT` alone: the env fallback only -runs inside `_init_cache`, which requires a cache block in the config. To get a real coordination -Redis, run `docker run -d -p 6379:6379 redis:7` and add to the config: - -``` -litellm_settings: - cache: true - cache_params: - type: redis - host: localhost - port: 6379 -``` - -`general_settings.coordination_redis` is the other supported path. - -## Devin Secrets Needed - -None for banner/UI-state testing; the proxy boots with the bundled dev config and a local Postgres. -Provider keys are only needed when a test actually issues LLM requests. From 96c82f1c0c37bdd9f0201af00b57fa074216f99d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:07:39 -0700 Subject: [PATCH 08/25] fix(router): forward auto-router alias params from the marker entry, not the first same-name deployment --- litellm/router.py | 65 +++++-- .../router_strategy/test_complexity_router.py | 172 ++++++++++++++++-- 2 files changed, 203 insertions(+), 34 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8917ef60e36..e5db6c1d392 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -95,6 +95,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( response_in_flight_token_count, ) from litellm.router_utils.auto_router_model_naming import ( + AUTO_ROUTER_MODEL_PREFIX, classify_strategy_router_model, ) from litellm.router_utils.batch_utils import ( @@ -316,6 +317,8 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") +_ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) + def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream]) -> bool: for chunk in chunks: @@ -11339,11 +11342,15 @@ class Router: return filtered - def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None": + def _select_pre_routing_strategy( + self, model: str, request_kwargs: dict + ) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None": """ Resolve the pre-routing strategy for `model`, disambiguating deployments that share a `model_name` by matching the request's tags against each registered strategy's tags before falling back to the first registered. + Returns the tagged wrapper so callers can locate the marker deployment + the strategy was registered from via its (model_name, tags) pair. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11354,7 +11361,7 @@ class Router: if not candidates: return None if len(candidates) == 1: - return candidates[0].strategy + return candidates[0] request_tags: Final = _get_tags_from_request_kwargs(request_kwargs) if request_tags: @@ -11362,11 +11369,11 @@ class Router: if tagged.tags and is_valid_deployment_tag( list(tagged.tags), request_tags, self.tag_filtering_match_any ): - return tagged.strategy + return tagged for tagged in candidates: if "default" in tagged.tags: - return tagged.strategy - return candidates[0].strategy + return tagged + return candidates[0] async def async_pre_routing_hook( self, @@ -11390,15 +11397,15 @@ class Router: if self.routing_plugins: await self._run_routing_plugins(model=model, request_kwargs=request_kwargs, messages=messages) - router_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs) - if router_strategy is None: + tagged_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs) + if tagged_strategy is None: self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None ) return None - pre_routing_hook_response: Final = await router_strategy.async_pre_routing_hook( + pre_routing_hook_response: Final = await tagged_strategy.strategy.async_pre_routing_hook( model=model, request_kwargs=request_kwargs, messages=messages, @@ -11416,22 +11423,42 @@ class Router: ) # `model` (the alias, e.g. "smart-router") is never the deployment actually - # called - apply the alias's own litellm_params (besides `model` itself, - # which is just the alias marker) to the request, since the tier/route - # deployment the hook selected won't have them. Router-only fields - # (tpm, rpm, weight, complexity_router_config, ...) are excluded from the - # actual outbound LLM call downstream by litellm.types.utils.all_litellm_params, + # called - apply the router marker's own litellm_params to the request, + # since the tier/route deployment the hook selected won't have them. The + # marker entry is looked up by its `auto_router/` model prefix and the + # selected strategy's tags, never by list position: plain deployments may + # share the alias `model_name` and must not leak their params (`api_base`, + # `api_key`, ...) onto the routed call. Router-only fields (tpm, rpm, + # weight, complexity_router_config, ...) are excluded from the actual + # outbound LLM call downstream by litellm.types.utils.all_litellm_params, # not here. if pre_routing_hook_response is not None: - alias_index: Final = self.model_name_to_deployment_indices.get(model, []) - if alias_index: - alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {}) - for key, value in alias_litellm_params.items(): - if key != "model" and value is not None: - request_kwargs.setdefault(key, value) + for key, value in self._forwardable_alias_marker_params(model=model, strategy_tags=tagged_strategy.tags): + request_kwargs.setdefault(key, value) return pre_routing_hook_response + def _forwardable_alias_marker_params( + self, model: str, strategy_tags: tuple[str, ...] + ) -> tuple[tuple[str, object], ...]: + marker_params: Final = tuple( + litellm_params + for idx in self.model_name_to_deployment_indices.get(model, ()) + if isinstance(litellm_params := self.model_list[idx].get("litellm_params", {}), dict) + and str(litellm_params.get("model", "")).startswith(AUTO_ROUTER_MODEL_PREFIX) + ) + tag_matched: Final = tuple( + params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags + ) + selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None) + if selected is None: + return () + return tuple( + (key, value) + for key, value in selected.items() + if key not in _ALIAS_PARAMS_NEVER_FORWARDED and value is not None + ) + @staticmethod def _record_routing_decision( request_kwargs: dict, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 94b6b68855b..a284e51091a 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1157,8 +1157,8 @@ class TestPreRoutingStrategyRegistry: TaggedPreRoutingStrategy(tags=("us",), strategy=us), ] } - assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us - assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn assert router._select_pre_routing_strategy("missing", {"metadata": {"tags": ["cn"]}}) is None router.complexity_routers = { @@ -1167,14 +1167,14 @@ class TestPreRoutingStrategyRegistry: TaggedPreRoutingStrategy(tags=("default",), strategy=fallback), ] } - assert router._select_pre_routing_strategy("smart", {}) is fallback + assert router._select_pre_routing_strategy("smart", {}).strategy is fallback router.complexity_routers = { "smart": [ TaggedPreRoutingStrategy(tags=("cn",), strategy=cn), TaggedPreRoutingStrategy(tags=("us",), strategy=us), ] } - assert router._select_pre_routing_strategy("smart", {}) is cn + assert router._select_pre_routing_strategy("smart", {}).strategy is cn class TestAsyncPreRoutingHookMultiFormat: @@ -2041,14 +2041,16 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["cache_control_injection_points"] == [{"location": "message", "role": "system"}] @pytest.mark.asyncio - async def test_alias_overrides_exclude_only_model(self): - """`model` (the alias marker, e.g. auto_router/complexity_router) is - excluded since it's never a real provider model. Router-only fields - like complexity_router_config DO flow through into request_kwargs at - this layer - they're filtered from the actual outbound LLM call - downstream by litellm.types.utils.all_litellm_params instead, not by - the router's pre-routing hook. See test_router_init_only_params_are_ - never_sent_to_a_provider for the guard on that downstream filter.""" + async def test_alias_overrides_exclude_only_marker_and_connection_params(self): + """`model` (the alias marker, e.g. auto_router/complexity_router) and + provider-connection params (api_base/api_key/api_version) are excluded + since they never describe the tier deployment actually called. + Router-only fields like complexity_router_config DO flow through into + request_kwargs at this layer - they're filtered from the actual + outbound LLM call downstream by litellm.types.utils.all_litellm_params + instead, not by the router's pre-routing hook. See + test_router_init_only_params_are_never_sent_to_a_provider for the + guard on that downstream filter.""" router = self._make_router() request_kwargs: Dict = {} @@ -2068,9 +2070,10 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["complexity_router_default_model"] == "gpt-4o" def test_router_init_only_params_are_never_sent_to_a_provider(self): - """The router's pre-routing hook only excludes `model` (see - test_alias_overrides_exclude_only_model above) - every other alias - litellm_param, including router-init-only fields like + """The router's pre-routing hook only excludes `model` and + provider-connection params (see test_alias_overrides_exclude_only_ + marker_and_connection_params above) - every other alias litellm_param, + including router-init-only fields like complexity_router_config, flows into request_kwargs unfiltered. That's only safe because litellm.completion()/acompletion() itself strips anything listed in all_litellm_params before building the provider @@ -2163,6 +2166,145 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["drop_params"] is True +class TestRouterPreRoutingSharedAliasName: + """ + Regression tests for https://github.com/BerriAI/litellm/issues/36619. + + A plain deployment and an `auto_router/` marker can share a `model_name`. + The alias-param forwarding after a pre-routing rewrite must read the + marker entry, never whichever same-name entry happens to sit first in + `model_list` - otherwise the plain entry's api_base/api_key get grafted + onto the routed tier's call (a Gemini path under api.openai.com, 404). + """ + + @staticmethod + def _plain_entry() -> dict: + return { + "model_name": "gpt4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-plain-entry", + "api_base": "https://plain-entry.example/v1", + }, + } + + @staticmethod + def _marker_entry() -> dict: + return { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/complexity_router", + "drop_params": True, + "complexity_router_config": {"tiers": {"SIMPLE": "gemini-flash", "MEDIUM": "gemini-flash"}}, + "complexity_router_default_model": "gemini-flash", + }, + } + + @staticmethod + def _tier_entry() -> dict: + return { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "api_key": "sk-tier"}, + } + + @pytest.mark.asyncio + @pytest.mark.parametrize("plain_entry_first", [True, False], ids=["plain_entry_first", "marker_entry_first"]) + async def test_marker_params_forwarded_regardless_of_model_list_order(self, plain_entry_first): + """In either config order the routed call gets the marker's own params + (drop_params) and never the plain sibling's api_base/api_key.""" + shared_name_entries = ( + [self._plain_entry(), self._marker_entry()] + if plain_entry_first + else [self._marker_entry(), self._plain_entry()] + ) + router = Router(model_list=[*shared_name_entries, self._tier_entry()]) + request_kwargs: Dict = {} + + result = await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + assert result is not None + assert result.model == "gemini-flash" + assert "api_base" not in request_kwargs + assert "api_key" not in request_kwargs + assert request_kwargs["drop_params"] is True + + @pytest.mark.asyncio + async def test_connection_params_on_the_marker_itself_are_not_forwarded(self): + """Even when the marker entry carries api_base/api_key/api_version, + they describe no real deployment and must not reach the routed call, + while the marker's other params still do.""" + marker_with_connection_params = { + "model_name": "smart", + "litellm_params": { + **self._marker_entry()["litellm_params"], + "api_key": "sk-marker", + "api_base": "https://marker.example/v1", + "api_version": "2024-01-01", + }, + } + router = Router(model_list=[marker_with_connection_params, self._tier_entry()]) + request_kwargs: Dict = {} + + result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert result is not None + assert "api_base" not in request_kwargs + assert "api_key" not in request_kwargs + assert "api_version" not in request_kwargs + assert request_kwargs["drop_params"] is True + + @pytest.mark.asyncio + async def test_tag_scoped_markers_forward_the_selected_markers_params(self): + """With two tag-scoped markers under one name, the forwarded params + come from the marker whose tags matched the request, not from the + first marker in the list.""" + + def tagged_marker(routed_model: str, tags: list, drop_params: bool | None) -> dict: + return { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": routed_model, + "complexity_router_config": {"tiers": {"SIMPLE": [routed_model], "MEDIUM": [routed_model]}}, + "tags": tags, + **({"drop_params": drop_params} if drop_params is not None else {}), + }, + } + + router = Router( + model_list=[ + tagged_marker("gpt-cn", ["cn"], None), + tagged_marker("gpt-us", ["us"], True), + ] + ) + + us_kwargs: Dict = {"metadata": {"tags": ["us"]}} + us_result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=us_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + assert us_result is not None and us_result.model == "gpt-us" + assert us_kwargs["drop_params"] is True + + cn_kwargs: Dict = {"metadata": {"tags": ["cn"]}} + cn_result = await router.async_pre_routing_hook( + model="smart", + request_kwargs=cn_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + assert cn_result is not None and cn_result.model == "gpt-cn" + assert "drop_params" not in cn_kwargs + + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self): config = ComplexityRouterConfig( From aa2426365126048beb1ba75104fdddecd67001b3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:09:05 -0700 Subject: [PATCH 09/25] fix(router): let untagged requests bypass a tagged pre-routing strategy on shared model names --- litellm/router.py | 22 +++++- .../router_strategy/test_complexity_router.py | 35 +++++++++ tests/test_litellm/test_router.py | 76 +++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8917ef60e36..ac1a3101f01 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11339,11 +11339,25 @@ class Router: return filtered + def _model_name_has_plain_deployments(self, model: str) -> bool: + """True when `model` also names regular (non strategy-router) deployments in the model_list.""" + indices: Final = self.model_name_to_deployment_indices.get(model) or () + return any( + classify_strategy_router_model(lp.get("model") or "") is None + for idx in indices + if (lp := self.model_list[idx].get("litellm_params")) + ) + def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None": """ Resolve the pre-routing strategy for `model`, disambiguating deployments that share a `model_name` by matching the request's tags against each registered strategy's tags before falling back to the first registered. + + With tag filtering enabled, strategies that all carry real tags matching + none of the request's do not capture it when the name also has plain + deployments: returning None hands the request to ordinary tag-aware + deployment selection. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11353,8 +11367,6 @@ class Router: ] if not candidates: return None - if len(candidates) == 1: - return candidates[0].strategy request_tags: Final = _get_tags_from_request_kwargs(request_kwargs) if request_tags: @@ -11366,6 +11378,12 @@ class Router: for tagged in candidates: if "default" in tagged.tags: return tagged.strategy + if ( + self.enable_tag_filtering + and all(tagged.tags for tagged in candidates) + and self._model_name_has_plain_deployments(model) + ): + return None return candidates[0].strategy async def async_pre_routing_hook( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 94b6b68855b..8abe80ca0d7 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry: } assert router._select_pre_routing_strategy("smart", {}) is cn + @staticmethod + def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router: + return Router( + model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}], + enable_tag_filtering=enable_tag_filtering, + ) + + def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=True) + cn, us = object(), object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn + + router.complexity_routers = { + "smart": [ + TaggedPreRoutingStrategy(tags=("cn",), strategy=cn), + TaggedPreRoutingStrategy(tags=("us",), strategy=us), + ] + } + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us + + router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)] + assert router._select_pre_routing_strategy("router-only", {}) is cn + + def test_select_keeps_capturing_when_tag_filtering_is_disabled(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=False) + cn = object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}) is cn + class TestAsyncPreRoutingHookMultiFormat: """Test async_pre_routing_hook with multiple input formats.""" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0a16b998f82..48931bd2fe1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7521,6 +7521,82 @@ class TestAutoRouterMaxInputCharsWiring: assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS +class TestTaggedAutoRouterOnSharedModelName: + """A tagged auto-router marker sharing its model_name with a plain deployment must not + capture requests whose tags don't match it when tag filtering is enabled (#36620).""" + + class _FixedRouteLayer: + def __call__(self, text: str): + from semantic_router.schema import RouteChoice + + return RouteChoice(name="gemini-flash") + + @classmethod + def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router": + pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra") + marker = { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/gpt4o-router", + "auto_router_config": json.dumps( + {"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]} + ), + "auto_router_default_model": "gemini-flash", + "auto_router_embedding_model": "text-embedding-3-small", + **({"tags": marker_tags} if marker_tags else {}), + }, + } + plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}} + tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}} + router = litellm.Router( + model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier], + enable_tag_filtering=enable_tag_filtering, + ) + router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer() + return router + + @staticmethod + async def _hook_response(router: "litellm.Router", request_kwargs: dict): + return await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + @pytest.mark.asyncio + async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + assert await self._hook_response(router, {}) is None + + @pytest.mark.asyncio + async def test_request_tagged_for_the_marker_is_still_semantically_routed(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {"metadata": {"tags": ["route"]}}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_marker_only_alias_still_captures_untagged_requests(self): + router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self): + router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: from litellm.types.router import AllowedFailsPolicy From bcba392b214977b5e7cf416b46edcf20f9637722 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:44:13 -0700 Subject: [PATCH 10/25] fix(router): exclude strategy marker deployments from selection when plain siblings exist --- litellm/router.py | 22 ++++++++++++++++------ tests/test_litellm/test_router.py | 12 ++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac1a3101f01..60e93a0526c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10699,6 +10699,15 @@ class Router: return None + @staticmethod + def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: + """True when the deployment is a strategy-router pseudo-model (`auto_router/` prefixed).""" + litellm_params: Final = deployment.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return False + deployment_model: Final = litellm_params.get("model") + return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None + def _common_checks_available_deployment( self, model: str, @@ -10826,7 +10835,12 @@ class Router: model ] # update the model to the actual value if an alias has been passed in - return model, healthy_deployments + marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) + if all(marker_flags) or not any(marker_flags): + return model, healthy_deployments + return model, [ # mutable-ok: matches this function's list contract expected by downstream filters + d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker + ] def _filter_deployments_by_model_access_groups( self, @@ -11342,11 +11356,7 @@ class Router: def _model_name_has_plain_deployments(self, model: str) -> bool: """True when `model` also names regular (non strategy-router) deployments in the model_list.""" indices: Final = self.model_name_to_deployment_indices.get(model) or () - return any( - classify_strategy_router_model(lp.get("model") or "") is None - for idx in indices - if (lp := self.model_list[idx].get("litellm_params")) - ) + return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None": """ diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 48931bd2fe1..640bd229b49 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7596,6 +7596,18 @@ class TestTaggedAutoRouterOnSharedModelName: assert response is not None assert response.model == "gemini-flash" + @pytest.mark.asyncio + async def test_untagged_selection_never_lands_on_the_marker_deployment(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + for _ in range(20): + deployment = await router.async_get_available_deployment( + model="gpt4o", + request_kwargs={}, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: From b7136243c7e53eed208f6b455ceb2211b2e32b80 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:17:52 -0700 Subject: [PATCH 11/25] test(router): cover the non-mapping litellm_params marker guard and drop redundant docstrings --- litellm/router.py | 2 -- tests/test_litellm/test_router.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 60e93a0526c..1eb2995f558 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10701,7 +10701,6 @@ class Router: @staticmethod def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: - """True when the deployment is a strategy-router pseudo-model (`auto_router/` prefixed).""" litellm_params: Final = deployment.get("litellm_params") if not isinstance(litellm_params, Mapping): return False @@ -11354,7 +11353,6 @@ class Router: return filtered def _model_name_has_plain_deployments(self, model: str) -> bool: - """True when `model` also names regular (non strategy-router) deployments in the model_list.""" indices: Final = self.model_name_to_deployment_indices.get(model) or () return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 640bd229b49..70c13600014 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7608,6 +7608,9 @@ class TestTaggedAutoRouterOnSharedModelName: ) assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): + assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: From 0f6e5abd491d391a336b8252db0c4da58c97a862 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:28:13 -0700 Subject: [PATCH 12/25] test(router): reference _model_name_has_plain_deployments directly for the router coverage gate --- tests/test_litellm/test_router.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 70c13600014..fe0d97b08b0 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7611,6 +7611,13 @@ class TestTaggedAutoRouterOnSharedModelName: def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False + def test_model_name_has_plain_deployments_reflects_the_pool(self): + mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + assert mixed._model_name_has_plain_deployments("gpt4o") is True + assert marker_only._model_name_has_plain_deployments("gpt4o") is False + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: From 3e41941e35300a5e406475c99100acfa3e809ca5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:36:01 -0700 Subject: [PATCH 13/25] test(router): reference _forwardable_alias_marker_params directly for the router coverage gate --- .../router_strategy/test_complexity_router.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index a284e51091a..efc9ecc74a5 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2304,6 +2304,15 @@ class TestRouterPreRoutingSharedAliasName: assert cn_result is not None and cn_result.model == "gpt-cn" assert "drop_params" not in cn_kwargs + def test_forwardable_alias_marker_params_reads_the_marker_entry_only(self): + router = Router(model_list=[self._plain_entry(), self._marker_entry(), self._tier_entry()]) + + forwarded = dict(router._forwardable_alias_marker_params(model="gpt4o", strategy_tags=())) + + assert forwarded["drop_params"] is True + assert "api_key" not in forwarded and "api_base" not in forwarded + assert router._forwardable_alias_marker_params(model="gemini-flash", strategy_tags=()) == () + class TestAdaptiveSoftFloors: def test_adaptive_defaults_use_cost_weighted_cold_policy(self): From d79b56481db15128f1643b9f66b091969e1a6d9f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:14:34 +0000 Subject: [PATCH 14/25] fix(model_prices): sync Groq registry with provider docs Add missing Groq models and provider-announced deprecation dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 77 +++++++++++++++++-- model_prices_and_context_window.json | 77 +++++++++++++++++-- 2 files changed, 138 insertions(+), 16 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b12e4a9fea3..089671c9779 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26109,11 +26109,12 @@ "supports_vision": true }, "groq/llama-3.1-8b-instant": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5e-08, "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -26121,9 +26122,10 @@ "supports_tool_choice": true }, "groq/llama-3.3-70b-versatile": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", - "max_input_tokens": 128000, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -26144,7 +26146,28 @@ "supports_response_schema": false, "supports_tool_choice": true }, + "groq/meta-llama/llama-prompt-guard-2-22m": { + "input_cost_per_token": 3e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 3e-08, + "source": "https://console.groq.com/docs/models" + }, + "groq/meta-llama/llama-prompt-guard-2-86m": { + "input_cost_per_token": 4e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 4e-08, + "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" + }, "groq/meta-llama/llama-guard-4-12b": { + "deprecation_date": "2026-03-05", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -26154,6 +26177,7 @@ "output_cost_per_token": 2e-07 }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { + "deprecation_date": "2026-03-09", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26167,6 +26191,7 @@ "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 1.1e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26180,6 +26205,7 @@ "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { + "deprecation_date": "2026-04-15", "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, "cache_read_input_token_cost": 5e-07, @@ -26197,8 +26223,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32766, - "max_tokens": 32766, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -26218,8 +26244,8 @@ "input_cost_per_token": 7.5e-08, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -26254,7 +26280,26 @@ "supports_tool_choice": true, "supports_web_search": true }, + "groq/canopylabs/orpheus-v1-english": { + "input_cost_per_character": 2.2e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" + }, + "groq/canopylabs/orpheus-arabic-saudi": { + "input_cost_per_character": 4e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/models" + }, "groq/playai-tts": { + "deprecation_date": "2025-12-31", "input_cost_per_character": 5e-05, "litellm_provider": "groq", "max_input_tokens": 10000, @@ -26262,7 +26307,23 @@ "max_tokens": 10000, "mode": "audio_speech" }, + "groq/qwen/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": true, + "supports_vision": true + }, "groq/qwen/qwen3-32b": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 2.9e-07, "litellm_provider": "groq", "max_input_tokens": 131000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b12e4a9fea3..089671c9779 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26109,11 +26109,12 @@ "supports_vision": true }, "groq/llama-3.1-8b-instant": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5e-08, "litellm_provider": "groq", - "max_input_tokens": 128000, - "max_output_tokens": 8192, - "max_tokens": 8192, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "output_cost_per_token": 8e-08, "supports_function_calling": true, @@ -26121,9 +26122,10 @@ "supports_tool_choice": true }, "groq/llama-3.3-70b-versatile": { + "deprecation_date": "2026-08-16", "input_cost_per_token": 5.9e-07, "litellm_provider": "groq", - "max_input_tokens": 128000, + "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", @@ -26144,7 +26146,28 @@ "supports_response_schema": false, "supports_tool_choice": true }, + "groq/meta-llama/llama-prompt-guard-2-22m": { + "input_cost_per_token": 3e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 3e-08, + "source": "https://console.groq.com/docs/models" + }, + "groq/meta-llama/llama-prompt-guard-2-86m": { + "input_cost_per_token": 4e-08, + "litellm_provider": "groq", + "max_input_tokens": 512, + "max_output_tokens": 512, + "max_tokens": 512, + "mode": "chat", + "output_cost_per_token": 4e-08, + "source": "https://console.groq.com/docs/model/meta-llama/llama-prompt-guard-2-86m" + }, "groq/meta-llama/llama-guard-4-12b": { + "deprecation_date": "2026-03-05", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 8192, @@ -26154,6 +26177,7 @@ "output_cost_per_token": 2e-07 }, "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { + "deprecation_date": "2026-03-09", "input_cost_per_token": 2e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26167,6 +26191,7 @@ "supports_vision": true }, "groq/meta-llama/llama-4-scout-17b-16e-instruct": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 1.1e-07, "litellm_provider": "groq", "max_input_tokens": 131072, @@ -26180,6 +26205,7 @@ "supports_vision": true }, "groq/moonshotai/kimi-k2-instruct-0905": { + "deprecation_date": "2026-04-15", "input_cost_per_token": 1e-06, "output_cost_per_token": 3e-06, "cache_read_input_token_cost": 5e-07, @@ -26197,8 +26223,8 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32766, - "max_tokens": 32766, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 6e-07, "search_context_cost_per_query": { @@ -26218,8 +26244,8 @@ "input_cost_per_token": 7.5e-08, "litellm_provider": "groq", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3e-07, "search_context_cost_per_query": { @@ -26254,7 +26280,26 @@ "supports_tool_choice": true, "supports_web_search": true }, + "groq/canopylabs/orpheus-v1-english": { + "input_cost_per_character": 2.2e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/model/canopylabs/orpheus-v1-english" + }, + "groq/canopylabs/orpheus-arabic-saudi": { + "input_cost_per_character": 4e-05, + "litellm_provider": "groq", + "max_input_tokens": 4000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "audio_speech", + "source": "https://console.groq.com/docs/models" + }, "groq/playai-tts": { + "deprecation_date": "2025-12-31", "input_cost_per_character": 5e-05, "litellm_provider": "groq", "max_input_tokens": 10000, @@ -26262,7 +26307,23 @@ "max_tokens": 10000, "mode": "audio_speech" }, + "groq/qwen/qwen3.6-27b": { + "input_cost_per_token": 6e-07, + "litellm_provider": "groq", + "max_input_tokens": 131072, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://console.groq.com/docs/model/qwen/qwen3.6-27b", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": true, + "supports_vision": true + }, "groq/qwen/qwen3-32b": { + "deprecation_date": "2026-07-17", "input_cost_per_token": 2.9e-07, "litellm_provider": "groq", "max_input_tokens": 131000, From ef305fe4ab563cdf0a6eb42c1c65049a833f01b0 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 12 Aug 2026 21:20:10 +0000 Subject: [PATCH 15/25] fix(bedrock_mantle): 1M context window and long-context pricing for GPT-5.6 Sol/Terra/Luna Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 18 +++++- model_prices_and_context_window.json | 18 +++++- .../llm_cost_calc/test_llm_cost_calc_utils.py | 64 +++++++++++++++++++ 3 files changed, 94 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b12e4a9fea3..9570c8d15de 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45804,11 +45804,15 @@ }, "bedrock_mantle/openai.gpt-5.6-sol": { "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45832,11 +45836,15 @@ }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45860,11 +45868,15 @@ }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b12e4a9fea3..9570c8d15de 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45804,11 +45804,15 @@ }, "bedrock_mantle/openai.gpt-5.6-sol": { "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_272k_tokens": 1.1e-05, "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05, "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_272k_tokens": 1.1e-06, "output_cost_per_token": 3.3e-05, + "output_cost_per_token_above_272k_tokens": 4.95e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45832,11 +45836,15 @@ }, "bedrock_mantle/openai.gpt-5.6-terra": { "input_cost_per_token": 2.2e-06, + "input_cost_per_token_above_272k_tokens": 4.4e-06, "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, "cache_read_input_token_cost": 2.2e-07, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, "output_cost_per_token": 1.32e-05, + "output_cost_per_token_above_272k_tokens": 1.98e-05, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -45860,11 +45868,15 @@ }, "bedrock_mantle/openai.gpt-5.6-luna": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_above_272k_tokens": 4.4e-07, "cache_creation_input_token_cost": 2.75e-07, + "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07, "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost_above_272k_tokens": 4.4e-08, "output_cost_per_token": 1.32e-06, + "output_cost_per_token_above_272k_tokens": 1.98e-06, "litellm_provider": "bedrock_mantle", - "max_input_tokens": 272000, + "max_input_tokens": 1000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 158cdb45f6b..5983607d708 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -485,6 +485,70 @@ def test_generic_cost_per_token_minimax_m3_above_512k_tokens(): assert round(completion_cost, 10) == round(expected_completion, 10) +@pytest.mark.parametrize( + "model", + [ + "bedrock_mantle/openai.gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-luna", + ], +) +def test_generic_cost_per_token_bedrock_mantle_gpt56_long_context(model): + """Bedrock GPT-5.6 supports a 1M context window, billed at the long-context rates above 272K.""" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_cost_map = litellm.model_cost[model] + assert model_cost_map["max_input_tokens"] == 1000000 + + cached_tokens = 100000 + completion_tokens = 1000 + + short_prompt_tokens = 272000 + short_usage = Usage( + prompt_tokens=short_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=short_prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + short_prompt_cost, short_completion_cost = generic_cost_per_token( + model=model, + usage=short_usage, + custom_llm_provider="bedrock_mantle", + ) + assert round(short_prompt_cost, 10) == round( + model_cost_map["input_cost_per_token"] * (short_prompt_tokens - cached_tokens) + + model_cost_map["cache_read_input_token_cost"] * cached_tokens, + 10, + ) + assert round(short_completion_cost, 10) == round( + model_cost_map["output_cost_per_token"] * completion_tokens, 10 + ) + + long_prompt_tokens = 900000 + long_usage = Usage( + prompt_tokens=long_prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=long_prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + long_prompt_cost, long_completion_cost = generic_cost_per_token( + model=model, + usage=long_usage, + custom_llm_provider="bedrock_mantle", + ) + assert round(long_prompt_cost, 10) == round( + model_cost_map["input_cost_per_token_above_272k_tokens"] + * (long_prompt_tokens - cached_tokens) + + model_cost_map["cache_read_input_token_cost_above_272k_tokens"] + * cached_tokens, + 10, + ) + assert round(long_completion_cost, 10) == round( + model_cost_map["output_cost_per_token_above_272k_tokens"] * completion_tokens, 10 + ) + + def test_generic_cost_per_token_honors_non_standard_above_threshold(): """Regression for #30344: get_model_info must keep arbitrary input/output_cost_per_token_above__tokens thresholds, not only the hard-coded From 1911269ddf182148c32b90294aa158d378b16bdc Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 12 Aug 2026 14:26:30 -0700 Subject: [PATCH 16/25] fix(router): never price a strategy-router alias (#36691) * fix: never price a strategy-router alias A strategy-router alias (auto_router/complexity_router/) is never the deployment that gets called or billed, but custom pricing configured on it was being treated as real pricing in two places: - registered in litellm.model_cost under the alias deployment id, so an explicit zero made _is_cost_explicitly_configured() report the group as a genuinely free model and every budget check was skipped, while the request routed to a paid deployment and accrued real spend - copied onto request_kwargs by the alias-params merge, so the routed deployment got re-registered at the alias price and the request billed 0.0 Both are fixed at the writer, so config, /model/new and price-map reload all take the same path Co-Authored-By: Claude * chore: annotate filtered cost-map copy for the mutable-collection gate Co-Authored-By: Claude --------- Co-authored-by: Claude --- litellm/router.py | 18 +++++- .../test_unmapped_model_budget_enforcement.py | 28 +++++++++ .../router_strategy/test_complexity_router.py | 38 ++++++++++++ .../test_router_model_cost_isolation.py | 58 +++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ed3c1ae676e..1f114f99118 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8612,7 +8612,18 @@ class Router: Nothing is recorded for replay: a refresh walks the live routers instead, so a deleted, repointed or never-added deployment, and a discarded router, drop out of the rebuild on their own. + + A strategy-router alias is never the deployment actually called or + billed, so custom pricing configured on it must not become a cost-map + price: an explicit zero would let ``_is_cost_explicitly_configured`` + treat the alias as a genuinely free model and waive budget checks for + requests that route to (and bill as) a real deployment. """ + if classify_strategy_router_model(model) is not None: + model_info = { # mutable-ok: filtered copy of the caller's entry, handed straight to register_model + k: v for k, v in model_info.items() if k not in CustomPricingLiteLLMParams.model_fields + } + if model_id is not None: litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False) @@ -11439,13 +11450,16 @@ class Router: # deployment the hook selected won't have them. Router-only fields # (tpm, rpm, weight, complexity_router_config, ...) are excluded from the # actual outbound LLM call downstream by litellm.types.utils.all_litellm_params, - # not here. + # not here. Custom pricing fields ARE call params, so they must be + # excluded here: they price the alias, not the deployment the hook + # selected, and forwarding them re-registers the routed deployment at + # the alias's price (an explicit 0 makes every alias request bill $0). if pre_routing_hook_response is not None: alias_index: Final = self.model_name_to_deployment_indices.get(model, []) if alias_index: alias_litellm_params: Final = self.model_list[alias_index[0]].get("litellm_params", {}) for key, value in alias_litellm_params.items(): - if key != "model" and value is not None: + if key != "model" and key not in CustomPricingLiteLLMParams.model_fields and value is not None: request_kwargs.setdefault(key, value) return pre_routing_hook_response diff --git a/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py b/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py index d7e32cf1c16..bbe343bcede 100644 --- a/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py +++ b/tests/test_litellm/proxy/auth/test_unmapped_model_budget_enforcement.py @@ -162,6 +162,34 @@ class TestUnmappedModelBudgetEnforcement: # Subsequent call sees the new pricing and enforces budget. assert _is_model_cost_zero(model="ramping-model", llm_router=router) is False + def test_strategy_router_alias_with_zero_pricing_enforces_budget(self): + """An auto-router alias is never the deployment that gets called or + billed, so zero pricing configured on it must not waive budget checks + for requests that route to (and bill as) a real paid deployment.""" + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router/smart-router", + "complexity_router_default_model": "paid-model", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "complexity_router_config": {"tiers": {"simple": "paid-model"}}, + }, + "model_info": {"id": "alias-id"}, + }, + { + "model_name": "paid-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": "paid-id"}, + }, + ] + ) + + assert "input_cost_per_token" not in litellm.model_cost.get("alias-id", {}) + assert _is_model_cost_zero(model="smart-router", llm_router=router) is False + def test_handles_router_without_zero_cost_cache_attribute(self): """Tolerate router-like objects (e.g. ``MagicMock`` stand-ins) that do not expose ``_zero_cost_cache`` — the auth check must still diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 1308c640a30..7fc78aee619 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2040,6 +2040,44 @@ class TestRouterPreRoutingAliasOverrides: assert request_kwargs["drop_params"] is True assert request_kwargs["cache_control_injection_points"] == [{"location": "message", "role": "system"}] + @pytest.mark.asyncio + async def test_alias_custom_pricing_is_not_applied_to_request_kwargs(self): + """Custom pricing on the alias prices the alias, not the tier deployment + the hook picked. Unlike the router-only fields, pricing fields are real + call params, so forwarding them would re-register the routed deployment + at the alias's price - an explicit 0 billing every request as free.""" + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_second": 0.0, + "drop_params": True, + "complexity_router_config": {"tiers": {"SIMPLE": "gpt-4o-mini"}}, + "complexity_router_default_model": "gpt-4o", + }, + }, + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}}, + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}}, + ] + ) + request_kwargs: dict = {} + + result = await router.async_pre_routing_hook( + model="smart-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert result is not None + # Non-pricing alias params still carry over. + assert request_kwargs["drop_params"] is True + for field in ("input_cost_per_token", "output_cost_per_token", "input_cost_per_second"): + assert field not in request_kwargs + @pytest.mark.asyncio async def test_alias_overrides_exclude_only_model(self): """`model` (the alias marker, e.g. auto_router/complexity_router) is diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index ea8a105ef6c..3fb4511e52c 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1471,3 +1471,61 @@ def test_replay_live_router_model_cost_rebuilds_every_live_router(): finally: litellm.model_cost = saved_model_cost _invalidate_model_cost_lowercase_map() + + +def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): + """ + A strategy-router alias is never the deployment actually called or billed, + so custom pricing configured on it must not be registered under its + model_id - an explicit zero there makes the budget check treat the alias + as a genuinely free model while requests bill as a real deployment. The + strip must also survive a price-data reload, which rebuilds entries by + walking the live routers. + """ + from litellm import utils as litellm_utils + monkeypatch.setattr( + litellm_utils, + "_runtime_registered_model_cost", + dict(litellm_utils._runtime_registered_model_cost), + ) + + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router/smart-router", + "complexity_router_default_model": "paid-model", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "complexity_router_config": {"tiers": {"simple": "paid-model"}}, + }, + "model_info": {"id": "strategy-alias-id", "max_input_tokens": 128000}, + }, + { + "model_name": "paid-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": "strategy-alias-paid-id"}, + }, + ], + ) + + def _assert_alias_unpriced(): + entry = litellm.model_cost.get("strategy-alias-id") + assert entry is not None, "Alias metadata should still be registered" + assert entry["max_input_tokens"] == 128000 + assert "input_cost_per_token" not in entry + assert "output_cost_per_token" not in entry + + _assert_alias_unpriced() + + saved_model_cost = litellm.model_cost + try: + _simulate_price_data_reload( + {"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}, + ) + _assert_alias_unpriced() + assert router.model_list + finally: + litellm.model_cost = saved_model_cost + _invalidate_model_cost_lowercase_map() From b6cdb27f44652b4b3f9a1d421bee1d7923022c5b Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 12 Aug 2026 21:32:06 +0000 Subject: [PATCH 17/25] test(bedrock_mantle): pin GPT-5.6 1M context window and long-context rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_bedrock_mantle_responses_transformation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index a47a56376a4..8281f3387d9 100644 --- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -1526,7 +1526,11 @@ class TestBedrockMantleResponsesPricing: assert info["cache_creation_input_token_cost"] == pytest.approx(cache_creation_cost) assert info["cache_read_input_token_cost"] == pytest.approx(cache_read_cost) assert info["output_cost_per_token"] == pytest.approx(output_cost) - assert info["max_input_tokens"] == 272000 + assert info["max_input_tokens"] == 1000000 + assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(input_cost * 2) + assert info["cache_creation_input_token_cost_above_272k_tokens"] == pytest.approx(cache_creation_cost * 2) + assert info["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx(cache_read_cost * 2) + assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(output_cost * 1.5) @pytest.mark.parametrize( "model, input_cost, output_cost", From e52ae039f2e3ffc3874c1fa2052fa4de501c3b85 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:48:02 +0000 Subject: [PATCH 18/25] feat(model_prices): add NVIDIA Nemotron 3.5 Lightning on OpenRouter and DeepInfra (#36696) * feat(model_prices): add NVIDIA Nemotron 3.5 Lightning on OpenRouter and DeepInfra Nemotron 3.5 Lightning shipped 2026-08-11 with public per-token pricing on OpenRouter and DeepInfra at $0.05/M in and $0.20/M out. Without cost map entries both ids raise "This model isn't mapped yet" and log at zero spend. * fix(model_prices): stop asserting an output cap for Nemotron 3.5 Lightning 262144 is the native context window, not the output budget, and neither OpenRouter nor DeepInfra publishes an output cap. Keeps max_input_tokens at the native 256K window: 1M needs VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 on a self-hosted deployment, so it is not what these hosted endpoints serve. * chore(tests): drop the Nemotron 3.5 Lightning metadata test Requested on the review thread: the cost map entries stand on their own. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 22 +++++++++++++++++++ model_prices_and_context_window.json | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b12e4a9fea3..00ba060e1e5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15552,6 +15552,17 @@ "supports_tool_choice": true, "supports_function_calling": true }, + "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": { + "max_input_tokens": 262144, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning", + "supports_tool_choice": true, + "supports_function_calling": true, + "supports_reasoning": true + }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -31754,6 +31765,17 @@ "supports_video_input": true, "supports_vision": true }, + "openrouter/nvidia/nemotron-3.5-lightning": { + "input_cost_per_token": 5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://openrouter.ai/nvidia/nemotron-3.5-lightning", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-3.5-turbo": { "input_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b12e4a9fea3..00ba060e1e5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15552,6 +15552,17 @@ "supports_tool_choice": true, "supports_function_calling": true }, + "deepinfra/nvidia/NVIDIA-Nemotron-3.5-Lightning": { + "max_input_tokens": 262144, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 2e-07, + "litellm_provider": "deepinfra", + "mode": "chat", + "source": "https://deepinfra.com/nvidia/NVIDIA-Nemotron-3.5-Lightning", + "supports_tool_choice": true, + "supports_function_calling": true, + "supports_reasoning": true + }, "deepinfra/nvidia/NVIDIA-Nemotron-Nano-9B-v2": { "max_tokens": 131072, "max_input_tokens": 131072, @@ -31754,6 +31765,17 @@ "supports_video_input": true, "supports_vision": true }, + "openrouter/nvidia/nemotron-3.5-lightning": { + "input_cost_per_token": 5e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-07, + "source": "https://openrouter.ai/nvidia/nemotron-3.5-lightning", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/openai/gpt-3.5-turbo": { "input_cost_per_token": 1.5e-06, "litellm_provider": "openrouter", From 5bc523a09f3357d905707b1e16e3af90151ee1ef Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 12 Aug 2026 15:27:13 -0700 Subject: [PATCH 19/25] feat(terraform/aws): make VPC, Aurora, and Redis optional (#36676) Adds vpc_id/public_subnet_ids/private_subnet_ids to deploy into existing networking, plus create_database/database_url and create_redis/redis_url to use existing data stores or none at all. Defaults keep today's module-owned behavior. A container reads a secret by ARN, which gives Terraform no edge to the _version that writes the value. The managed-Aurora path hid that behind the cluster create; the bring-your-own path has nothing slow in between, so the migration, the bootstrap, and both services now depend on the versions explicitly. Supplied private subnets must cover two AZs whenever Aurora or ElastiCache is module-created, since both subnet groups require it, and a check block warns when a Redis-less stack can run more than one gateway process, because per-key rate limits are counted per process without Redis. --- .github/workflows/test-terraform-modules.yml | 54 ++++ terraform/litellm/aws/README.md | 65 ++++- terraform/litellm/aws/alb.tf | 15 +- terraform/litellm/aws/bootstrap.tf | 68 +++-- terraform/litellm/aws/ecs.tf | 102 +++++-- .../litellm/aws/examples/default/main.tf | 10 + .../litellm/aws/examples/default/outputs.tf | 10 + .../examples/default/terraform.tfvars.example | 32 ++- .../litellm/aws/examples/default/variables.tf | 58 +++- terraform/litellm/aws/iam.tf | 15 +- terraform/litellm/aws/locals.tf | 30 ++ terraform/litellm/aws/migrations.tf | 4 +- terraform/litellm/aws/network.tf | 73 +++-- terraform/litellm/aws/outputs.tf | 42 ++- terraform/litellm/aws/rds.tf | 31 +- terraform/litellm/aws/redis.tf | 20 +- terraform/litellm/aws/secrets.tf | 48 +++- .../aws/tests/byo_infrastructure.tftest.hcl | 272 ++++++++++++++++++ terraform/litellm/aws/variables.tf | 108 ++++++- 19 files changed, 932 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/test-terraform-modules.yml create mode 100644 terraform/litellm/aws/tests/byo_infrastructure.tftest.hcl diff --git a/.github/workflows/test-terraform-modules.yml b/.github/workflows/test-terraform-modules.yml new file mode 100644 index 00000000000..0e3e5330453 --- /dev/null +++ b/.github/workflows/test-terraform-modules.yml @@ -0,0 +1,54 @@ +name: Terraform Modules + +on: + push: + paths: + - "terraform/litellm/aws/**" + - ".github/workflows/test-terraform-modules.yml" + pull_request: + branches: + - main + - litellm_internal_staging + - litellm_oss_staging + - "litellm_**" + paths: + - "terraform/litellm/aws/**" + - ".github/workflows/test-terraform-modules.yml" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + aws-module: + name: fmt, validate, test (aws) + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: terraform/litellm/aws + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: 1.13.3 + terraform_wrapper: false + + - name: fmt + run: terraform fmt -recursive -check -diff + + - name: init + run: terraform init -backend=false -input=false + + - name: validate + run: terraform validate + + # Plan-only, mock_provider-backed: no AWS credentials, no API calls. + - name: test + run: terraform test diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md index 40a9da66c70..389027bf5ca 100644 --- a/terraform/litellm/aws/README.md +++ b/terraform/litellm/aws/README.md @@ -2,9 +2,9 @@ Deploys the componentized LiteLLM proxy on AWS: -- **VPC** with public + private subnets across the AZs you pass in, one NAT gateway -- **Aurora Postgres** cluster — one writer instance + one reader instance, **IAM database authentication enabled** -- **ElastiCache Redis** (private, replication group with multi-AZ failover and at-rest + in-transit encryption) for caching + rate limiting +- **VPC** with public + private subnets across the AZs you pass in, one NAT gateway (skipped when you pass an existing `vpc_id`) +- **Aurora Postgres** cluster — one writer instance + one reader instance, **IAM database authentication enabled** (skipped when `create_database = false`) +- **ElastiCache Redis** (private, replication group with multi-AZ failover and at-rest + in-transit encryption) for caching + rate limiting (skipped when `create_redis = false`) - **S3 bucket** (private, versioned, SSE-S3) — exposed to gateway + backend as `S3_BUCKET_NAME` / `S3_REGION_NAME` for cache backend, request log archival, and `/v1/files` storage - **Secrets Manager** entries for `LITELLM_MASTER_KEY` (auto-generated, `sk-…`) and the Aurora master password (bootstrap-only) - **ECS Fargate cluster** running three services — `gateway`, `backend`, `ui` @@ -14,6 +14,58 @@ Deploys the componentized LiteLLM proxy on AWS: - Everything else (management API: `/key/*`, `/user/*`, …) → `backend` - **One-off migration task** (`litellm-migrations`) that runs `prisma migrate deploy` from the dedicated `ghcr.io/berriai/litellm-migrations` image +## Bring your own networking, database, and Redis + +The three infrastructure pieces the stack would otherwise own are each +optional, so it can slot into an account where networking and data stores are +already provisioned (often by another team, in another Terraform state). + +**Networking.** Set `vpc_id` plus `public_subnet_ids` and `private_subnet_ids` +and no VPC, subnet, route table, internet gateway, or NAT gateway is created. +The ALB goes in the public subnets, the ECS tasks and any subnet group the +stack still needs go in the private ones, and `vpc_cidr` / `azs` go unused. +The private subnets need their own egress (NAT gateway, or VPC endpoints +covering ECR, S3, CloudWatch Logs, and Secrets Manager) since tasks pull +images, resolve secrets, and call LLM providers. + +Security groups stay module-owned in either mode: the ALB group, the tasks +group, and the database/cache groups when it creates those. To let the tasks +reach infrastructure the module doesn't manage, either allow inbound from the +group named by the `task_security_group_id` output, or attach a group of your +own with `additional_task_security_group_ids`. + +```hcl +vpc_id = "vpc-0123456789abcdef0" +public_subnet_ids = ["subnet-aaa", "subnet-bbb"] +private_subnet_ids = ["subnet-ccc", "subnet-ddd"] +``` + +**Database and Redis.** `create_database` and `create_redis` default to `true` +(today's behavior). Set one to `false` and pass a connection string to use +something you already run: the value lands in a Secrets Manager entry and +reaches gateway, backend, and the migration task as `DATABASE_URL` / +`REDIS_URL`, both of which outrank the discrete `DATABASE_*` / `REDIS_*` vars +in the proxy, so nothing appears in plain text in a task definition. + +```hcl +create_database = false +database_url = "postgresql://litellm:...@db.internal:5432/litellm" +create_redis = false +redis_url = "rediss://:...@cache.internal:6379" +``` + +The schema migration still runs on every apply against an existing database; +only the Aurora-specific IAM-user bootstrap drops out, since those credentials +are already in the URL. + +Leaving the URL empty runs without the component entirely: + +- No database: no virtual keys, teams, spend tracking, or UI persistence, and + `STORE_MODEL_IN_DB` is not set, so models come from `proxy_config`. Requests + authenticate with `LITELLM_MASTER_KEY` only. +- No Redis: rate limits, budgets, and router cooldowns are per-task rather + than cluster-wide, which is only sane at one task per service. + ## Aurora + IAM auth The cluster runs with `iam_database_authentication_enabled = true`. Enabling @@ -345,7 +397,7 @@ trial / dev stacks only. ## Storage and database retention -Three opt-in tripwires guard against accidental data loss on +Two opt-in tripwires guard against accidental data loss on `terraform destroy`: - **`skip_final_snapshot`** (Aurora; default `false`) — destroying the @@ -354,6 +406,9 @@ Three opt-in tripwires guard against accidental data loss on `/v1/files` content, and the S3 cache backend; default `false`) — `terraform destroy` against a non-empty bucket fails. +Neither applies to a database you brought yourself: its lifecycle stays with +whoever provisioned it, and `terraform destroy` leaves it alone. + Flip either to `true` only for ephemeral / CI stacks where you accept losing the contents. @@ -365,7 +420,7 @@ losing the contents. | `examples/default/` | Thin root: `aws` provider (with an optional `default_tags` slot for org-wide tags) + a call to the module. The one-command deploy path. | | `variables.tf` | All input variables | | `locals.tf` | Path-prefix lists for ALB routing (mirror of `helm/.../ingress.yaml`) | -| `network.tf` | VPC, subnets, IGW, NAT, route tables, security groups | +| `network.tf` | VPC, subnets, IGW, NAT, route tables (all optional), security groups | | `secrets.tf` | Secrets Manager entries + random passwords | | `rds.tf` | Aurora Postgres cluster + writer / reader instances | | `redis.tf` | ElastiCache Redis | diff --git a/terraform/litellm/aws/alb.tf b/terraform/litellm/aws/alb.tf index 786b9d9a5b9..bb07a83caa7 100644 --- a/terraform/litellm/aws/alb.tf +++ b/terraform/litellm/aws/alb.tf @@ -3,10 +3,17 @@ resource "aws_lb" "this" { load_balancer_type = "application" internal = false security_groups = [aws_security_group.alb.id] - subnets = aws_subnet.public[*].id + subnets = local.public_subnet_ids idle_timeout = 120 + lifecycle { + precondition { + condition = length(local.public_subnet_ids) >= 2 + error_message = "The ALB needs at least 2 public subnets in different AZs. Set `public_subnet_ids` when using `vpc_id`, or list at least 2 `azs` when the module creates the VPC." + } + } + tags = local.tags } @@ -25,7 +32,7 @@ resource "aws_lb_target_group" "gateway" { port = 4000 protocol = "HTTP" target_type = "ip" - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id health_check { path = "/health/readiness" @@ -46,7 +53,7 @@ resource "aws_lb_target_group" "backend" { port = 4001 protocol = "HTTP" target_type = "ip" - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id health_check { path = "/health/readiness" @@ -67,7 +74,7 @@ resource "aws_lb_target_group" "ui" { port = 3000 protocol = "HTTP" target_type = "ip" - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id health_check { path = "/healthz" diff --git a/terraform/litellm/aws/bootstrap.tf b/terraform/litellm/aws/bootstrap.tf index b0bc38d44fb..bc335f10780 100644 --- a/terraform/litellm/aws/bootstrap.tf +++ b/terraform/litellm/aws/bootstrap.tf @@ -1,9 +1,12 @@ # Auto-runs the two manual steps that used to follow `terraform apply`: # # 1. Create the IAM-authed Postgres user (litellm_app) — uses the postgres:16 -# image with the master password from Secrets Manager. +# image with the master password from Secrets Manager. Only relevant to +# the Aurora cluster this module creates, so it is skipped when +# create_database = false. # 2. Run prisma migrate deploy — reuses the existing aws_ecs_task_definition -# .migrations task def from migrations.tf. +# .migrations task def from migrations.tf. Runs against an existing +# database too, and only disappears when there is no database at all. # # Both are invoked via `terraform_data` provisioners. Gateway/backend services # in ecs.tf depend on `terraform_data.migration`, so on a fresh apply they @@ -23,13 +26,14 @@ # extras — see iam.tf). The DB master password lives in a separate secret used # only here, so we grant access in an additive policy. resource "aws_iam_policy" "bootstrap_secrets" { - name = "${local.name}-bootstrap-secrets-access" + count = var.create_database ? 1 : 0 + name = "${local.name}-bootstrap-secrets-access" policy = jsonencode({ Version = "2012-10-17" Statement = [{ Effect = "Allow" Action = ["secretsmanager:GetSecretValue"] - Resource = [aws_secretsmanager_secret.db_master_password.arn] + Resource = [aws_secretsmanager_secret.db_master_password[0].arn] }] }) @@ -37,12 +41,14 @@ resource "aws_iam_policy" "bootstrap_secrets" { } resource "aws_iam_role_policy_attachment" "task_execution_bootstrap_secrets" { + count = var.create_database ? 1 : 0 role = aws_iam_role.task_execution.name - policy_arn = aws_iam_policy.bootstrap_secrets.arn + policy_arn = aws_iam_policy.bootstrap_secrets[0].arn } # ---------- Bootstrap task def ---------- resource "aws_cloudwatch_log_group" "bootstrap_db" { + count = var.create_database ? 1 : 0 name = "/ecs/${local.name}/bootstrap-db" retention_in_days = var.log_retention_days @@ -68,6 +74,7 @@ locals { } resource "aws_ecs_task_definition" "bootstrap_db" { + count = var.create_database ? 1 : 0 family = "${local.name}-bootstrap-db" network_mode = "awsvpc" requires_compatibilities = ["FARGATE"] @@ -82,15 +89,15 @@ resource "aws_ecs_task_definition" "bootstrap_db" { essential = true environment = [ - { name = "PGHOST", value = aws_rds_cluster.this.endpoint }, - { name = "PGPORT", value = tostring(aws_rds_cluster.this.port) }, + { name = "PGHOST", value = aws_rds_cluster.this[0].endpoint }, + { name = "PGPORT", value = tostring(aws_rds_cluster.this[0].port) }, { name = "PGUSER", value = var.db_master_username }, { name = "PGDATABASE", value = var.db_name }, { name = "BOOTSTRAP_SQL", value = local.bootstrap_sql }, ] secrets = [ # `:password::` extracts the password field out of the JSON secret. - { name = "PGPASSWORD", valueFrom = "${aws_secretsmanager_secret.db_master_password.arn}:password::" }, + { name = "PGPASSWORD", valueFrom = "${aws_secretsmanager_secret.db_master_password[0].arn}:password::" }, ] entryPoint = ["sh", "-c"] @@ -99,7 +106,7 @@ resource "aws_ecs_task_definition" "bootstrap_db" { logConfiguration = { logDriver = "awslogs" options = { - awslogs-group = aws_cloudwatch_log_group.bootstrap_db.name + awslogs-group = aws_cloudwatch_log_group.bootstrap_db[0].name awslogs-region = var.region awslogs-stream-prefix = "bootstrap" } @@ -111,20 +118,22 @@ resource "aws_ecs_task_definition" "bootstrap_db" { # ---------- Bootstrap trigger ---------- resource "terraform_data" "bootstrap_db" { + count = var.create_database ? 1 : 0 + triggers_replace = { - cluster_resource_id = aws_rds_cluster.this.cluster_resource_id - task_def_revision = aws_ecs_task_definition.bootstrap_db.revision + cluster_resource_id = aws_rds_cluster.this[0].cluster_resource_id + task_def_revision = aws_ecs_task_definition.bootstrap_db[0].revision } provisioner "local-exec" { interpreter = ["bash", "-c"] environment = { CLUSTER = aws_ecs_cluster.this.name - TASK_DEF = aws_ecs_task_definition.bootstrap_db.arn - SUBNETS = join(",", aws_subnet.private[*].id) - SG = aws_security_group.tasks.id + TASK_DEF = aws_ecs_task_definition.bootstrap_db[0].arn + SUBNETS = join(",", local.private_subnet_ids) + SG = join(",", local.task_security_group_ids) REGION = var.region - LOG_GRP = aws_cloudwatch_log_group.bootstrap_db.name + LOG_GRP = aws_cloudwatch_log_group.bootstrap_db[0].name } command = <<-EOT set -euo pipefail @@ -144,9 +153,13 @@ resource "terraform_data" "bootstrap_db" { EOT } + # Same secret-by-ARN gap as the migration below. The margin here is wide, + # since the writer instance takes minutes while the version write does not, + # but both hang off the cluster in parallel and nothing orders them. depends_on = [ aws_rds_cluster_instance.writer, aws_iam_role_policy_attachment.task_execution_bootstrap_secrets, + aws_secretsmanager_secret_version.db_master_password, ] } @@ -154,20 +167,22 @@ resource "terraform_data" "bootstrap_db" { # Reuses the task definition from migrations.tf — this resource just invokes # it and waits. resource "terraform_data" "migration" { + count = local.database_enabled ? 1 : 0 + triggers_replace = { - task_def_revision = aws_ecs_task_definition.migrations.revision - bootstrap_id = terraform_data.bootstrap_db.id + task_def_revision = aws_ecs_task_definition.migrations[0].revision + bootstrap_id = join(",", terraform_data.bootstrap_db[*].id) } provisioner "local-exec" { interpreter = ["bash", "-c"] environment = { CLUSTER = aws_ecs_cluster.this.name - TASK_DEF = aws_ecs_task_definition.migrations.arn - SUBNETS = join(",", aws_subnet.private[*].id) - SG = aws_security_group.tasks.id + TASK_DEF = aws_ecs_task_definition.migrations[0].arn + SUBNETS = join(",", local.private_subnet_ids) + SG = join(",", local.task_security_group_ids) REGION = var.region - LOG_GRP = aws_cloudwatch_log_group.migrations.name + LOG_GRP = aws_cloudwatch_log_group.migrations[0].name } command = <<-EOT set -euo pipefail @@ -187,5 +202,14 @@ resource "terraform_data" "migration" { EOT } - depends_on = [terraform_data.bootstrap_db] + # A container reads a secret by ARN, so Terraform sees no edge from the + # ARN to the _version that gives it a value. The managed-Aurora path hides + # that: the cluster create takes long enough that the version always lands + # first. A bring-your-own database has nothing slow in between, so without + # this the run-task below can fire against a valueless secret and fail the + # apply with ResourceInitializationError. + depends_on = [ + terraform_data.bootstrap_db, + aws_secretsmanager_secret_version.database_url, + ] } diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf index 10a1bebc8c9..01b730dac65 100644 --- a/terraform/litellm/aws/ecs.tf +++ b/terraform/litellm/aws/ecs.tf @@ -31,6 +31,7 @@ resource "aws_cloudwatch_log_group" "ui" { } resource "aws_cloudwatch_log_group" "migrations" { + count = local.database_enabled ? 1 : 0 name = "/ecs/${local.name}/migrations" retention_in_days = var.log_retention_days @@ -38,11 +39,13 @@ resource "aws_cloudwatch_log_group" "migrations" { } # Shared env block fed to gateway, backend, and the migration task. Mirrors -# the helm chart's `litellm.serverEnv` helper on the IAM-auth branch: -# DATABASE_URL is assembled at runtime by +# the helm chart's `litellm.serverEnv` helper on the IAM-auth branch: for the +# module-created Aurora, DATABASE_URL is assembled at runtime by # litellm/proxy/auth/rds_iam_token.py::init_iam_db_url_from_env from # HOST/PORT/USER/NAME plus an IAM-signed token, so no DB password is needed -# in the task definition. +# in the task definition. An existing database instead arrives as a +# DATABASE_URL secret (var.database_url), which run.py and the proxy both +# take as-is. locals { # OTel v2 is opt-in and gated on otel_endpoint, matching the GCP stack. # When set, LITELLM_OTEL_V2 flips on alongside the OTEL_* block, with @@ -103,29 +106,50 @@ locals { ] : [], ) - shared_env = [ + managed_db_env = var.create_database ? [ { name = "IAM_TOKEN_DB_AUTH", value = "true" }, - { name = "DATABASE_HOST", value = aws_rds_cluster.this.endpoint }, - { name = "DATABASE_PORT", value = tostring(aws_rds_cluster.this.port) }, + { name = "DATABASE_HOST", value = aws_rds_cluster.this[0].endpoint }, + { name = "DATABASE_PORT", value = tostring(aws_rds_cluster.this[0].port) }, { name = "DATABASE_USER", value = var.db_username }, { name = "DATABASE_NAME", value = var.db_name }, - { name = "DATABASE_HOST_READ_REPLICA", value = aws_rds_cluster.this.reader_endpoint }, - { name = "DATABASE_PORT_READ_REPLICA", value = tostring(aws_rds_cluster.this.port) }, - { name = "REDIS_HOST", value = aws_elasticache_replication_group.this.primary_endpoint_address }, - { name = "REDIS_PORT", value = tostring(aws_elasticache_replication_group.this.port) }, + { name = "DATABASE_HOST_READ_REPLICA", value = aws_rds_cluster.this[0].reader_endpoint }, + { name = "DATABASE_PORT_READ_REPLICA", value = tostring(aws_rds_cluster.this[0].port) }, + ] : [] + + managed_redis_env = var.create_redis ? [ + { name = "REDIS_HOST", value = aws_elasticache_replication_group.this[0].primary_endpoint_address }, + { name = "REDIS_PORT", value = tostring(aws_elasticache_replication_group.this[0].port) }, # transit_encryption_enabled = true on the replication group means the # proxy must connect via rediss://. _redis.get_redis_url_from_environment # honors REDIS_SSL to flip the scheme. { name = "REDIS_SSL", value = "true" }, - # S3 bucket — referenced from proxy_config via os.environ/S3_BUCKET_NAME - # (e.g. cache backend, request log archival, /files passthrough). - { name = "S3_BUCKET_NAME", value = aws_s3_bucket.this.bucket }, - { name = "S3_REGION_NAME", value = var.region }, - # boto3 inside generate_iam_auth_token reads AWS_REGION_NAME first, then - # AWS_REGION. Set both for compatibility. - { name = "AWS_REGION", value = var.region }, - { name = "AWS_REGION_NAME", value = var.region }, - ] + ] : [] + + shared_env = concat( + local.managed_db_env, + local.managed_redis_env, + [ + # S3 bucket — referenced from proxy_config via os.environ/S3_BUCKET_NAME + # (e.g. cache backend, request log archival, /files passthrough). + { name = "S3_BUCKET_NAME", value = aws_s3_bucket.this.bucket }, + { name = "S3_REGION_NAME", value = var.region }, + # boto3 inside generate_iam_auth_token reads AWS_REGION_NAME first, then + # AWS_REGION. Set both for compatibility. + { name = "AWS_REGION", value = var.region }, + { name = "AWS_REGION_NAME", value = var.region }, + ], + ) + + # DATABASE_URL / REDIS_URL both outrank the discrete host/port vars in the + # proxy, so the BYO branch needs nothing removed from shared_env: the + # managed_*_env blocks are already empty whenever these are set. + byo_database_secrets = local.byo_database ? [ + { name = "DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url[0].arn }, + ] : [] + + byo_redis_secrets = local.byo_redis ? [ + { name = "REDIS_URL", valueFrom = aws_secretsmanager_secret.redis_url[0].arn }, + ] : [] shared_secrets = concat( [ @@ -134,6 +158,8 @@ locals { var.litellm_license == "" ? [] : [ { name = "LITELLM_LICENSE", valueFrom = aws_secretsmanager_secret.license[0].arn }, ], + local.byo_database_secrets, + local.byo_redis_secrets, local.otel_secrets, local.billing_metrics_secrets, ) @@ -151,9 +177,11 @@ locals { for k, v in var.backend_extra_env : { name = k, value = v } ] - backend_default_env = [ + # Storing models in the DB needs a DB. Without one the backend reads its + # model list from proxy_config only. + backend_default_env = local.database_enabled ? [ { name = "STORE_MODEL_IN_DB", value = "true" }, - ] + ] : [] gateway_extra_secrets_list = [ for k, v in var.gateway_extra_secrets : { name = k, valueFrom = v } ] @@ -286,8 +314,8 @@ resource "aws_ecs_service" "gateway" { launch_type = "FARGATE" network_configuration { - subnets = aws_subnet.private[*].id - security_groups = [aws_security_group.tasks.id] + subnets = local.private_subnet_ids + security_groups = local.task_security_group_ids assign_public_ip = false } @@ -308,10 +336,20 @@ resource "aws_ecs_service" "gateway" { # Don't start until the schema migration has run. Otherwise the proxy # boots, Prisma fails on the missing tables, and ECS thrashes the task. + # The _version entries are listed because a task reads its secrets by ARN, + # which gives Terraform no edge to the resource that writes the value; the + # migration covers that ordering only while a database exists. depends_on = [ aws_lb_listener.http, aws_lb_listener.https, terraform_data.migration, + aws_secretsmanager_secret_version.master_key, + aws_secretsmanager_secret_version.license, + aws_secretsmanager_secret_version.database_url, + aws_secretsmanager_secret_version.redis_url, + aws_secretsmanager_secret_version.billing_metrics_client_cert, + aws_secretsmanager_secret_version.billing_metrics_client_key, + aws_secretsmanager_secret_version.billing_metrics_ca_cert, ] tags = local.tags @@ -381,8 +419,8 @@ resource "aws_ecs_service" "backend" { launch_type = "FARGATE" network_configuration { - subnets = aws_subnet.private[*].id - security_groups = [aws_security_group.tasks.id] + subnets = local.private_subnet_ids + security_groups = local.task_security_group_ids assign_public_ip = false } @@ -399,10 +437,20 @@ resource "aws_ecs_service" "backend" { ignore_changes = [desired_count] } + # Same secret-version ordering as the gateway, plus UI_PASSWORD, which only + # the backend consumes. depends_on = [ aws_lb_listener.http, aws_lb_listener.https, terraform_data.migration, + aws_secretsmanager_secret_version.master_key, + aws_secretsmanager_secret_version.license, + aws_secretsmanager_secret_version.ui_password, + aws_secretsmanager_secret_version.database_url, + aws_secretsmanager_secret_version.redis_url, + aws_secretsmanager_secret_version.billing_metrics_client_cert, + aws_secretsmanager_secret_version.billing_metrics_client_key, + aws_secretsmanager_secret_version.billing_metrics_ca_cert, ] tags = local.tags @@ -451,8 +499,8 @@ resource "aws_ecs_service" "ui" { launch_type = "FARGATE" network_configuration { - subnets = aws_subnet.private[*].id - security_groups = [aws_security_group.tasks.id] + subnets = local.private_subnet_ids + security_groups = local.task_security_group_ids assign_public_ip = false } diff --git a/terraform/litellm/aws/examples/default/main.tf b/terraform/litellm/aws/examples/default/main.tf index 3d421099aed..2eeaf6adb50 100644 --- a/terraform/litellm/aws/examples/default/main.tf +++ b/terraform/litellm/aws/examples/default/main.tf @@ -24,6 +24,16 @@ module "litellm" { env = var.env azs = var.azs + vpc_id = var.vpc_id + public_subnet_ids = var.public_subnet_ids + private_subnet_ids = var.private_subnet_ids + additional_task_security_group_ids = var.additional_task_security_group_ids + + create_database = var.create_database + database_url = var.database_url + create_redis = var.create_redis + redis_url = var.redis_url + litellm_master_key = var.litellm_master_key litellm_license = var.litellm_license ui_password = var.ui_password diff --git a/terraform/litellm/aws/examples/default/outputs.tf b/terraform/litellm/aws/examples/default/outputs.tf index 235c069933c..9fe2090c407 100644 --- a/terraform/litellm/aws/examples/default/outputs.tf +++ b/terraform/litellm/aws/examples/default/outputs.tf @@ -13,6 +13,16 @@ output "ecs_cluster" { value = module.litellm.ecs_cluster } +output "vpc_id" { + description = "VPC the stack runs in, whether module-created or supplied." + value = module.litellm.vpc_id +} + +output "task_security_group_id" { + description = "Tasks security group. Allow this inbound on an existing database or Redis." + value = module.litellm.task_security_group_id +} + output "aurora_writer_endpoint" { description = "Aurora writer endpoint." value = module.litellm.aurora_writer_endpoint diff --git a/terraform/litellm/aws/examples/default/terraform.tfvars.example b/terraform/litellm/aws/examples/default/terraform.tfvars.example index 061ca2a9b82..59301ea6aa5 100644 --- a/terraform/litellm/aws/examples/default/terraform.tfvars.example +++ b/terraform/litellm/aws/examples/default/terraform.tfvars.example @@ -1,5 +1,35 @@ region = "us-west-2" -azs = ["us-west-2a", "us-west-2b"] + +# Networking: by default the module creates a VPC, public/private subnets in +# each AZ listed here, an internet gateway, a NAT gateway, and route tables. +azs = ["us-west-2a", "us-west-2b"] + +# To deploy into networking you already own, drop `azs` and set these +# instead. Nothing network-related is created then, so the private subnets +# need their own egress for LLM providers, image pulls, and Secrets Manager. +# vpc_id = "vpc-0123456789abcdef0" +# public_subnet_ids = ["subnet-aaa", "subnet-bbb"] +# private_subnet_ids = ["subnet-ccc", "subnet-ddd"] +# +# The tasks get their own security group either way. To reach a store that +# only allows a group you already have, attach it here as well; the +# `task_security_group_id` output names the module's own group. +# additional_task_security_group_ids = ["sg-0123456789abcdef0"] + +# Data stores: Aurora Postgres and ElastiCache Redis are created by default. +# Set create_* = false to point at your own, passing a connection string +# (stored in Secrets Manager, injected as DATABASE_URL / REDIS_URL). Make +# sure they allow inbound from the stack's tasks security group, which the +# `task_security_group_id` output names. +# create_database = false +# database_url = "postgresql://litellm:...@db.internal:5432/litellm" +# create_redis = false +# redis_url = "rediss://:...@cache.internal:6379" +# +# Leaving the URL empty runs without that component: no database means no +# virtual keys, spend tracking, or UI persistence (master-key auth only), and +# no Redis means rate limits, budgets, and router cooldowns go per-task +# instead of cluster-wide. # Resource naming: every AWS resource the stack creates is named # `${tenant}-litellm-${env}` (or that plus a per-resource suffix). E.g. diff --git a/terraform/litellm/aws/examples/default/variables.tf b/terraform/litellm/aws/examples/default/variables.tf index 74522118a93..d8ab56b13af 100644 --- a/terraform/litellm/aws/examples/default/variables.tf +++ b/terraform/litellm/aws/examples/default/variables.tf @@ -21,8 +21,64 @@ variable "env" { } variable "azs" { - description = "Availability zones for subnets. At least 2 (RDS + ALB)." + description = "Availability zones for the subnets the module creates. At least 2 (RDS + ALB). Unused when vpc_id is set." type = list(string) + default = [] +} + +# Bring-your-own networking. Leave vpc_id empty to have the module create the +# VPC, subnets, NAT gateway, and route tables. +variable "vpc_id" { + description = "Existing VPC to deploy into. Empty → module creates its own networking." + type = string + default = "" +} + +variable "public_subnet_ids" { + description = "Existing public subnets for the ALB (≥ 2 AZs). Required with vpc_id." + type = list(string) + default = [] +} + +variable "private_subnet_ids" { + description = "Existing private subnets for tasks, Aurora, and Redis. Required with vpc_id." + type = list(string) + default = [] +} + +variable "additional_task_security_group_ids" { + description = "Extra security groups for the tasks, e.g. one an existing database already allows." + type = list(string) + default = [] +} + +# Bring-your-own data stores. create_* false with an empty URL runs without +# that component: no DB means no key management or spend tracking, no Redis +# means per-task rate limits instead of cluster-wide. +variable "create_database" { + description = "Create the Aurora Postgres cluster. False → use database_url, or run DB-less." + type = bool + default = true +} + +variable "database_url" { + description = "Postgres connection string for an existing database. Read only when create_database = false." + type = string + default = "" + sensitive = true +} + +variable "create_redis" { + description = "Create the ElastiCache Redis group. False → use redis_url, or run without Redis." + type = bool + default = true +} + +variable "redis_url" { + description = "Connection string for an existing Redis. Read only when create_redis = false." + type = string + default = "" + sensitive = true } # Sensitive — prefer TF_VAR_litellm_master_key / TF_VAR_litellm_license / diff --git a/terraform/litellm/aws/iam.tf b/terraform/litellm/aws/iam.tf index 63c6c26f184..3c55f07b02a 100644 --- a/terraform/litellm/aws/iam.tf +++ b/terraform/litellm/aws/iam.tf @@ -56,6 +56,8 @@ data "aws_iam_policy_document" "secrets_access" { aws_secretsmanager_secret.billing_metrics_client_cert[*].arn, aws_secretsmanager_secret.billing_metrics_client_key[*].arn, aws_secretsmanager_secret.billing_metrics_ca_cert[*].arn, + aws_secretsmanager_secret.database_url[*].arn, + aws_secretsmanager_secret.redis_url[*].arn, local.extra_secret_arns, var.otel_headers_secret_arn == "" ? [] : [var.otel_headers_secret_arn], ) @@ -79,6 +81,9 @@ resource "aws_iam_role_policy_attachment" "task_execution_secrets" { # Assumed by the running container. Gets `rds-db:connect` so the proxy can # mint IAM-signed Postgres tokens for the app user. Layer additional # policies here (e.g. Bedrock invoke, S3 read) when the proxy needs them. +# IAM auth only applies to the Aurora cluster this module creates: an +# existing database is reached with the credentials embedded in +# var.database_url, so the policy is skipped there. resource "aws_iam_role" "task" { name = "${local.name}-task" @@ -90,24 +95,28 @@ resource "aws_iam_role" "task" { data "aws_caller_identity" "current" {} data "aws_iam_policy_document" "rds_iam_connect" { + count = var.create_database ? 1 : 0 + statement { actions = ["rds-db:connect"] resources = [ - "arn:aws:rds-db:${var.region}:${data.aws_caller_identity.current.account_id}:dbuser:${aws_rds_cluster.this.cluster_resource_id}/${var.db_username}", + "arn:aws:rds-db:${var.region}:${data.aws_caller_identity.current.account_id}:dbuser:${aws_rds_cluster.this[0].cluster_resource_id}/${var.db_username}", ] } } resource "aws_iam_policy" "rds_iam_connect" { + count = var.create_database ? 1 : 0 name = "${local.name}-rds-iam-connect" - policy = data.aws_iam_policy_document.rds_iam_connect.json + policy = data.aws_iam_policy_document.rds_iam_connect[0].json tags = local.tags } resource "aws_iam_role_policy_attachment" "task_rds_iam_connect" { + count = var.create_database ? 1 : 0 role = aws_iam_role.task.name - policy_arn = aws_iam_policy.rds_iam_connect.arn + policy_arn = aws_iam_policy.rds_iam_connect[0].arn } # ---------- UI task role ---------- diff --git a/terraform/litellm/aws/locals.tf b/terraform/litellm/aws/locals.tf index b5e28272d04..33f63fc4205 100644 --- a/terraform/litellm/aws/locals.tf +++ b/terraform/litellm/aws/locals.tf @@ -25,6 +25,36 @@ locals { var.tags, ) + # Networking, database, and cache are each either module-owned or + # bring-your-own. Everything downstream reads these locals rather than the + # resources, so a resource going to zero instances doesn't ripple. + create_vpc = var.vpc_id == "" + vpc_id = local.create_vpc ? aws_vpc.this[0].id : var.vpc_id + public_subnet_ids = local.create_vpc ? aws_subnet.public[*].id : var.public_subnet_ids + private_subnet_ids = local.create_vpc ? aws_subnet.private[*].id : var.private_subnet_ids + + task_security_group_ids = concat([aws_security_group.tasks.id], var.additional_task_security_group_ids) + + # `byo_*` is the existing-store branch, `database_enabled` is either branch. + # Neither branch means the component is absent: no DB (no key management, + # spend tracking, or UI persistence) or no Redis (per-task rate limits and + # cooldowns instead of cluster-wide). + # nonsensitive() on the emptiness check only: without it the sensitivity of + # the URLs propagates into every value derived from these flags, redacting + # unrelated task-definition and output diffs in the plan. + byo_database = !var.create_database && nonsensitive(var.database_url != "") + byo_redis = !var.create_redis && nonsensitive(var.redis_url != "") + database_enabled = var.create_database || local.byo_database + redis_enabled = var.create_redis || local.byo_redis + + # Aurora and ElastiCache subnet groups both demand two AZs, so supplied + # private subnets have to cover two whenever either store is module-created. + managed_stores_need_two_azs = var.create_database || var.create_redis + + # Every uvicorn worker in every gateway task counts its own rate limits when + # there is no Redis to share them through, so the ceiling is tasks x workers. + max_gateway_processes = (var.gateway_autoscaling_enabled ? var.gateway_max_capacity : var.gateway_desired_count) * var.gateway_num_workers + gateway_path_prefixes = [ "/v1/chat/*", "/chat/*", "/v1/completions*", "/completions*", diff --git a/terraform/litellm/aws/migrations.tf b/terraform/litellm/aws/migrations.tf index 62880ebf165..e924b29eba0 100644 --- a/terraform/litellm/aws/migrations.tf +++ b/terraform/litellm/aws/migrations.tf @@ -13,6 +13,7 @@ # every apply (after the IAM-authed user has been created). The # `migration_run_command` output is preserved for break-glass manual re-runs. resource "aws_ecs_task_definition" "migrations" { + count = local.database_enabled ? 1 : 0 family = "${local.name}-migrations" network_mode = "awsvpc" requires_compatibilities = ["FARGATE"] @@ -32,11 +33,12 @@ resource "aws_ecs_task_definition" "migrations" { # No entryPoint/command override — the image's ENTRYPOINT runs run.py. environment = local.shared_env + secrets = local.byo_database_secrets logConfiguration = { logDriver = "awslogs" options = { - awslogs-group = aws_cloudwatch_log_group.migrations.name + awslogs-group = aws_cloudwatch_log_group.migrations[0].name awslogs-region = var.region awslogs-stream-prefix = "migrations" } diff --git a/terraform/litellm/aws/network.tf b/terraform/litellm/aws/network.tf index 2f104da6a6b..4563eefbba5 100644 --- a/terraform/litellm/aws/network.tf +++ b/terraform/litellm/aws/network.tf @@ -1,24 +1,34 @@ -data "aws_availability_zones" "available" { - state = "available" -} +# Networking is created only when the caller didn't supply a VPC. With +# var.vpc_id set, every resource in this file except the security groups has +# zero instances and the stack consumes the caller's subnets through +# local.public_subnet_ids / local.private_subnet_ids (see locals.tf). resource "aws_vpc" "this" { + count = local.create_vpc ? 1 : 0 cidr_block = var.vpc_cidr enable_dns_hostnames = true enable_dns_support = true + lifecycle { + precondition { + condition = length(var.azs) >= 2 + error_message = "Provide at least 2 availability zones in `azs`, or set `vpc_id` + `public_subnet_ids` + `private_subnet_ids` to deploy into an existing VPC." + } + } + tags = merge(local.tags, { Name = local.name }) } resource "aws_internet_gateway" "this" { - vpc_id = aws_vpc.this.id + count = local.create_vpc ? 1 : 0 + vpc_id = aws_vpc.this[0].id tags = merge(local.tags, { Name = local.name }) } # Public subnets (ALB + NAT). One per AZ. resource "aws_subnet" "public" { - count = length(var.azs) - vpc_id = aws_vpc.this.id + count = local.create_vpc ? length(var.azs) : 0 + vpc_id = aws_vpc.this[0].id cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index) availability_zone = var.azs[count.index] map_public_ip_on_launch = true @@ -29,8 +39,8 @@ resource "aws_subnet" "public" { # Private subnets (ECS tasks, RDS, ElastiCache). One per AZ, separate from # public range. resource "aws_subnet" "private" { - count = length(var.azs) - vpc_id = aws_vpc.this.id + count = local.create_vpc ? length(var.azs) : 0 + vpc_id = aws_vpc.this[0].id cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index + 10) availability_zone = var.azs[count.index] @@ -38,6 +48,7 @@ resource "aws_subnet" "private" { } resource "aws_eip" "nat" { + count = local.create_vpc ? 1 : 0 domain = "vpc" tags = merge(local.tags, { Name = "${local.name}-nat" }) @@ -47,7 +58,8 @@ resource "aws_eip" "nat" { # Single NAT gateway in the first public subnet. For HA, replicate per AZ — # adds ~$30/mo per gateway, so off by default for a baseline deployment. resource "aws_nat_gateway" "this" { - allocation_id = aws_eip.nat.id + count = local.create_vpc ? 1 : 0 + allocation_id = aws_eip.nat[0].id subnet_id = aws_subnet.public[0].id tags = merge(local.tags, { Name = local.name }) @@ -56,45 +68,53 @@ resource "aws_nat_gateway" "this" { } resource "aws_route_table" "public" { - vpc_id = aws_vpc.this.id + count = local.create_vpc ? 1 : 0 + vpc_id = aws_vpc.this[0].id route { cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.this.id + gateway_id = aws_internet_gateway.this[0].id } tags = merge(local.tags, { Name = "${local.name}-public" }) } resource "aws_route_table_association" "public" { - count = length(var.azs) + count = local.create_vpc ? length(var.azs) : 0 subnet_id = aws_subnet.public[count.index].id - route_table_id = aws_route_table.public.id + route_table_id = aws_route_table.public[0].id } resource "aws_route_table" "private" { - vpc_id = aws_vpc.this.id + count = local.create_vpc ? 1 : 0 + vpc_id = aws_vpc.this[0].id route { cidr_block = "0.0.0.0/0" - nat_gateway_id = aws_nat_gateway.this.id + nat_gateway_id = aws_nat_gateway.this[0].id } tags = merge(local.tags, { Name = "${local.name}-private" }) } resource "aws_route_table_association" "private" { - count = length(var.azs) + count = local.create_vpc ? length(var.azs) : 0 subnet_id = aws_subnet.private[count.index].id - route_table_id = aws_route_table.private.id + route_table_id = aws_route_table.private[0].id } # ---------- Security groups ---------- +# +# Always module-owned, in local.vpc_id, so the stack keeps a least-privilege +# path between its own components even when it borrows someone else's VPC. +# Existing databases and caches reached over var.database_url / var.redis_url +# need to allow inbound from the tasks group (or from a group passed via +# var.additional_task_security_group_ids). resource "aws_security_group" "alb" { name = "${local.name}-alb" description = "Inbound HTTP/HTTPS to the LiteLLM ALB." - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id ingress { description = "HTTP from anywhere" @@ -126,7 +146,7 @@ resource "aws_security_group" "alb" { resource "aws_security_group" "tasks" { name = "${local.name}-tasks" description = "ECS tasks (gateway/backend/ui)." - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id ingress { description = "ALB to tasks" @@ -144,13 +164,23 @@ resource "aws_security_group" "tasks" { cidr_blocks = ["0.0.0.0/0"] } + # The tasks group is created in every mode, so this is where the + # bring-your-own-VPC inputs get checked. + lifecycle { + precondition { + condition = local.create_vpc || length(var.private_subnet_ids) >= (local.managed_stores_need_two_azs ? 2 : 1) + error_message = "`private_subnet_ids` is required when `vpc_id` is set: the tasks, Aurora, and ElastiCache all live in private subnets. Aurora and ElastiCache subnet groups need subnets in at least 2 AZs, so pass 2 unless both `create_database` and `create_redis` are false." + } + } + tags = local.tags } resource "aws_security_group" "rds" { + count = var.create_database ? 1 : 0 name = "${local.name}-rds" description = "RDS Postgres - tasks only." - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id ingress { description = "Postgres from ECS tasks" @@ -164,9 +194,10 @@ resource "aws_security_group" "rds" { } resource "aws_security_group" "redis" { + count = var.create_redis ? 1 : 0 name = "${local.name}-redis" description = "ElastiCache Redis - tasks only." - vpc_id = aws_vpc.this.id + vpc_id = local.vpc_id ingress { description = "Redis from ECS tasks" diff --git a/terraform/litellm/aws/outputs.tf b/terraform/litellm/aws/outputs.tf index 9c36b1a7e0f..d4509fbb7a1 100644 --- a/terraform/litellm/aws/outputs.tf +++ b/terraform/litellm/aws/outputs.tf @@ -13,19 +13,29 @@ output "ecs_cluster" { value = aws_ecs_cluster.this.name } +output "vpc_id" { + description = "VPC the stack runs in, whether module-created or passed in via `vpc_id`." + value = local.vpc_id +} + +output "task_security_group_id" { + description = "Security group attached to the ECS tasks. Allow inbound from this group on an existing database or Redis reached over `database_url` / `redis_url`." + value = aws_security_group.tasks.id +} + output "aurora_writer_endpoint" { - description = "Aurora writer endpoint (cluster endpoint). Used by gateway/backend as DATABASE_HOST." - value = aws_rds_cluster.this.endpoint + description = "Aurora writer endpoint (cluster endpoint). Used by gateway/backend as DATABASE_HOST. Null when `create_database = false`." + value = one(aws_rds_cluster.this[*].endpoint) } output "aurora_reader_endpoint" { - description = "Aurora reader endpoint. Used by gateway/backend as DATABASE_HOST_READ_REPLICA." - value = aws_rds_cluster.this.reader_endpoint + description = "Aurora reader endpoint. Used by gateway/backend as DATABASE_HOST_READ_REPLICA. Null when `create_database = false`." + value = one(aws_rds_cluster.this[*].reader_endpoint) } output "redis_endpoint" { - description = "ElastiCache Redis primary endpoint (TLS, transit_encryption_enabled = true)." - value = "${aws_elasticache_replication_group.this.primary_endpoint_address}:${aws_elasticache_replication_group.this.port}" + description = "ElastiCache Redis primary endpoint (TLS, transit_encryption_enabled = true). Null when `create_redis = false`." + value = one([for r in aws_elasticache_replication_group.this : "${r.primary_endpoint_address}:${r.port}"]) } output "s3_bucket" { @@ -39,15 +49,17 @@ output "master_key_secret_arn" { } output "db_master_password_secret_arn" { - description = "Secrets Manager ARN holding the Aurora master credentials (bootstrap-only). Used to create the IAM-authed application user." - value = aws_secretsmanager_secret.db_master_password.arn + description = "Secrets Manager ARN holding the Aurora master credentials (bootstrap-only). Used to create the IAM-authed application user. Null when `create_database = false`." + value = one(aws_secretsmanager_secret.db_master_password[*].arn) } # Pre-baked SQL to run once as the master user, creating the IAM-authed # application user that gateway/backend/migration tasks will authenticate as. +# Irrelevant to an existing database reached over `database_url`, whose +# credentials are already in the URL. output "db_bootstrap_sql" { - description = "Run this once as the master DB user (after the first apply) to create the IAM-authed app user." - value = <<-SQL + description = "Run this once as the master DB user (after the first apply) to create the IAM-authed app user. Empty when `create_database = false`." + value = !var.create_database ? "" : <<-SQL CREATE USER ${var.db_username}; GRANT rds_iam TO ${var.db_username}; GRANT ALL PRIVILEGES ON DATABASE ${var.db_name} TO ${var.db_username}; @@ -60,13 +72,13 @@ output "db_bootstrap_sql" { # Pre-baked command for running the one-off migration task. ECS run-task # needs the subnet + SG IDs at call time, so we render the full command. output "migration_run_command" { - description = "Shell command that runs the one-off prisma migration task against Aurora. Run this once, after the bootstrap SQL above, before sending traffic." - value = format( + description = "Shell command that runs the one-off prisma migration task against the database. Run this once, after the bootstrap SQL above, before sending traffic. Empty when the stack has no database." + value = !local.database_enabled ? "" : format( "aws ecs run-task --cluster %s --launch-type FARGATE --task-definition %s --network-configuration 'awsvpcConfiguration={subnets=[%s],securityGroups=[%s],assignPublicIp=DISABLED}' --region %s", aws_ecs_cluster.this.name, - aws_ecs_task_definition.migrations.arn, - join(",", aws_subnet.private[*].id), - aws_security_group.tasks.id, + aws_ecs_task_definition.migrations[0].arn, + join(",", local.private_subnet_ids), + join(",", local.task_security_group_ids), var.region, ) } diff --git a/terraform/litellm/aws/rds.tf b/terraform/litellm/aws/rds.tf index d9b7351a805..d42be34e808 100644 --- a/terraform/litellm/aws/rds.tf +++ b/terraform/litellm/aws/rds.tf @@ -1,5 +1,7 @@ # Aurora Postgres cluster with one writer + one reader instance, IAM -# database authentication enabled. +# database authentication enabled. Skipped entirely when +# create_database = false, in which case the stack either talks to the +# database named by var.database_url or runs without one. # # Important: enabling IAM auth on the cluster does not by itself grant any # Postgres user the ability to log in with an IAM token. After the first @@ -17,13 +19,15 @@ # superusers — keep it for break-glass only. resource "aws_db_subnet_group" "this" { + count = var.create_database ? 1 : 0 name = "${local.name}-db" - subnet_ids = aws_subnet.private[*].id + subnet_ids = local.private_subnet_ids tags = local.tags } resource "aws_rds_cluster_parameter_group" "this" { + count = var.create_database ? 1 : 0 name = "${local.name}-cluster-pg" family = "aurora-postgresql${split(".", var.db_engine_version)[0]}" description = "LiteLLM Aurora Postgres cluster parameters." @@ -32,16 +36,17 @@ resource "aws_rds_cluster_parameter_group" "this" { } resource "aws_rds_cluster" "this" { + count = var.create_database ? 1 : 0 cluster_identifier = local.name engine = "aurora-postgresql" engine_mode = "provisioned" engine_version = var.db_engine_version database_name = var.db_name master_username = var.db_master_username - master_password = random_password.db_master_password.result - db_subnet_group_name = aws_db_subnet_group.this.name - vpc_security_group_ids = [aws_security_group.rds.id] - db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.this.name + master_password = random_password.db_master_password[0].result + db_subnet_group_name = aws_db_subnet_group.this[0].name + vpc_security_group_ids = [aws_security_group.rds[0].id] + db_cluster_parameter_group_name = aws_rds_cluster_parameter_group.this[0].name iam_database_authentication_enabled = true storage_encrypted = true @@ -61,11 +66,12 @@ resource "aws_rds_cluster" "this" { } resource "aws_rds_cluster_instance" "writer" { + count = var.create_database ? 1 : 0 identifier = "${local.name}-writer" - cluster_identifier = aws_rds_cluster.this.id + cluster_identifier = aws_rds_cluster.this[0].id instance_class = var.db_instance_class - engine = aws_rds_cluster.this.engine - engine_version = aws_rds_cluster.this.engine_version + engine = aws_rds_cluster.this[0].engine + engine_version = aws_rds_cluster.this[0].engine_version publicly_accessible = false performance_insights_enabled = true @@ -78,11 +84,12 @@ resource "aws_rds_cluster_instance" "writer" { } resource "aws_rds_cluster_instance" "reader" { + count = var.create_database ? 1 : 0 identifier = "${local.name}-reader" - cluster_identifier = aws_rds_cluster.this.id + cluster_identifier = aws_rds_cluster.this[0].id instance_class = var.db_instance_class - engine = aws_rds_cluster.this.engine - engine_version = aws_rds_cluster.this.engine_version + engine = aws_rds_cluster.this[0].engine + engine_version = aws_rds_cluster.this[0].engine_version publicly_accessible = false performance_insights_enabled = true diff --git a/terraform/litellm/aws/redis.tf b/terraform/litellm/aws/redis.tf index 071cbc6d46f..ca43d85e306 100644 --- a/terraform/litellm/aws/redis.tf +++ b/terraform/litellm/aws/redis.tf @@ -1,6 +1,7 @@ resource "aws_elasticache_subnet_group" "this" { + count = var.create_redis ? 1 : 0 name = "${local.name}-redis" - subnet_ids = aws_subnet.private[*].id + subnet_ids = local.private_subnet_ids tags = local.tags } @@ -13,6 +14,7 @@ resource "aws_elasticache_subnet_group" "this" { # TLS-protected — the proxy connects via the rediss:// scheme thanks to # REDIS_SSL=true in the shared task env (see ecs.tf). resource "aws_elasticache_replication_group" "this" { + count = var.create_redis ? 1 : 0 replication_group_id = "${local.name}-redis" description = "LiteLLM ElastiCache Redis" @@ -23,8 +25,8 @@ resource "aws_elasticache_replication_group" "this" { parameter_group_name = "default.redis7" port = 6379 - subnet_group_name = aws_elasticache_subnet_group.this.name - security_group_ids = [aws_security_group.redis.id] + subnet_group_name = aws_elasticache_subnet_group.this[0].name + security_group_ids = [aws_security_group.redis[0].id] automatic_failover_enabled = var.redis_num_replicas >= 1 multi_az_enabled = var.redis_num_replicas >= 1 @@ -35,3 +37,15 @@ resource "aws_elasticache_replication_group" "this" { tags = local.tags } + +# Rate limits, budgets, and router cooldowns are shared through Redis. Without +# it each gateway process counts on its own, so a caller spread across tasks +# collects the full per-key allowance from every one of them. A `check` rather +# than a precondition: running without Redis is a legitimate choice when you do +# not rely on per-key limits, so this warns instead of blocking the plan. +check "redis_less_rate_limits_are_per_process" { + assert { + condition = local.redis_enabled || local.max_gateway_processes <= 1 + error_message = "No Redis is configured while the gateway can run up to ${local.max_gateway_processes} processes, so per-key RPM/TPM limits, budgets, and cooldowns apply per process and a caller can multiply them across tasks. Set `create_redis = true`, pass `redis_url`, or hold the gateway to one process (`gateway_autoscaling_enabled = false`, `gateway_desired_count = 1`, `gateway_num_workers = 1`)." + } +} diff --git a/terraform/litellm/aws/secrets.tf b/terraform/litellm/aws/secrets.tf index 85d3eb4502c..921bae4d827 100644 --- a/terraform/litellm/aws/secrets.tf +++ b/terraform/litellm/aws/secrets.tf @@ -10,6 +10,7 @@ resource "random_password" "master_key" { # user (see rds.tf header). Runtime services authenticate via IAM tokens # and never read this secret. resource "random_password" "db_master_password" { + count = var.create_database ? 1 : 0 length = 32 special = false min_lower = 4 @@ -130,6 +131,7 @@ resource "aws_secretsmanager_secret_version" "billing_metrics_ca_cert" { } resource "aws_secretsmanager_secret" "db_master_password" { + count = var.create_database ? 1 : 0 name = "${local.name}-db-master-password" description = "Aurora master-user password - bootstrap only. Runtime auth is IAM-token." recovery_window_in_days = 0 @@ -138,12 +140,50 @@ resource "aws_secretsmanager_secret" "db_master_password" { } resource "aws_secretsmanager_secret_version" "db_master_password" { - secret_id = aws_secretsmanager_secret.db_master_password.id + count = var.create_database ? 1 : 0 + secret_id = aws_secretsmanager_secret.db_master_password[0].id secret_string = jsonencode({ username = var.db_master_username - password = random_password.db_master_password.result - host = aws_rds_cluster.this.endpoint - port = aws_rds_cluster.this.port + password = random_password.db_master_password[0].result + host = aws_rds_cluster.this[0].endpoint + port = aws_rds_cluster.this[0].port dbname = var.db_name }) } + +# Bring-your-own connection strings. Both hold credentials, so they go to +# Secrets Manager and reach the containers as ECS `secrets` rather than as +# plain-text env in the task definition. +resource "aws_secretsmanager_secret" "database_url" { + count = local.byo_database ? 1 : 0 + + name = "${local.name}-database-url" + description = "DATABASE_URL for an existing Postgres, used when create_database = false." + recovery_window_in_days = 0 + + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "database_url" { + count = local.byo_database ? 1 : 0 + + secret_id = aws_secretsmanager_secret.database_url[0].id + secret_string = var.database_url +} + +resource "aws_secretsmanager_secret" "redis_url" { + count = local.byo_redis ? 1 : 0 + + name = "${local.name}-redis-url" + description = "REDIS_URL for an existing Redis, used when create_redis = false." + recovery_window_in_days = 0 + + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "redis_url" { + count = local.byo_redis ? 1 : 0 + + secret_id = aws_secretsmanager_secret.redis_url[0].id + secret_string = var.redis_url +} diff --git a/terraform/litellm/aws/tests/byo_infrastructure.tftest.hcl b/terraform/litellm/aws/tests/byo_infrastructure.tftest.hcl new file mode 100644 index 00000000000..5a619bc98b1 --- /dev/null +++ b/terraform/litellm/aws/tests/byo_infrastructure.tftest.hcl @@ -0,0 +1,272 @@ +# Plan-only coverage for the four networking/database/cache permutations. +# `mock_provider` keeps this offline: no AWS credentials, no API calls, no +# resources. Run from terraform/litellm/aws with `terraform test`. + +mock_provider "aws" { + # IAM policy documents are validated as JSON by the provider, so the + # generated placeholder string has to be replaced with a parsable one. + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}" + } + } +} +mock_provider "random" {} + +variables { + region = "us-east-1" + tenant = "acme" + env = "test" + allow_plaintext_alb = true +} + +run "module_owns_everything_by_default" { + command = plan + + variables { + azs = ["us-east-1a", "us-east-1b"] + } + + assert { + condition = length(aws_vpc.this) == 1 && length(aws_nat_gateway.this) == 1 && length(aws_subnet.private) == 2 + error_message = "The default path must still create its own VPC, NAT gateway, and one private subnet per AZ." + } + + assert { + condition = length(aws_rds_cluster.this) == 1 && length(aws_elasticache_replication_group.this) == 1 + error_message = "The default path must still create Aurora and ElastiCache." + } + + assert { + condition = length(aws_secretsmanager_secret.database_url) == 0 && length(aws_secretsmanager_secret.redis_url) == 0 + error_message = "Connection-string secrets belong to the bring-your-own path only." + } + + assert { + condition = length(local.managed_db_env) == 7 && length(local.managed_redis_env) == 3 + error_message = "Gateway, backend, and migration tasks must keep the discrete DATABASE_*/REDIS_* env for the module-created stores." + } + + assert { + condition = length(terraform_data.bootstrap_db) == 1 && length(aws_ecs_task_definition.migrations) == 1 + error_message = "The IAM-user bootstrap and the schema migration must both run against the module-created Aurora." + } +} + +run "existing_vpc_creates_no_networking" { + command = plan + + variables { + vpc_id = "vpc-00000000000000001" + public_subnet_ids = ["subnet-pub-a", "subnet-pub-b"] + private_subnet_ids = ["subnet-priv-a", "subnet-priv-b"] + additional_task_security_group_ids = ["sg-caller-owned"] + } + + assert { + condition = alltrue([ + length(aws_vpc.this) == 0, + length(aws_subnet.public) == 0, + length(aws_subnet.private) == 0, + length(aws_internet_gateway.this) == 0, + length(aws_nat_gateway.this) == 0, + length(aws_eip.nat) == 0, + length(aws_route_table.public) == 0, + length(aws_route_table.private) == 0, + ]) + error_message = "An existing vpc_id must suppress every network resource, including the route tables and NAT gateway." + } + + assert { + condition = aws_lb.this.subnets == toset(var.public_subnet_ids) + error_message = "The ALB must land in the caller's public subnets." + } + + assert { + condition = alltrue([ + aws_db_subnet_group.this[0].subnet_ids == toset(var.private_subnet_ids), + aws_elasticache_subnet_group.this[0].subnet_ids == toset(var.private_subnet_ids), + aws_ecs_service.gateway.network_configuration[0].subnets == toset(var.private_subnet_ids), + ]) + error_message = "Tasks, Aurora, and ElastiCache must land in the caller's private subnets." + } + + assert { + condition = length(local.task_security_group_ids) == 2 + error_message = "additional_task_security_group_ids must be attached alongside the module's own tasks group." + } +} + +run "existing_database_and_redis_replace_the_managed_ones" { + command = plan + + variables { + azs = ["us-east-1a", "us-east-1b"] + create_database = false + database_url = "postgresql://litellm:pw@db.internal:5432/litellm" + create_redis = false + redis_url = "rediss://:pw@cache.internal:6379" + } + + assert { + condition = alltrue([ + length(aws_rds_cluster.this) == 0, + length(aws_rds_cluster_instance.writer) == 0, + length(aws_db_subnet_group.this) == 0, + length(aws_security_group.rds) == 0, + length(aws_elasticache_replication_group.this) == 0, + length(aws_elasticache_subnet_group.this) == 0, + length(aws_security_group.redis) == 0, + ]) + error_message = "Pointing at an existing database and cache must create neither Aurora nor ElastiCache." + } + + assert { + condition = length(local.managed_db_env) == 0 && length(local.managed_redis_env) == 0 + error_message = "The discrete DATABASE_*/REDIS_* env vars must be dropped so DATABASE_URL/REDIS_URL are the only connection targets." + } + + assert { + condition = alltrue([ + length([for s in local.shared_secrets : s if s.name == "DATABASE_URL"]) == 1, + length([for s in local.shared_secrets : s if s.name == "REDIS_URL"]) == 1, + ]) + error_message = "Both connection strings must reach the containers as Secrets Manager references, not plain-text env." + } + + assert { + condition = length(terraform_data.bootstrap_db) == 0 && length(aws_ecs_task_definition.migrations) == 1 + error_message = "An existing database still needs the schema migration, but not the Aurora IAM-user bootstrap." + } + + assert { + condition = length([for e in local.backend_default_env : e if e.name == "STORE_MODEL_IN_DB"]) == 1 + error_message = "STORE_MODEL_IN_DB must stay set when a database is reachable." + } +} + +run "vpc_without_subnets_fails_at_plan" { + command = plan + + variables { + vpc_id = "vpc-00000000000000001" + } + + expect_failures = [ + aws_lb.this, + aws_security_group.tasks, + ] +} + +run "neither_vpc_nor_azs_fails_at_plan" { + command = plan + + expect_failures = [ + aws_vpc.this, + ] +} + +# Aurora and ElastiCache subnet groups need two AZs, so one private subnet is +# only enough when neither store is module-created. +run "one_private_subnet_fails_while_a_managed_store_needs_two_azs" { + command = plan + + variables { + vpc_id = "vpc-00000000000000001" + public_subnet_ids = ["subnet-pub-a", "subnet-pub-b"] + private_subnet_ids = ["subnet-priv-a"] + } + + expect_failures = [ + aws_security_group.tasks, + ] +} + +run "one_private_subnet_is_enough_without_managed_stores" { + command = plan + + variables { + vpc_id = "vpc-00000000000000001" + public_subnet_ids = ["subnet-pub-a", "subnet-pub-b"] + private_subnet_ids = ["subnet-priv-a"] + create_database = false + create_redis = false + # Single process, so the Redis-less rate-limit check stays quiet and this + # run is only exercising the subnet rule. + gateway_autoscaling_enabled = false + gateway_desired_count = 1 + gateway_num_workers = 1 + } + + assert { + condition = length(aws_security_group.tasks.vpc_id) > 0 + error_message = "With no module-created database or cache, a single private subnet must plan cleanly." + } +} + +# The default sizing is 10 tasks under autoscaling, so a Redis-less stack must +# warn that per-key limits are counted per process. +run "redis_less_multi_process_gateway_is_flagged" { + command = plan + + variables { + azs = ["us-east-1a", "us-east-1b"] + create_redis = false + } + + expect_failures = [ + check.redis_less_rate_limits_are_per_process, + ] +} + +run "redis_less_single_process_gateway_is_not_flagged" { + command = plan + + variables { + azs = ["us-east-1a", "us-east-1b"] + create_redis = false + gateway_autoscaling_enabled = false + gateway_desired_count = 1 + gateway_num_workers = 1 + } + + assert { + condition = local.max_gateway_processes == 1 + error_message = "One task with one worker is a single process, which is the supported way to run without Redis." + } +} + +run "no_database_and_no_redis_drops_the_schema_migration" { + command = plan + + variables { + azs = ["us-east-1a", "us-east-1b"] + create_database = false + create_redis = false + # Single process, so the Redis-less rate-limit check stays quiet here; it + # has its own run above. + gateway_autoscaling_enabled = false + gateway_desired_count = 1 + gateway_num_workers = 1 + } + + assert { + condition = alltrue([ + length(aws_ecs_task_definition.migrations) == 0, + length(terraform_data.migration) == 0, + length(aws_iam_policy.rds_iam_connect) == 0, + length(aws_secretsmanager_secret.db_master_password) == 0, + ]) + error_message = "With no database at all there is nothing to migrate, bootstrap, or grant rds-db:connect on." + } + + assert { + condition = length(local.backend_default_env) == 0 + error_message = "STORE_MODEL_IN_DB must not be set without a database to store models in." + } + + assert { + condition = length(local.shared_env) == 4 + error_message = "The shared env must narrow to the S3 bucket and region pair when both data stores are gone." + } +} diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf index c2ed1db14b1..522138953d6 100644 --- a/terraform/litellm/aws/variables.tf +++ b/terraform/litellm/aws/variables.tf @@ -74,20 +74,63 @@ variable "ui_password" { } # ---------- Networking ---------- +# +# Two modes: +# +# 1. Module-owned (default, `vpc_id = ""`): the stack creates a VPC, public +# and private subnets per AZ, an internet gateway, a NAT gateway, and +# the route tables wiring them together. `vpc_cidr` + `azs` drive it. +# 2. Bring-your-own (`vpc_id` set): the stack creates no networking and +# places the ALB in `public_subnet_ids` and every task, plus the Aurora +# and ElastiCache subnet groups, in `private_subnet_ids`. `vpc_cidr` and +# `azs` are then unused. + +variable "vpc_id" { + description = <<-EOT + Existing VPC to deploy into. Leave empty ("") to have the module create + its own VPC, subnets, NAT gateway, and route tables. When set, + `public_subnet_ids` and `private_subnet_ids` are required and no + networking is created: the private subnets must already have egress + (NAT gateway or equivalent) so tasks can reach LLM providers, ECR/GHCR, + and Secrets Manager. + EOT + type = string + default = "" +} + +variable "public_subnet_ids" { + description = "Existing public subnets for the ALB, in at least 2 AZs. Required when `vpc_id` is set, ignored otherwise." + type = list(string) + default = [] +} + +variable "private_subnet_ids" { + description = "Existing private subnets for the ECS tasks, Aurora, and ElastiCache. Required when `vpc_id` is set, ignored otherwise." + type = list(string) + default = [] +} + +variable "additional_task_security_group_ids" { + description = <<-EOT + Extra security groups to attach to the ECS tasks, on top of the one the + module creates. Useful with `vpc_id`: attach a group your existing + database or cache already allows inbound from, instead of editing their + ingress rules. + EOT + type = list(string) + default = [] +} variable "vpc_cidr" { - description = "CIDR block for the VPC." + description = "CIDR block for the VPC the module creates. Unused when `vpc_id` is set." type = string default = "10.40.0.0/16" } variable "azs" { - description = "Availability zones to spread subnets across. At least 2 required for RDS and ALB." + description = "Availability zones to spread the module-created subnets across. At least 2 required for Aurora and the ALB. Unused when `vpc_id` is set." type = list(string) - validation { - condition = length(var.azs) >= 2 - error_message = "Provide at least 2 availability zones." - } + default = [] } # ---------- Component images ---------- @@ -279,6 +322,34 @@ variable "ui_cpu_target" { # ---------- RDS ---------- +variable "create_database" { + description = <<-EOT + Create the Aurora Postgres cluster (default). Set false to skip it and + either point the stack at an existing database via `database_url`, or + run without a database at all when `database_url` is also empty. The + DB-less mode drops key management, spend tracking, and the admin UI's + persistence: the proxy then serves traffic authenticated by + LITELLM_MASTER_KEY only. + EOT + type = bool + default = true +} + +variable "database_url" { + description = <<-EOT + Postgres connection string for an existing database, e.g. + `postgresql://user:pass@host:5432/litellm`. Only read when + `create_database = false`. Stored in a + `-litellm--database-url` Secrets Manager entry and injected + into gateway, backend, and the migration task as DATABASE_URL, so the + value never lands in a task definition. The schema migration still runs + against it on every apply. + EOT + type = string + default = "" + sensitive = true +} + variable "db_instance_class" { description = "Aurora instance class for both writer and reader." type = string @@ -311,6 +382,31 @@ variable "db_username" { # ---------- Redis ---------- +variable "create_redis" { + description = <<-EOT + Create the ElastiCache Redis replication group (default). Set false to + skip it and either point the stack at an existing cache via `redis_url`, + or run with no Redis at all when `redis_url` is also empty. Without + Redis the proxy loses cross-task state: rate limits, budgets, and the + router's cooldowns become per-task instead of cluster-wide. + EOT + type = bool + default = true +} + +variable "redis_url" { + description = <<-EOT + Connection string for an existing Redis, e.g. + `rediss://:password@host:6379`. Only read when `create_redis = false`. + Stored in a `-litellm--redis-url` Secrets Manager entry and + injected as REDIS_URL, which takes precedence over REDIS_HOST/REDIS_PORT + in the proxy. + EOT + type = string + default = "" + sensitive = true +} + variable "redis_node_type" { description = "ElastiCache node type." type = string From 50ff555d7900716e447a3008cb8c56d851d73f21 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 12 Aug 2026 16:47:45 -0700 Subject: [PATCH 20/25] fix(ui): show and edit key-level router settings on a virtual key (#36674) The virtual-key edit view never rendered the router settings stored on a key, so fallbacks configured at creation could not be verified or changed afterwards. The key info panel now summarises them and the edit view embeds the router settings accordion. The accordion is a fixed-field editor, so its value is merged over the stored object instead of replacing it. Routing fields it cannot render, such as tag_routing_prefix or model_group_retry_policy, survive an unrelated edit, while a field it does own that the admin emptied still goes out as null so the clear reaches the server. Emptying every field sends {}, which the proxy reads as no key-level override, so the key falls back to its team and global routing rather than being pinned to a blob of nulls. --- .../RouterSettingsSummary.test.tsx | 28 +++++ .../RouterSettingsSummary.tsx | 56 +++++++++ .../routerSettingsPayload.test.ts | 115 ++++++++++++++++++ .../routerSettingsPayload.ts | 63 ++++++++++ .../routerSettingsWiring.test.tsx | 79 ++++++++++++ .../components/key_team_helpers/key_list.tsx | 1 + .../templates/key_edit_view.test.tsx | 95 +++++++++++++++ .../components/templates/key_edit_view.tsx | 22 +++- .../templates/key_info_view.test.tsx | 19 +++ .../components/templates/key_info_view.tsx | 11 ++ 10 files changed, 488 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts create mode 100644 ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts create mode 100644 ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx new file mode 100644 index 00000000000..51eecc421a7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx @@ -0,0 +1,28 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import RouterSettingsSummary from "./RouterSettingsSummary"; + +describe("RouterSettingsSummary", () => { + it("should list each configured fallback mapping", () => { + render( + , + ); + + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o, claude-sonnet")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o-mini")).toBeInTheDocument(); + expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument(); + }); + + it("should show the empty state when every setting is null", () => { + render(); + + expect(screen.getByText("No router settings configured")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx new file mode 100644 index 00000000000..daaddc0cdfb --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx @@ -0,0 +1,56 @@ +import { Badge } from "@/components/ui/badge"; +import { hasRouterSettings } from "./routerSettingsPayload"; + +interface RouterSettingsSummaryProps { + routerSettings: Record | null | undefined; + emptyText?: string; +} + +const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => { + if (!Array.isArray(fallbacks)) return []; + return fallbacks.flatMap((entry) => + entry && typeof entry === "object" ? (Object.entries(entry) as Array<[string, string[]]>) : [], + ); +}; + +export default function RouterSettingsSummary({ + routerSettings, + emptyText = "No router settings configured", +}: RouterSettingsSummaryProps) { + if (!hasRouterSettings(routerSettings)) { + return
{emptyText}
; + } + + const settings = routerSettings as Record; + const fallbacks = fallbackEntries(settings.fallbacks); + + return ( +
+ {settings.routing_strategy != null && ( +
+ Routing Strategy: {String(settings.routing_strategy)} +
+ )} + {settings.num_retries != null &&
Number of Retries: {String(settings.num_retries)}
} + {settings.allowed_fails != null &&
Allowed Failures: {String(settings.allowed_fails)}
} + {settings.cooldown_time != null &&
Cooldown Time: {String(settings.cooldown_time)}s
} + {settings.timeout != null &&
Timeout: {String(settings.timeout)}s
} + {settings.retry_after != null &&
Retry After: {String(settings.retry_after)}s
} + {Boolean(settings.enable_tag_filtering) &&
Tag Filtering: Enabled
} + {fallbacks.length > 0 && ( +
+
Fallbacks:
+
+ {fallbacks.map(([model, targets]) => ( +
+ {model} + -> + {Array.isArray(targets) ? targets.join(", ") : String(targets)} +
+ ))} +
+
+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts new file mode 100644 index 00000000000..542ed17d7a2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from "vitest"; +import { hasRouterSettings, routerSettingsEditorValue, routerSettingsUpdate } from "./routerSettingsPayload"; + +describe("hasRouterSettings", () => { + it("should treat unset, empty and all-null settings as absent", () => { + expect(hasRouterSettings(undefined)).toBe(false); + expect(hasRouterSettings(null)).toBe(false); + expect(hasRouterSettings({})).toBe(false); + expect( + hasRouterSettings({ num_retries: null, fallbacks: [], model_group_alias: {}, enable_tag_filtering: false }), + ).toBe(false); + }); + + it("should detect configured settings", () => { + expect(hasRouterSettings({ num_retries: 3 })).toBe(true); + expect(hasRouterSettings({ fallbacks: [{ "gpt-4": ["gpt-4o"] }] })).toBe(true); + expect(hasRouterSettings({ enable_tag_filtering: true })).toBe(true); + expect(hasRouterSettings({ num_retries: 0 })).toBe(true); + }); +}); + +describe("routerSettingsEditorValue", () => { + it("should hand the editor only the fields it renders", () => { + expect( + routerSettingsEditorValue({ + num_retries: 2, + tag_routing_prefix: "team-", + fallbacks: [{ "gpt-4": ["gpt-4o"] }], + }), + ).toStrictEqual({ router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] } }); + }); + + it("should keep an explicitly stored null so the editor renders it as empty", () => { + expect(routerSettingsEditorValue({ num_retries: null })).toStrictEqual({ router_settings: { num_retries: null } }); + }); + + it("should leave the editor uninitialised when the key has no stored settings", () => { + expect(routerSettingsEditorValue(null)).toBeUndefined(); + expect(routerSettingsEditorValue(undefined)).toBeUndefined(); + }); +}); + +describe("routerSettingsUpdate", () => { + const fallbacks = [{ "gpt-4": ["gpt-4o"] }]; + // Accepted by UpdateRouterConfig on /key/update but not rendered by the accordion. + const unsupported = { + tag_routing_prefix: "team-", + model_group_retry_policy: { "gpt-4": { TimeoutErrorRetries: 2 } }, + }; + + // The editor is a fixed-field form, so the merge has to hold two opposing guarantees at once: + // routing fields it cannot render survive, and a field it does render that the admin emptied + // is still sent as null. Orderings vary because an object spread resolves collisions by position. + const storedOrderings: Array<[string, Record]> = [ + ["unsupported fields first", { ...unsupported, num_retries: 2, fallbacks }], + ["unsupported fields last", { num_retries: 2, fallbacks, ...unsupported }], + [ + "unsupported fields interleaved", + { + tag_routing_prefix: unsupported.tag_routing_prefix, + num_retries: 2, + model_group_retry_policy: unsupported.model_group_retry_policy, + fallbacks, + }, + ], + ]; + + it.each(storedOrderings)( + "should keep unsupported stored fields and still clear an emptied editor field (%s)", + (_ordering, stored) => { + const result = routerSettingsUpdate({ num_retries: 4, fallbacks: null }, stored); + + expect(result).toMatchObject({ ...unsupported, num_retries: 4, fallbacks: null }); + }, + ); + + it("should send an empty object, not a null blob, when the last stored setting is cleared", () => { + expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, num_retries: 2 })).toEqual({}); + }); + + it("should keep clearing owned fields explicitly while an unsupported setting still stands", () => { + expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, { fallbacks, ...unsupported })).toMatchObject({ + ...unsupported, + fallbacks: null, + num_retries: null, + }); + }); + + it("should null every editor-owned field the editor left out", () => { + expect(routerSettingsUpdate({ fallbacks: null }, { timeout: 30, ...unsupported })).toEqual({ + ...unsupported, + routing_strategy: null, + allowed_fails: null, + cooldown_time: null, + num_retries: null, + timeout: null, + retry_after: null, + fallbacks: null, + context_window_fallbacks: null, + retry_policy: null, + model_group_alias: null, + enable_tag_filtering: null, + routing_strategy_args: null, + }); + }); + + it("should send the edited settings when the user configured something", () => { + expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks }); + }); + + it("should leave the field off when nothing is stored and nothing was configured", () => { + expect(routerSettingsUpdate({ fallbacks: null, num_retries: null }, {})).toBeUndefined(); + expect(routerSettingsUpdate(undefined, { fallbacks })).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts new file mode 100644 index 00000000000..1d048425a6d --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts @@ -0,0 +1,63 @@ +import { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; + +export type RouterSettings = RouterSettingsAccordionValue["router_settings"]; + +const EDITOR_OWNED_FIELDS: Record = { + routing_strategy: true, + allowed_fails: true, + cooldown_time: true, + num_retries: true, + timeout: true, + retry_after: true, + fallbacks: true, + context_window_fallbacks: true, + retry_policy: true, + model_group_alias: true, + enable_tag_filtering: true, + routing_strategy_args: true, +}; + +const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array; + +const isMeaningfulRouterSetting = (value: unknown): boolean => { + if (value === null || value === undefined || value === "" || value === false) return false; + if (Array.isArray(value)) return value.length > 0; + if (typeof value === "object") return Object.keys(value).length > 0; + return true; +}; + +export const hasRouterSettings = (settings: Record | null | undefined): boolean => + settings != null && Object.values(settings).some(isMeaningfulRouterSetting); + +/** + * The stored settings narrowed to what the editor renders. The stored blob is untyped JSON + * from /key/info, so this projection is the one place it is read as RouterSettings. + */ +export const routerSettingsEditorValue = ( + stored: Record | null | undefined, +): RouterSettingsAccordionValue | undefined => + stored + ? { + router_settings: Object.fromEntries( + EDITOR_OWNED_KEYS.filter((key) => key in stored).map((key) => [key, stored[key]]), + ) as RouterSettings, + } + : undefined; + +/** + * Router settings to put on a /key/update payload, or undefined to leave the field off. + * The editor only renders EDITOR_OWNED_FIELDS, so its value is merged over the stored object + * rather than replacing it, and routing fields the editor cannot show survive an unrelated edit. + * Emptying every field sends {}, which the proxy reads as "no key-level override" so the key + * falls back to its team and global settings, where an all-null blob would pin it to nulls. + */ +export const routerSettingsUpdate = ( + edited: RouterSettings | null | undefined, + stored: Record | null | undefined, +): Record | undefined => { + if (!edited) return undefined; + const editorOwned = Object.fromEntries(EDITOR_OWNED_KEYS.map((key) => [key, edited[key] ?? null])); + const merged: Record = { ...stored, ...editorOwned }; + if (hasRouterSettings(merged)) return merged; + return hasRouterSettings(stored) ? {} : undefined; +}; diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx b/ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx new file mode 100644 index 00000000000..d6ed0ae07db --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsWiring.test.tsx @@ -0,0 +1,79 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import type { ReactElement, ReactNode } from "react"; +import { describe, expect, it, vi } from "vitest"; +import type { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig"; +import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm"; +import RouterSettingsAccordion from "./RouterSettingsAccordion"; +import { routerSettingsEditorValue } from "./routerSettingsPayload"; + +vi.mock("../networking", () => ({ + getRouterSettingsCall: vi.fn().mockResolvedValue({}), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5.5" }, { model_group: "gpt-4o-mini" }]), + fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([]), +})); + +vi.mock("@tremor/react", () => ({ + TabGroup: ({ children }: { children: ReactNode }) =>
{children}
, + TabList: ({ children }: { children: ReactNode }) =>
{children}
, + Tab: ({ children }: { children: ReactNode }) =>
{children}
, + TabPanels: ({ children }: { children: ReactNode }) =>
{children}
, + TabPanel: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock("../router_settings/RouterSettingsForm", () => ({ + default: ({ value }: { value: RouterSettingsFormValue }) => ( +
{JSON.stringify(value.routerSettings)}
+ ), +})); + +vi.mock("../Settings/RouterSettings/Fallbacks/FallbackSelectionForm", () => ({ + FallbackSelectionForm: ({ groups }: { groups: FallbackGroup[] }) => ( +
+ {groups.map((g) => `${g.primaryModel ?? "none"}->${g.fallbackModels.join("|") || "none"}`).join(" ")} +
+ ), +})); + +// Captured verbatim from GET /key/info on a live proxy for a key created through the UI. +// tag_routing_prefix is stored on the key and accepted by /key/update, but the accordion +// has no control for it, so the projection must drop it without losing the rest. +const KEY_INFO_ROUTER_SETTINGS: Record = { + fallbacks: [{ "gpt-5.5": ["gpt-4o-mini"] }], + num_retries: 3, + tag_routing_prefix: "team-", +}; + +const renderAccordion = (stored: Record | null): ReactElement => ( + + + +); + +describe("key router settings wiring, /key/info payload through to rendered output", () => { + it("should prefill both tabs from the stored settings", async () => { + render(renderAccordion(KEY_INFO_ROUTER_SETTINGS)); + + await waitFor(() => { + expect(screen.getByTestId("fallbacks")).toHaveTextContent("gpt-5.5->gpt-4o-mini"); + }); + expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "{}")).toMatchObject({ num_retries: 3 }); + }); + + it("should not leak a field the accordion has no control for into the editor", async () => { + render(renderAccordion(KEY_INFO_ROUTER_SETTINGS)); + + await waitFor(() => expect(screen.getByTestId("loadbalancing")).toBeInTheDocument()); + expect(screen.getByTestId("loadbalancing").textContent).not.toContain("tag_routing_prefix"); + }); + + it("should render an empty editor for a key holding only fields it cannot show", async () => { + render(renderAccordion({ tag_routing_prefix: "team-" })); + + await waitFor(() => expect(screen.getByTestId("fallbacks")).toHaveTextContent("none->none")); + expect(JSON.parse(screen.getByTestId("loadbalancing").textContent ?? "null")).toEqual({}); + }); +}); diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx index 920d1f4af5a..caa5e0a38f0 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/key_list.tsx @@ -95,6 +95,7 @@ export interface KeyResponse { object_permission?: ObjectPermission | null; access_group_ids?: string[]; budget_fallbacks?: Record; + router_settings?: Record | null; budget_limits?: Array<{ budget_duration: string; max_budget: number; reset_at?: string }>; auto_rotate?: boolean; rotation_interval?: string; diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx index fb8ab87e45f..caad6fc30fc 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.test.tsx @@ -59,6 +59,24 @@ vi.mock("../organisms/create_key_button", () => ({ fetchTeamModels: vi.fn().mockResolvedValue(["team-model-1", "team-model-2"]), })); +const routerSettingsMocks = vi.hoisted(() => ({ + receivedValue: undefined as { router_settings: Record } | undefined, + editedValue: null as Record | null, +})); + +vi.mock("../common_components/RouterSettingsAccordion", async () => { + const { forwardRef, useImperativeHandle } = await import("react"); + return { + default: forwardRef(({ value }: { value?: { router_settings: Record } }, ref) => { + routerSettingsMocks.receivedValue = value; + useImperativeHandle(ref, () => ({ + getValue: () => ({ router_settings: routerSettingsMocks.editedValue ?? value?.router_settings ?? {} }), + })); + return
; + }), + }; +}); + vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ useOrganizations: vi.fn().mockReturnValue({ data: [ @@ -160,6 +178,83 @@ describe("KeyEditView", () => { last_rotation_at: undefined, key_rotation_at: undefined, }; + describe("router settings", () => { + const UNSUPPORTED_STORED_FIELD = { tag_routing_prefix: "team-" }; + const STORED_ROUTER_SETTINGS = { + num_retries: 2, + fallbacks: [{ "gpt-4": ["gpt-4o"] }], + ...UNSUPPORTED_STORED_FIELD, + }; + + const renderWithRouterSettings = (onSubmit: (values: Record) => Promise) => + renderWithProviders( + {}} + onSubmit={onSubmit} + accessToken="test-token" + userID="test-user" + userRole="proxy_admin" + premiumUser={true} + />, + ); + + beforeEach(() => { + routerSettingsMocks.receivedValue = undefined; + routerSettingsMocks.editedValue = null; + }); + + it("should load the fields it renders into the editor and withhold the ones it does not", async () => { + renderWithRouterSettings(async () => {}); + + await waitFor(() => { + expect(routerSettingsMocks.receivedValue).toStrictEqual({ + router_settings: { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o"] }] }, + }); + }); + }); + + it("should submit edited fallbacks alongside routing fields the editor cannot show", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderWithRouterSettings(onSubmit); + routerSettingsMocks.editedValue = { num_retries: 2, fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }] }; + + fireEvent.click(screen.getByText("Save Changes")); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledWith( + expect.objectContaining({ + router_settings: expect.objectContaining({ + ...UNSUPPORTED_STORED_FIELD, + num_retries: 2, + fallbacks: [{ "gpt-4": ["gpt-4o", "gpt-4o-mini"] }], + }), + }), + ); + }); + }); + + it("should submit cleared router settings so removing every fallback is persisted", async () => { + const onSubmit = vi.fn().mockResolvedValue(undefined); + renderWithRouterSettings(onSubmit); + routerSettingsMocks.editedValue = { num_retries: null, fallbacks: null }; + + fireEvent.click(screen.getByText("Save Changes")); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledWith( + expect.objectContaining({ + router_settings: expect.objectContaining({ + ...UNSUPPORTED_STORED_FIELD, + num_retries: null, + fallbacks: null, + }), + }), + ); + }); + }); + }); + it("should render", async () => { const { getByText } = renderWithProviders( >( keyData.budget_fallbacks && typeof keyData.budget_fallbacks === "object" ? keyData.budget_fallbacks : {}, ); + const routerSettingsRef = useRef(null); const { data: organizations, isLoading: isOrganizationsLoading } = useOrganizations(); const { data: projects } = useProjects(); const { data: uiSettingsData } = useUISettings(); @@ -286,6 +289,14 @@ export function KeyEditView({ values.budget_fallbacks = {}; } + const routerSettings = routerSettingsUpdate( + routerSettingsRef.current?.getValue()?.router_settings, + keyData.router_settings, + ); + if (routerSettings) { + values.router_settings = routerSettings; + } + await onSubmit(withNormalizedEstimates(values)); } finally { setIsKeySaving(false); @@ -799,6 +810,15 @@ export function KeyEditView({ )} + + + + { }); }); + it("should render the key's saved router fallbacks", async () => { + vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); + + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + + expect(await screen.findByText("Router Settings")).toBeInTheDocument(); + expect(screen.getByText("gpt-4")).toBeInTheDocument(); + expect(screen.getByText("gpt-4o")).toBeInTheDocument(); + expect(screen.getByText("Number of Retries: 2")).toBeInTheDocument(); + }); + it("should render tags", async () => { vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 6fd6547a995..f7720874968 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -13,6 +13,8 @@ import { isProxyAdminRole, isUserTeamAdminForSingleTeam, rolesWithWriteAccess } import { mapDisplayToInternalNames, mapInternalToDisplayNames } from "../callback_info_helpers"; import AutoRotationView from "../common_components/AutoRotationView"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import RouterSettingsSummary from "../common_components/RouterSettingsSummary"; +import { hasRouterSettings } from "../common_components/routerSettingsPayload"; import { extractLoggingSettings, formatMetadataForDisplay, stripTagsFromMetadata } from "../key_info_utils"; import { KeyResponse } from "../key_team_helpers/key_list"; import LoggingSettingsView from "../logging_settings_view"; @@ -837,6 +839,15 @@ export default function KeyInfoView({
)} + {hasRouterSettings(currentKeyData.router_settings) && ( +
+ Router Settings +
+ +
+
+ )} +
Tags
From ce3d30a43414f5b719fcbd78d92ba1794e50ecb6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 13 Aug 2026 00:45:49 +0000 Subject: [PATCH 21/25] fix(router): read no tags, instead of raising, from a non-dict metadata bucket Pre-routing now reads the request's tags on every request with a registered strategy, including the single-strategy case that used to short-circuit before looking at tags. Metadata is request-controlled, so a caller that sends `litellm_metadata` (or `tags`) as a string or any other non-dict shape crashed tag lookup with an AttributeError instead of routing untagged. --- litellm/litellm_core_utils/core_helpers.py | 4 +-- litellm/router_strategy/tag_based_routing.py | 35 ++++++++++++++----- .../test_router_tag_routing.py | 27 ++++++++++++++ 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 2462c282041..de1092bc02f 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,7 +1,7 @@ # What is this? ## Helper utilities import copy -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -181,7 +181,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad def get_metadata_variable_name_from_kwargs( - kwargs: dict, + kwargs: Mapping[str, object], ) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 40ed89ebfd8..1120323b4f9 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -584,8 +584,26 @@ async def get_deployments_for_tag( return healthy_deployments +def _tags_in_metadata(metadata: object) -> list[str]: + """ + Tags out of a metadata bucket the caller controls the shape of. + + A request can send its metadata (and its ``tags``) as anything the JSON body + allowed, an unparsed string or null included, so any shape that is not a list + of string tags carries no tags rather than raising. + """ + if not isinstance(metadata, Mapping): + return [] + typed_metadata: Final[Mapping[str, object]] = metadata + tags: Final = typed_metadata.get("tags") + if isinstance(tags, str) or not isinstance(tags, Sequence): + return [] + typed_tags: Final[Sequence[object]] = tags + return [tag for tag in typed_tags if isinstance(tag, str)] + + def _get_tags_from_request_kwargs( - request_kwargs: dict[Any, Any] | None = None, + request_kwargs: Mapping[Any, Any] | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, ) -> list[str]: """ @@ -604,12 +622,11 @@ def _get_tags_from_request_kwargs( return [] resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs) if resolved_variable_name in request_kwargs: - metadata: Final = request_kwargs[resolved_variable_name] or {} - tags = metadata.get("tags", []) - return tags if tags is not None else [] - elif "litellm_params" in request_kwargs: - litellm_params: Final = request_kwargs["litellm_params"] or {} - _metadata: Final = litellm_params.get(resolved_variable_name, {}) or {} - tags = _metadata.get("tags", []) - return tags if tags is not None else [] + return _tags_in_metadata(request_kwargs[resolved_variable_name]) + if "litellm_params" in request_kwargs: + litellm_params: Final = request_kwargs["litellm_params"] + if not isinstance(litellm_params, Mapping): + return [] + typed_litellm_params: Final[Mapping[str, object]] = litellm_params + return _tags_in_metadata(typed_litellm_params.get(resolved_variable_name)) return [] diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 93011bb29cc..73491490b14 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -423,6 +423,33 @@ def test_get_tags_from_request_kwargs_various_inputs(): assert _get_tags_from_request_kwargs({"foo": "bar"}) == [] +@pytest.mark.parametrize( + "request_kwargs", + [ + {"metadata": "not-a-dict"}, + {"litellm_metadata": "not-a-dict"}, + {"litellm_metadata": ["not", "a", "dict"]}, + {"litellm_params": "not-a-dict"}, + {"litellm_params": {"metadata": "not-a-dict"}}, + {"metadata": {"tags": "free"}}, + {"metadata": {"tags": {"free": "paid"}}}, + ], +) +def test_get_tags_from_request_kwargs_reads_no_tags_from_a_non_dict_shape(request_kwargs): + """Metadata and `tags` are request-controlled, so a client can send either as a + string, a list or null. Every shape that cannot hold string tags reads as untagged + instead of raising, because callers run on the hot request path.""" + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs(request_kwargs) == [] + + +def test_get_tags_from_request_kwargs_keeps_only_string_tags(): + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs({"metadata": {"tags": ["free", 7, None, "paid"]}}) == ["free", "paid"] + + # --- _split_tags unit tests --- From 3864e1241560ff8e2fc98bcc3c84bd03f6f63d8a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:10:50 -0700 Subject: [PATCH 22/25] fix(spend): stop losing spend log rows when a flush is cancelled (#34826) --- litellm/proxy/proxy_server.py | 22 +- litellm/proxy/utils.py | 58 +++++- .../proxy/proxy_server/test_lifecycle.py | 44 ++++ .../proxy/utils/prisma_and_spend/conftest.py | 1 + .../prisma_and_spend/test_spend_functions.py | 194 ++++++++++++++++++ 5 files changed, 310 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3ac002ce8cc..64a91b880cd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -845,6 +845,22 @@ def cleanup_router_config_variables(): prisma_client = None +async def _flush_spend_logs_queue_on_shutdown() -> None: + if prisma_client is None: + return + + try: + from litellm.proxy.utils import drain_spend_logs_queue + + await drain_spend_logs_queue( + prisma_client=prisma_client, + db_writer_client=db_writer_client, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception as e: # noqa: BLE001 # shutdown must continue even if the drain fails + verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e) + + async def proxy_shutdown_event(): global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server") @@ -1255,6 +1271,8 @@ async def proxy_startup_event(app: FastAPI): except Exception as e: verbose_proxy_logger.error("Error stopping DB health watchdog task: %s", e) + await _flush_spend_logs_queue_on_shutdown() + await proxy_config.stop_config_sync_subscriber() await proxy_config.stop_auth_cache_invalidation_subscriber() @@ -8731,14 +8749,14 @@ class ProxyStartupEvent: if general_settings.get("disable_spend_logs", False) is False: from litellm.proxy.utils import _monitor_spend_logs_queue - # Start background task to monitor spend logs queue size - asyncio.create_task( + monitor_task: Final = asyncio.create_task( _monitor_spend_logs_queue( prisma_client=prisma_client, db_writer_client=db_writer_client, proxy_logging_obj=proxy_logging_obj, ) ) + prisma_client.spend_logs_queue_monitor_task = monitor_task # rebind-ok: the client owns its monitor handle ### ADD NEW MODELS ### store_model_in_db = get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cce8379ab25..8a1fae42789 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import copy import hashlib import inspect @@ -3006,6 +3007,7 @@ async def prefetch_config_params(prisma_client: "PrismaClient | None", param_nam class PrismaClient: spend_log_transactions: list = [] _spend_log_transactions_lock = asyncio.Lock() + spend_logs_queue_monitor_task: "asyncio.Task[None] | None" = None tool_usage_transactions: list["ToolUsageTransaction"] = [] _tool_usage_transactions_lock = asyncio.Lock() autorouter_turn_transactions: ClassVar[ @@ -5722,13 +5724,22 @@ async def update_spend_logs_job( logs_to_process: Final = prisma_client.spend_log_transactions[:MAX_LOGS_PER_INTERVAL] prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[len(logs_to_process) :] - await ProxyUpdateSpend.update_spend_logs( - n_retry_times=n_retry_times, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - db_writer_client=db_writer_client, - logs_to_process=logs_to_process, - ) + try: + await ProxyUpdateSpend.update_spend_logs( + n_retry_times=n_retry_times, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + db_writer_client=db_writer_client, + logs_to_process=logs_to_process, + ) + except asyncio.CancelledError: + async with prisma_client._spend_log_transactions_lock: + prisma_client.spend_log_transactions[:0] = logs_to_process + verbose_proxy_logger.warning( + "Spend tracking - spend log write cancelled, requeued %d rows for the next flush", + len(logs_to_process), + ) + raise # Guardrail/policy usage tracking (same batch, outside spend-logs update) try: @@ -5787,6 +5798,39 @@ async def update_spend_logs_job( ) +MAX_SPEND_LOG_DRAIN_ITERATIONS: Final = 20 + + +async def drain_spend_logs_queue( + prisma_client: PrismaClient, + db_writer_client: "AsyncHTTPHandler | None", + proxy_logging_obj: ProxyLogging, +) -> None: + monitor_task: Final = prisma_client.spend_logs_queue_monitor_task + if monitor_task is not None: + monitor_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await monitor_task + prisma_client.spend_logs_queue_monitor_task = None # rebind-ok: the client owns its monitor handle + + for _ in range(MAX_SPEND_LOG_DRAIN_ITERATIONS): + if await _total_queued_spend_transactions(prisma_client) == 0: + return + await update_spend_logs_job( + prisma_client=prisma_client, + db_writer_client=db_writer_client, + proxy_logging_obj=proxy_logging_obj, + ) + + remaining: Final = await _total_queued_spend_transactions(prisma_client) + if remaining > 0: + spend_log_error( + "Spend tracking - %d spend log rows still queued after %d drain passes", + remaining, + MAX_SPEND_LOG_DRAIN_ITERATIONS, + ) + + async def _monitor_spend_logs_queue( prisma_client: PrismaClient, db_writer_client: AsyncHTTPHandler | None, diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py index 40ca7e3a64e..ad41592db1e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py +++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py @@ -205,6 +205,50 @@ async def test_proxy_shutdown_event_prisma_disconnect_raises_error(monkeypatch): await proxy_shutdown_event() +# --------------------------------------------------------------------------- +# _flush_spend_logs_queue_on_shutdown +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_flush_spend_logs_queue_on_shutdown_drains_before_disconnect(monkeypatch): + fake_prisma = MagicMock() + monkeypatch.setattr(ps, "prisma_client", fake_prisma, raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + drain = AsyncMock() + import litellm.proxy.utils as utils_mod + + monkeypatch.setattr(utils_mod, "drain_spend_logs_queue", drain) + + await ps._flush_spend_logs_queue_on_shutdown() + + observed = { + "drain_calls": drain.await_count, + "drain_prisma": drain.await_args.kwargs["prisma_client"] is fake_prisma, + } + assert observed == { + "drain_calls": 1, + "drain_prisma": True, + } + + +@pytest.mark.asyncio +async def test_flush_spend_logs_queue_on_shutdown_swallows_drain_errors(monkeypatch): + monkeypatch.setattr(ps, "prisma_client", MagicMock(), raising=False) + monkeypatch.setattr(ps, "db_writer_client", None, raising=False) + + import litellm.proxy.utils as utils_mod + + monkeypatch.setattr( + utils_mod, + "drain_spend_logs_queue", + AsyncMock(side_effect=RuntimeError("db gone")), + ) + + await ps._flush_spend_logs_queue_on_shutdown() + + # --------------------------------------------------------------------------- # _initialize_shared_aiohttp_session # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py index 74c9abd9978..19abcb5d66d 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py @@ -128,6 +128,7 @@ def mock_prisma_client() -> MagicMock: client.proxy_logging_obj.failure_handler = AsyncMock() client.spend_log_transactions = [] client._spend_log_transactions_lock = asyncio.Lock() + client.spend_logs_queue_monitor_task = None client.tool_usage_transactions = [] client._tool_usage_transactions_lock = asyncio.Lock() client.jsonify_object = lambda data: dict(data) diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py index d9eeb168611..54d59e690f9 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py @@ -17,8 +17,10 @@ from unittest.mock import AsyncMock, MagicMock import pytest from litellm.proxy.utils import ( + MAX_SPEND_LOG_DRAIN_ITERATIONS, _monitor_spend_logs_queue, _raise_failed_update_spend_exception, + drain_spend_logs_queue, update_daily_tag_spend, update_spend, update_spend_logs_job, @@ -263,6 +265,198 @@ async def test_update_spend_logs_job_processes_and_clears_queue( } +@pytest.mark.asyncio +async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled( + mock_prisma_client: Any, make_spend_log_row: Any +) -> None: + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [ + make_spend_log_row(request_id="r1"), + make_spend_log_row(request_id="r2"), + ] + + row_arriving_mid_flush = make_spend_log_row(request_id="r3") + + async def _cancel_mid_write(*args: Any, **kwargs: Any) -> None: + mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush) + raise asyncio.CancelledError() + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( + side_effect=_cancel_mid_write + ) + + with pytest.raises(asyncio.CancelledError): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert [ + row["request_id"] for row in mock_prisma_client.spend_log_transactions + ] == ["r1", "r2", "r3"] + + +@pytest.mark.asyncio +async def test_update_spend_logs_job_does_not_requeue_when_cancelled_after_write( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """Rows are already committed once guardrail tracking runs, so replaying + them would double-count the non-idempotent daily guardrail increments. + """ + import litellm.proxy.guardrails.usage_tracking as guard_mod + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock() + + monkeypatch.setattr( + guard_mod, + "process_spend_logs_guardrail_usage", + AsyncMock(side_effect=asyncio.CancelledError()), + raising=False, + ) + + with pytest.raises(asyncio.CancelledError): + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert mock_prisma_client.spend_log_transactions == [] + + +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + + monkeypatch.setattr( + guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False + ) + monkeypatch.setattr( + tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False + ) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + written: list[str] = [] + + async def _write(*args: Any, **kwargs: Any) -> None: + written.extend(row["request_id"] for row in kwargs["data"]) + if len(written) == 1: + mock_prisma_client.spend_log_transactions.append( + make_spend_log_row(request_id="r2") + ) + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write) + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert written == ["r1", "r2"] + assert mock_prisma_client.spend_log_transactions == [] + + +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + + monkeypatch.setattr( + guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False + ) + monkeypatch.setattr( + tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False + ) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + write_started = asyncio.Event() + written: list[str] = [] + write_calls = {"n": 0} + + async def _write(*args: Any, **kwargs: Any) -> None: + write_calls["n"] += 1 + if write_calls["n"] == 1: + write_started.set() + await asyncio.Event().wait() + written.extend(row["request_id"] for row in kwargs["data"]) + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write) + + async def _monitor() -> None: + await update_spend_logs_job( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + mock_prisma_client.spend_logs_queue_monitor_task = asyncio.create_task(_monitor()) + await write_started.wait() + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert written == ["r1"] + assert mock_prisma_client.spend_log_transactions == [] + assert mock_prisma_client.spend_logs_queue_monitor_task is None + + +@pytest.mark.asyncio +async def test_drain_spend_logs_queue_gives_up_after_max_passes( + mock_prisma_client: Any, make_spend_log_row: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.db.spend_log_tool_index as tool_mod + import litellm.proxy.guardrails.usage_tracking as guard_mod + + monkeypatch.setattr( + guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False + ) + monkeypatch.setattr( + tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False + ) + + proxy_logging = MagicMock() + proxy_logging.failure_handler = AsyncMock() + mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")] + + async def _write_and_refill(*args: Any, **kwargs: Any) -> None: + mock_prisma_client.spend_log_transactions.append(make_spend_log_row()) + + mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock( + side_effect=_write_and_refill + ) + + await drain_spend_logs_queue( + prisma_client=mock_prisma_client, + db_writer_client=None, + proxy_logging_obj=proxy_logging, + ) + + assert ( + mock_prisma_client.db.litellm_spendlogs.create_many.await_count + == MAX_SPEND_LOG_DRAIN_ITERATIONS + ) + + @pytest.mark.asyncio async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty( mock_prisma_client: Any, From de4765ae8863ae4e9b4a00d6bb7d2143558fe2cf Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 13 Aug 2026 03:20:15 +0000 Subject: [PATCH 23/25] docs(claude): drop the @ prefix from the PR template path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index a3c24b84ea8..85ba96980b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,7 +29,7 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions -When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule +When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively From d86336a7c6f5c97b5fb46413a8a1c9d77f426220 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 12 Aug 2026 20:39:58 -0700 Subject: [PATCH 24/25] fix(langfuse): emit otel trace version and release on the keys langfuse v4 reads (#36702) * fix(langfuse): emit otel trace version and release on the keys langfuse v4 reads The langfuse_otel exporter wrote version to langfuse.generation.version and langfuse.trace.version, and release to langfuse.trace.release. Langfuse v4 recognizes neither, so both landed in the generic span attribute bag and every trace reported version and release as null. v4 has a single langfuse.version key, lifted to the trace when it sits on the root span, plus langfuse.release. Also routes the otel v2 preset's per-request headers through the shared builder so key-scoped and team-scoped exports carry x-langfuse-ingestion-version like the other three exporter paths already do. * fix(langfuse): give trace_version precedence over version on the shared v4 key Matches the documented contract in docs/observability/langfuse_integration.md and the legacy langfuse SDK callback, which both treat trace_version as the authoritative trace version with version as its fallback. --- .../integrations/langfuse/langfuse_otel.py | 10 ++-- litellm/integrations/otel/presets/langfuse.py | 8 ++- litellm/types/integrations/langfuse_otel.py | 5 +- .../integrations/otel/test_otel_v2_dynamic.py | 12 +++++ .../integrations/test_langfuse_otel.py | 51 +++++++++++++++++-- 5 files changed, 72 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index 7de42c00ede..a93c45ef840 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -90,7 +90,6 @@ class LangfuseOtelLogger(OpenTelemetry): "generation_name": LangfuseSpanAttributes.GENERATION_NAME, "generation_id": LangfuseSpanAttributes.GENERATION_ID, "parent_observation_id": LangfuseSpanAttributes.PARENT_OBSERVATION_ID, - "version": LangfuseSpanAttributes.GENERATION_VERSION, "mask_input": LangfuseSpanAttributes.MASK_INPUT, "mask_output": LangfuseSpanAttributes.MASK_OUTPUT, "trace_user_id": LangfuseSpanAttributes.TRACE_USER_ID, @@ -99,13 +98,18 @@ class LangfuseOtelLogger(OpenTelemetry): "trace_name": LangfuseSpanAttributes.TRACE_NAME, "trace_id": LangfuseSpanAttributes.TRACE_ID, "trace_metadata": LangfuseSpanAttributes.TRACE_METADATA, - "trace_version": LangfuseSpanAttributes.TRACE_VERSION, - "trace_release": LangfuseSpanAttributes.TRACE_RELEASE, + "trace_release": LangfuseSpanAttributes.RELEASE, "existing_trace_id": LangfuseSpanAttributes.EXISTING_TRACE_ID, "update_trace_keys": LangfuseSpanAttributes.UPDATE_TRACE_KEYS, "debug_langfuse": LangfuseSpanAttributes.DEBUG_LANGFUSE, } + version: Final = ( + metadata.get("trace_version") if metadata.get("trace_version") is not None else metadata.get("version") + ) + if version is not None: + safe_set_attribute(span, LangfuseSpanAttributes.VERSION.value, version) + for key, enum_attr in mapping.items(): if key in metadata and metadata[key] is not None: value = metadata[key] diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index 5104ee2ff55..c2f64422eff 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -42,9 +42,7 @@ def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, public_key: Final = params.get("langfuse_public_key") secret_key: Final = params.get("langfuse_secret_key") if public_key and secret_key: - return { - "Authorization": _V1Langfuse._get_langfuse_authorization_header( - public_key=public_key, secret_key=secret_key - ) - } + return _V1Langfuse._build_langfuse_otel_headers( + _V1Langfuse._get_langfuse_authorization_header(public_key=public_key, secret_key=secret_key) + ) return {} diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index 9ef48bdcdd0..c58dc567cda 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -16,12 +16,13 @@ class LangfuseOtelConfig(BaseModel): class LangfuseSpanAttributes(str, Enum): LANGFUSE_ENVIRONMENT = "langfuse.environment" + VERSION = "langfuse.version" + RELEASE = "langfuse.release" # ---- Generation-level metadata ---- GENERATION_NAME = "langfuse.generation.name" GENERATION_ID = "langfuse.generation.id" PARENT_OBSERVATION_ID = "langfuse.generation.parent_observation_id" - GENERATION_VERSION = "langfuse.generation.version" MASK_INPUT = "langfuse.generation.mask_input" MASK_OUTPUT = "langfuse.generation.mask_output" @@ -36,8 +37,6 @@ class LangfuseSpanAttributes(str, Enum): TRACE_NAME = "langfuse.trace.name" TRACE_ID = "langfuse.trace.id" TRACE_METADATA = "langfuse.trace.metadata" - TRACE_VERSION = "langfuse.trace.version" - TRACE_RELEASE = "langfuse.trace.release" EXISTING_TRACE_ID = "langfuse.trace.existing_id" UPDATE_TRACE_KEYS = "langfuse.trace.update_keys" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py index f7c0b5452fe..e44c56e1fdf 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_dynamic.py @@ -1,5 +1,6 @@ """Per-request multi-tenant credential routing (V1 parity).""" +import base64 import os import sys @@ -42,6 +43,17 @@ def test_langfuse_dynamic_headers_need_both_keys(): assert headers is not None and "Authorization" in headers +def test_langfuse_dynamic_headers_carry_v4_ingestion_version(): + headers = dynamic_otlp_headers( + "langfuse_otel", {"langfuse_public_key": "pk", "langfuse_secret_key": "sk"} + ) + expected_auth = "Basic " + base64.b64encode(b"pk:sk").decode() + assert headers == { + "Authorization": expected_auth, + "x-langfuse-ingestion-version": "4", + } + + def test_weave_dynamic_headers(): headers = dynamic_otlp_headers( "weave_otel", {"wandb_api_key": "w", "weave_project_id": "p"} diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 28f138c7acd..9392f974570 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -211,7 +211,7 @@ class TestLangfuseOtelIntegration: LangfuseSpanAttributes.GENERATION_NAME.value: "gen-name", LangfuseSpanAttributes.GENERATION_ID.value: "gen-id", LangfuseSpanAttributes.PARENT_OBSERVATION_ID.value: "parent-id", - LangfuseSpanAttributes.GENERATION_VERSION.value: "v1", + LangfuseSpanAttributes.VERSION.value: "t-ver", LangfuseSpanAttributes.MASK_INPUT.value: True, LangfuseSpanAttributes.MASK_OUTPUT.value: False, LangfuseSpanAttributes.TRACE_USER_ID.value: "user-123", @@ -221,8 +221,7 @@ class TestLangfuseOtelIntegration: LangfuseSpanAttributes.TRACE_NAME.value: "trace-name", LangfuseSpanAttributes.TRACE_ID.value: "traceid", # stripped dashes LangfuseSpanAttributes.TRACE_METADATA.value: json.dumps({"k": "v"}), - LangfuseSpanAttributes.TRACE_VERSION.value: "t-ver", - LangfuseSpanAttributes.TRACE_RELEASE.value: "rel-1", + LangfuseSpanAttributes.RELEASE.value: "rel-1", LangfuseSpanAttributes.EXISTING_TRACE_ID.value: "existing-id", LangfuseSpanAttributes.UPDATE_TRACE_KEYS.value: json.dumps( ["key1", "key2"] @@ -240,6 +239,52 @@ class TestLangfuseOtelIntegration: actual == expected ), "Mismatch between expected and actual OTEL attribute mapping." + @pytest.mark.parametrize( + "metadata, expected_version", + [ + ( + {"version": "v-observation", "trace_version": "v-trace"}, + "v-trace", + ), + ({"trace_version": "v-trace"}, "v-trace"), + ({"version": "v-observation"}, "v-observation"), + ({"version": "v-observation", "trace_version": ""}, ""), + ({}, None), + ], + ids=[ + "trace-version-wins-as-documented", + "trace-only", + "observation-version-is-the-fallback", + "empty-trace-version-is-not-absent", + "neither-key-emits-nothing", + ], + ) + def test_version_emitted_on_langfuse_v4_key(self, metadata, expected_version): + kwargs = {"litellm_params": {"metadata": {"trace_release": "rel-9", **metadata}}} + + with patch( + "litellm.integrations.arize._utils.safe_set_attribute" + ) as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes( + MagicMock(), kwargs, None + ) + + emitted = { + call.args[1]: call.args[2] for call in mock_safe_set_attribute.call_args_list + } + + if expected_version is None: + assert "langfuse.version" not in emitted + else: + assert emitted["langfuse.version"] == expected_version + assert emitted["langfuse.release"] == "rel-9" + for retired_key in ( + "langfuse.generation.version", + "langfuse.trace.version", + "langfuse.trace.release", + ): + assert retired_key not in emitted + def test_set_langfuse_specific_attributes_with_content(self): """Test that _set_langfuse_specific_attributes correctly sets observation.output with regular content response.""" from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes From 73e555a3e3a761db51ca51d813b418516e1721aa Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:32:57 -0700 Subject: [PATCH 25/25] test(interactions): follow Google spec drift replacing Turn with typed steps (#36730) * test(interactions): follow Google spec drift replacing Turn with typed steps * test(interactions): send step and content-list input to the live Gemini API --- .../test_google_interactions_integration.py | 21 ++++------ .../interactions/test_openapi_compliance.py | 40 ++++++++++++++----- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/test_litellm/interactions/test_google_interactions_integration.py index 9c651cc94f5..41f0fa0d7fb 100644 --- a/tests/test_litellm/interactions/test_google_interactions_integration.py +++ b/tests/test_litellm/interactions/test_google_interactions_integration.py @@ -55,17 +55,10 @@ class TestGoogleInteractionsCreate: print(f"Usage: {response.usage}") def test_create_with_content_list(self, api_key): - """Test creating an interaction with a structured content list (Turn format).""" + """Test creating an interaction with a structured content list (Content[] input).""" response = interactions.create( model="gemini/gemini-2.5-flash", - input=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is the capital of France?"} - ], - } - ], + input=[{"type": "text", "text": "What is the capital of France?"}], api_key=api_key, ) @@ -169,25 +162,25 @@ class TestGoogleInteractionsStreaming: class TestGoogleInteractionsMultiTurn: - """Tests for multi-turn conversations using Turn[] input.""" + """Tests for multi-turn conversations using Step[] input.""" def test_multi_turn_conversation(self, api_key): - """Test a multi-turn conversation per OpenAPI spec (Turn[] format).""" + """Test a multi-turn conversation per OpenAPI spec (Step[] format).""" response = interactions.create( model="gemini/gemini-2.5-flash", input=[ { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "My name is Alice."}], }, { - "role": "model", + "type": "model_output", "content": [ {"type": "text", "text": "Hello Alice! Nice to meet you."} ], }, { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "What is my name?"}], }, ], diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 1fe343ca6ee..2665f8703a6 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -167,17 +167,39 @@ class TestRequestCompliance: assert text_schema["properties"]["type"].get("const") == "text" print("✓ TextContent schema is correct") - def test_turn_schema(self, spec_dict): - """Verify Turn schema for multi-turn conversations.""" - turn_schema = spec_dict["components"]["schemas"]["Turn"] + def test_step_schema(self, spec_dict): + """Verify step-based multi-turn input. - assert "role" in turn_schema["properties"] - assert "content" in turn_schema["properties"] + Google replaced the role-carrying `Turn` schema with typed steps + (spec update of Aug 13, 2026): conversation history is now a `Step[]` + where `UserInputStep`/`ModelOutputStep` pin `type` values that our + transformations read to recover the role. Assert exactly what our code + depends on: `InteractionsInput` accepts a Step array, both step kinds + are part of the `Step` union, each pins its `type` const, and each + carries a `Content[]` content field. + """ + input_schema = spec_dict["components"]["schemas"]["InteractionsInput"] + step_array_items = [ + option["items"]["$ref"].split("/")[-1] + for option in input_schema["oneOf"] + if option.get("type") == "array" and "$ref" in option.get("items", {}) + ] + assert "Step" in step_array_items, f"InteractionsInput should accept Step[], got arrays of {step_array_items}" - # Content can be string or Content[] - content_prop = turn_schema["properties"]["content"] - assert "oneOf" in content_prop - print("✓ Turn schema supports role + content") + step_variants = { + option["$ref"].split("/")[-1] + for option in spec_dict["components"]["schemas"]["Step"]["oneOf"] + if "$ref" in option + } + assert {"UserInputStep", "ModelOutputStep"} <= step_variants, f"Step union is missing role steps: {step_variants}" + + for step_name, type_value in [("UserInputStep", "user_input"), ("ModelOutputStep", "model_output")]: + step_schema = spec_dict["components"]["schemas"][step_name] + assert step_schema["properties"]["type"].get("const") == type_value + assert "type" in step_schema["required"] + content_items = step_schema["properties"]["content"]["items"] + assert content_items["$ref"].split("/")[-1] == "Content" + print(f"✓ {step_name} pins type '{type_value}' with Content[] content") class TestResponseCompliance: