scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_mcp_oauth_custom_token_header

Browse source
This commit is contained in:
Devin AI 2026-08-20 17:46:12 +00:00
commit 96eea17bf1
149 changed files with 10245 additions and 4836 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.circleci/config.yml
View file

@ -1025,7 +1025,7 @@ jobs:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

32
.github/ci-coverage-allowlist.yml vendored
View file

@ -63,30 +63,6 @@ test_paths:
- tests/litellm/test_router_retry_backoff_headers.py
- tests/litellm/test_sambanova_model_metadata.py
- tests/litellm/test_stream_chunk_builder_images.py
- reason: >-
Legacy proxy suite superseded by the proxy shards; no job invokes it and whether it still
describes supported behaviour is unresolved
paths:
- tests/old_proxy_tests/tests/test_anthropic_context_caching.py
- tests/old_proxy_tests/tests/test_anthropic_sdk.py
- tests/old_proxy_tests/tests/test_async.py
- tests/old_proxy_tests/tests/test_gemini_context_caching.py
- tests/old_proxy_tests/tests/test_langchain_embedding.py
- tests/old_proxy_tests/tests/test_langchain_request.py
- tests/old_proxy_tests/tests/test_llamaindex.py
- tests/old_proxy_tests/tests/test_mistral_sdk.py
- tests/old_proxy_tests/tests/test_openai_embedding.py
- tests/old_proxy_tests/tests/test_openai_exception_request.py
- tests/old_proxy_tests/tests/test_openai_request.py
- tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
- tests/old_proxy_tests/tests/test_openai_simple_embedding.py
- tests/old_proxy_tests/tests/test_openai_tts_request.py
- tests/old_proxy_tests/tests/test_pass_through_langfuse.py
- tests/old_proxy_tests/tests/test_q.py
- tests/old_proxy_tests/tests/test_simple_traceparent_openai.py
- tests/old_proxy_tests/tests/test_vertex_sdk_forward_headers.py
- tests/old_proxy_tests/tests/test_vtx_embedding.py
- tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py
- reason: >-
No job invokes this suite and its files mix pure transformation tests with ones driving live
vendor vector stores, so assigning them needs a per-file decision
@ -116,6 +92,14 @@ test_paths:
- tests/load_tests/test_otel_load_test.py
- tests/load_tests/test_vertex_embeddings_load_test.py
- tests/load_tests/test_vertex_load_tests.py
- reason: >-
A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a
pull request job. Until 2026-08-20 the CircleCI agent job hid them behind a grep -v
that this census could not see; the glob now excludes them structurally and this entry
is the decision on the record. Revisit when the A2A bridge gets a recorded-wire fixture
paths:
- tests/agent_tests/local_only_agent_tests
- reason: >-
Third-party integration tests that skip themselves without OCI configuration or sandbox
credentials, neither of which a pull request job holds

85
.github/scripts/assert_ci_coverage.py vendored
View file

@ -25,6 +25,15 @@ DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
# itself decomposed one level deeper and is checked through its own entry.
SHARDED_ROOTS: tuple[str, ...] = (
"tests/proxy_unit_tests",
"tests/test_litellm",
"tests/test_litellm/proxy",
)
@dataclass(frozen=True, slots=True)
class AllowEntry:
@ -107,20 +116,32 @@ def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
)
def _glob_to_regex(token: str) -> re.Pattern[str]:
def _glob_to_regex(token: str, *, subtree: bool) -> re.Pattern[str]:
parts = re.split(r"(\*\*/|\*\*|\*|\?)", token)
translated = "".join(
{"**/": r"(?:.*/)?", "**": r".*", "*": r"[^/]*", "?": r"[^/]"}.get(part, re.escape(part)) for part in parts
)
return re.compile(rf"{translated}(?:/.*)?$")
return re.compile(rf"{translated}(?:/.*)?$" if subtree else rf"{translated}$")
def _token_covers(token: str, relative_path: str) -> bool:
if GLOB_CHARS & set(token):
return _glob_to_regex(token).match(relative_path) is not None
return _glob_to_regex(token, subtree=True).match(relative_path) is not None
return relative_path == token or relative_path.startswith(f"{token}/")
def _token_names(token: str, relative_path: str) -> bool:
"""Whether the token names this path itself, rather than merely containing it.
A sharded tree has no catch-all bucket, so the ancestor token the census is happy
with (`tests/x` standing in for everything below it) is exactly what would let a
newly added child ride along without a shard.
"""
if GLOB_CHARS & set(token):
return _glob_to_regex(token, subtree=False).match(relative_path) is not None
return token == relative_path
def _test_files() -> tuple[str, ...]:
return tuple(
sorted(
@ -166,6 +187,45 @@ def _describe(paths: tuple[str, ...]) -> str:
return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}"
def _holds_tests(directory: pathlib.Path) -> bool:
return any(directory.rglob("test_*.py"))
def _shard_children(root: str, repo_root: pathlib.Path = REPO_ROOT) -> tuple[str, ...]:
"""Children of a sharded root that carry tests, so each one needs its own shard.
A directory earns an entry by containing a test file rather than by being named
`test_*`, which is what keeps fixture directories (`test_configs`, `expected_*`)
out without a hand-maintained list of exceptions.
"""
return tuple(
sorted(
child.relative_to(repo_root).as_posix()
for child in (repo_root / root).iterdir()
if not child.name.startswith(".")
and (
_holds_tests(child)
if child.is_dir()
else child.name.startswith("test_") and child.suffix == ".py"
)
)
)
def _unassigned_shard_children(
tokens: frozenset[str],
roots: tuple[str, ...] = SHARDED_ROOTS,
repo_root: pathlib.Path = REPO_ROOT,
) -> tuple[Finding, ...]:
return tuple(
Finding(subject=child, detail=f"holds tests but no shard of {root} names it")
for root in roots
if (repo_root / root).is_dir()
for child in _shard_children(root, repo_root)
if child not in roots and not any(_token_names(token, child) for token in tokens)
)
def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
return tuple(
Finding(subject=relative_path, detail="built by no job")
@ -229,7 +289,26 @@ def _report(title: str, findings: tuple[Finding, ...], remedy: str) -> None:
_write("")
def _check_shards() -> int:
findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
if findings:
_report(
"test directories and files that no shard claims",
findings,
"Add each to the shard it belongs to. A directory that is itself split across "
"several shards belongs in SHARDED_ROOTS instead, so its own children get checked.",
)
return 1
counted = sum(len(_shard_children(root)) for root in SHARDED_ROOTS if (REPO_ROOT / root).is_dir())
_write(f"OK: all {counted} test children across {len(SHARDED_ROOTS)} sharded trees are assigned to a shard.")
return 0
def main() -> int:
if "--shards" in sys.argv[1:]:
return _check_shards()
allowlist = _load_allowlist()
scalars = _all_scalars()

0
.github/workflows/auto_update_price_and_context_window_file.py → .github/scripts/auto_update_price_and_context_window_file.py vendored
View file

0
.github/workflows/run_llm_translation_tests.py → .github/scripts/run_llm_translation_tests.py vendored
View file

557
.github/scripts/triage_rollout_heads_up.py vendored
View file

@ -1,557 +0,0 @@
#!/usr/bin/env python3
"""One-shot 7-day heads-up sweep for the Agent Shin rollout.
Posts a friendly "the OSS triage bot kicks in next Monday" comment on every
open external PR/issue that currently *would* fail the new rubric — i.e.,
every PR/issue Agent Shin would close once the rollout completes. The point
is to give contributors a full week to fix their description before the bot
ever takes a destructive action, so nobody is surprised by an auto-close.
The script is designed to run **exactly once** at rollout, fired by a manual
``workflow_dispatch`` (``dry_run=false``) on the heads-up workflow. Re-runs
are safe: every comment is stamped with the hidden ``HEADS_UP_MARKER`` and
PRs/issues that already carry the marker are skipped.
Dry-run vs. real run
--------------------
Defaults to dry-run. Passing ``--close`` flips into real mode. Every GitHub
mutation goes through ``_agent_shin_actions``, which has a one-line
``if dry_run: log else: do_it`` per call, so the only difference between a
dry-run preview and the real run is the call site that actually hits the
GitHub API.
Local preview::
python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm
Real run (the manual rollout dispatch uses this)::
python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm --close
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import sys
from pathlib import Path
from typing import Any
# Make the sibling triage_with_llm + _agent_shin_actions importable when this
# script is invoked directly (the GitHub workflow does `python3 .github/scripts/...`).
_SCRIPTS_DIR = Path(__file__).resolve().parent
if str(_SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(_SCRIPTS_DIR))
from _agent_shin_actions import maybe_post_comment # noqa: E402
from agent_shin_shared import ( # noqa: E402
AGENT_SHIN_DEFAULT_BOT_LOGIN,
ALLOWLIST_LOGINS,
list_open_items,
)
from triage_with_llm import ( # noqa: E402
DEFAULT_MODEL,
call_llm_judge,
fetch_issue,
fetch_pr,
gh,
is_internal_contributor,
review_gate,
triage,
)
# Hidden marker so re-runs skip PRs/issues we've already notified. Distinct from
# the within-grace / ready / regressed markers so it can't be confused with the
# steady-state lifecycle comments.
HEADS_UP_MARKER = "<!-- agent-shin:rollout-heads-up -->"
# Placeholder until the litellm-docs PR ships. The rollout blog post explains
# the new rubric, the 7-day grace, and how to recover after an auto-close.
# TODO(docs): replace with the canonical URL once the litellm-docs PR merges.
ROLLOUT_BLOG_URL = "https://docs.litellm.ai/docs/agent_shin_triage_rollout"
# Default cutoff is one week from "now". Computed at runtime so the wording
# stays correct even if the rollout is merged later than planned. The user can
# override with --close-on YYYY-MM-DD when running the script manually.
DEFAULT_GRACE_DAYS = 7
# The daily auto-close sweeps (close_low_quality_prs.yml at 09:00 UTC and
# review_gate.yml at 09:30 UTC) are what actually close a still-failing item,
# so the deadline we promise contributors has to name that wall-clock moment.
ACTIVATION_TIME_UTC = "09:00 UTC"
def _format_cutoff(cutoff: dt.date) -> str:
"""Human-readable, timezone-explicit cutoff, e.g. ``Monday, June 1, 2026
(09:00 UTC)`` — the moment a still-failing PR/issue gets closed."""
return (
f"{cutoff.strftime('%A, %B')} {cutoff.day}, {cutoff.year} "
f"({ACTIVATION_TIME_UTC})"
)
def _rubric_section_pr() -> str:
return (
"**Going forward, every external PR needs ONE of:**\n"
"\n"
"- A linked GitHub issue using a closing keyword: "
"`Fixes #1234`, `Closes #1234`, or `Resolves #1234`, OR\n"
"- All three of: a clear **problem description**, **expected vs. "
"actual behavior**, and **end-to-end QA proof** (at least one of a "
"short screen recording / video, before/after screenshots, or the "
"exact commands you ran with their real output; mocked or stubbed "
"runs don't count).\n"
"\n"
"PRs also need a **Greptile confidence score of 4/5 or higher** before "
"the bot will tag them `ready for review`. You can `@greptileai` to "
"request a fresh review at any time, including after the PR is closed."
)
def _rubric_section_issue() -> str:
return (
"**Going forward, every external issue needs:**\n"
"\n"
"- For **bug reports**: end-to-end evidence of the bug (at least one "
"of a screen recording / video, a screenshot, or the exact commands "
"you ran with their real output / traceback) plus expected vs. actual "
"behavior. Written steps with no run output don't count, and mocked "
"or stubbed runs don't count.\n"
"- For **feature requests**: a clear description of the proposed "
"feature plus a use case + concrete example (config, API call, UI "
"flow, or scenario showing what's blocked today)."
)
def _description_only_note(kind: str) -> str:
noun = "PR" if kind == "pr" else "issue"
return (
f"⚠️ **The requirements must live in the {noun} *description*, not in "
"comments.** Some PRs/issues collect 100+ comments from humans and "
"bots; reading the entire thread on every triage run would balloon "
"GitHub API usage (we'd start getting 429'd) and blow out the LLM "
"judge's context. The bot only reads the description, so anything "
"you add as a comment will be invisible to it."
)
def _missing_section(verdict: dict, greptile_score: int | None) -> str:
"""Bullet list of what's currently missing on this PR/issue.
Combines the LLM judge's `missing` list (rubric items) with a Greptile
shortfall (for PRs) so the contributor sees one list of things to fix.
"""
missing = list(verdict.get("missing") or [])
if greptile_score is not None and greptile_score < 4:
missing.insert(
0,
f"Greptile's most recent review scored this PR {greptile_score}/5 "
"(below the 4/5 bar Agent Shin will require).",
)
if not missing:
return (
"_The bot couldn't articulate a specific missing piece; see the "
"rubric link above and double-check the description includes all "
"of it before the rollout._"
)
bullets = "\n".join(f"- {m}" for m in missing)
return f"**What this one is currently missing:**\n\n{bullets}"
def _recovery_section(kind: str) -> str:
if kind == "pr":
return (
"**If the bot closes this PR after the rollout:** update the "
"description with the missing pieces, then either open a fresh "
"PR or comment `@agent-shin reconsider` on the closed PR. If "
"Greptile re-scores you at 4/5 or higher I'll reopen and tag "
"the PR `ready for review`. (`@greptileai` works on closed PRs "
"too; a fresh review is one of the signals that lifts you back "
"into the queue.) This is **not** us losing interest in your "
"change; far from it. We just need open PRs to be a list of "
"things a maintainer can act on, so we can get to yours faster."
)
return (
"**If the bot closes this issue after the rollout:** edit the issue "
"description to add the missing pieces, then comment `@agent-shin "
"reconsider` on the closed issue. I'll re-evaluate and, if the rubric "
"is met, reopen it. (GitHub doesn't let external authors reopen an "
"issue a maintainer or bot closed, so the comment is the reliable "
"path.) This is **not** us saying the bug isn't real or the request "
"isn't useful; it's so the remaining open issues are a list of things "
"a maintainer can act on."
)
def format_heads_up_comment(
*, kind: str, verdict: dict, greptile_score: int | None, cutoff: dt.date
) -> str:
"""Compose the friendly 7-day heads-up comment posted on a failing PR/issue."""
noun = "PR" if kind == "pr" else "issue"
rubric = _rubric_section_pr() if kind == "pr" else _rubric_section_issue()
cutoff_str = _format_cutoff(cutoff)
explanation = (verdict.get("explanation") or "").strip()
explanation_block = (
f"> _(The judge's note for this one: {explanation})_\n\n" if explanation else ""
)
return (
"🚅 **Heads-up: we're turning on the OSS triage bot in "
f"{DEFAULT_GRACE_DAYS} days, on {cutoff_str}.**\n"
"\n"
"We're rolling out **Agent Shin**, an LLM-as-judge triage bot for "
f"external {noun}s. Once it's live, the bot reads each open "
f"{noun}'s description, scores it against a small rubric, and "
f"auto-closes any {noun} that's missing the basics, with a single "
f"comment explaining what's missing and how to recover. Full "
f"context: [Agent Shin rollout blog post]({ROLLOUT_BLOG_URL}).\n"
"\n"
f"{rubric}\n"
"\n"
f"{_description_only_note(kind)}\n"
"\n"
f"{_missing_section(verdict, greptile_score)}\n"
"\n"
f"{explanation_block}"
"**Timeline (you have a week):**\n"
"\n"
f"- We turn the bot on in {DEFAULT_GRACE_DAYS} days, on "
f"**{cutoff_str}**. You have until then to update this {noun}'s "
"description with the missing pieces above.\n"
f"- If this {noun} still fails the rubric at **{cutoff_str}**, "
"we'll close it.\n"
f"- From then on the bot runs daily, and every {noun} that fails "
"the rubric gets a **2-hour lifetime**: one warning comment, then "
"auto-close 2 hours later.\n"
"\n"
f"{_recovery_section(kind)}\n"
"\n"
f"{HEADS_UP_MARKER}"
)
def _list_open_numbers(repo: str, kind: str) -> list[int]:
"""Return every open PR or issue number in ``repo``.
Delegates to ``list_open_items`` so the full backlog is fetched (no cap)
and the `gh {pr,issue} list` invocation stays in one shared place. ``gh
issue list`` would include PRs, but ``list_open_items`` uses the dedicated
command per kind, so the two never mix.
"""
return [
item["number"] for item in list_open_items(kind, repo=repo, fields="number")
]
def _has_heads_up_marker(item: dict) -> bool:
"""Cheap fast-path: check the PR/issue body itself for the marker.
The marker is appended to the *comment* we post, not the body, so this
will only fire if the body literally contains the marker text. We still
do the comment-marker check separately below; this body check just lets
us short-circuit for PRs/issues that quote the marker for any reason.
"""
body = item.get("body") or ""
return HEADS_UP_MARKER in body
def _comments_have_marker(repo: str, number: int) -> bool:
"""True if the bot already posted a comment carrying the marker.
Used for idempotency: a re-run skips items the previous run notified.
Filters by author (matching the sibling marker-checks in
``triage_with_llm._has_marker`` and
``agent_shin_shared.seconds_since_latest_marker_comment``) so a
contributor who quotes the heads-up via GitHub's "Quote reply" — which
preserves HTML comments in the raw markdown — can't trick the
idempotency check into silently skipping a real heads-up.
Comments live on the unified issues endpoint regardless of whether the
item is a PR or an issue, so no ``kind`` argument is required here.
"""
expected_login = (
os.environ.get("AGENT_SHIN_BOT_LOGIN") or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
raw = gh(
"api",
"--paginate",
f"repos/{repo}/issues/{number}/comments?per_page=100",
)
for line in raw.splitlines():
line = line.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
comments = payload if isinstance(payload, list) else [payload]
for comment in comments:
author = ((comment.get("user") or {}).get("login") or "").lower()
if author != expected_login:
continue
if HEADS_UP_MARKER in (comment.get("body") or ""):
return True
return False
def _evaluate_pr(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
"""Run the future PR rubric (review_gate) in dry-run and return the result."""
return review_gate(
repo=repo,
number=number,
close=False, # we only want the verdict, never act here
model=model,
judge=judge,
)
def _evaluate_issue(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
"""Run the future issue rubric (triage kind='issue') in dry-run."""
return triage(
repo=repo,
kind="issue",
number=number,
close=False,
model=model,
judge=judge,
)
def _would_be_closed(kind: str, result: dict) -> bool:
"""True if the future triage would auto-close this PR/issue based on the
rubric (regardless of grace-period gating).
For PRs we trust ``review_gate``'s ``passing`` field — it combines the LLM
verdict and the Greptile score. For issues we read the LLM verdict
directly. Both fields are ``None``/missing on skip paths
(skip-internal-author, skip-llm-error, etc.) where the future bot would
NOT close the item — those return False.
"""
if kind == "pr":
passing = result.get("passing")
if passing is None:
return False # skipped — nothing for the heads-up to warn about
return passing is False
verdict = result.get("verdict") or {}
return (verdict.get("verdict") or "").lower() == "fail"
def _process_one(
*,
repo: str,
kind: str,
number: int,
model: str,
cutoff: dt.date,
dry_run: bool,
judge: Any = None,
skip_marker_check: bool = False,
allowlist: frozenset[str] = ALLOWLIST_LOGINS,
) -> dict:
"""Evaluate one PR/issue and post a heads-up if it would be auto-closed.
Returns a per-item dict for the summary table.
"""
base = {"kind": kind, "number": number}
fetcher = fetch_pr if kind == "pr" else fetch_issue
item = fetcher(repo, number)
if (item.get("state") or "") != "open":
return {**base, "action": "skip-not-open"}
if allowlist:
login = (item.get("user") or {}).get("login") or ""
if login.lower() not in allowlist:
return {**base, "action": "skip-not-allowlisted"}
elif is_internal_contributor(item):
return {**base, "action": "skip-internal-author"}
if not skip_marker_check and _has_heads_up_marker(item):
return {**base, "action": "skip-already-marked-in-body"}
if not skip_marker_check and _comments_have_marker(repo, number):
return {**base, "action": "skip-already-notified"}
if kind == "pr":
result = _evaluate_pr(repo=repo, number=number, model=model, judge=judge)
else:
result = _evaluate_issue(repo=repo, number=number, model=model, judge=judge)
if not _would_be_closed(kind, result):
return {**base, "action": "skip-passing", "evaluator": result.get("action")}
verdict = result.get("verdict") or {}
greptile_score = result.get("greptile_score") if kind == "pr" else None
comment = format_heads_up_comment(
kind=kind, verdict=verdict, greptile_score=greptile_score, cutoff=cutoff
)
maybe_post_comment(repo, number, comment, dry_run=dry_run)
return {
**base,
"action": "heads-up-posted" if not dry_run else "would-post-heads-up",
"verdict": (verdict.get("verdict") or "").lower(),
"greptile_score": greptile_score,
}
def _print_summary(results: list[dict]) -> None:
"""Tally per-action counts so a dry-run preview tells you at a glance how
many comments the real run would post."""
counts: dict[str, int] = {}
for r in results:
counts[r["action"]] = counts.get(r["action"], 0) + 1
print("\n=== rollout heads-up summary ===")
for action in sorted(counts):
print(f" {action:35s} {counts[action]}")
print(f" total {len(results)}")
def run(
*,
repo: str,
close: bool,
cutoff: dt.date,
model: str,
kinds: tuple[str, ...] = ("pr", "issue"),
judge: Any = None,
only_numbers: dict[str, list[int]] | None = None,
skip_marker_check: bool = False,
) -> list[dict]:
"""Sweep ``repo`` and post heads-up comments. Returns the per-item results."""
dry_run = not close
if dry_run:
print(
f"[DRY RUN] sweeping {repo}; --close not passed, no comments will be posted."
)
else:
print(f"[REAL RUN] sweeping {repo}; comments WILL be posted.")
print(f"Cutoff date in comment body: {cutoff.isoformat()}")
results: list[dict] = []
for kind in kinds:
if only_numbers and kind in only_numbers:
numbers = list(only_numbers[kind])
else:
numbers = _list_open_numbers(repo, kind)
print(f"\n--- {kind}s: {len(numbers)} open ---")
for n in numbers:
try:
result = _process_one(
repo=repo,
kind=kind,
number=n,
model=model,
cutoff=cutoff,
dry_run=dry_run,
judge=judge,
skip_marker_check=skip_marker_check,
)
except (
Exception
) as exc: # noqa: BLE001 - per-item errors don't abort the sweep
result = {
"kind": kind,
"number": n,
"action": "error",
"error": str(exc),
}
print(f"!! {kind}#{n}: {exc}", file=sys.stderr)
print(f" {kind}#{n}: {result['action']}")
results.append(result)
_print_summary(results)
return results
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", required=True, help="owner/repo")
parser.add_argument(
"--close",
action="store_true",
help=(
"Actually post comments. Without this flag the script is in "
"dry-run mode and only logs what it would do."
),
)
parser.add_argument(
"--close-on",
type=dt.date.fromisoformat,
default=None,
help=(
"Cutoff date shown in the heads-up comment as the rollout date "
f"(default: today + {DEFAULT_GRACE_DAYS} days)."
),
)
parser.add_argument(
"--model",
default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL,
help=f"Model for the rubric LLM judge (default: {DEFAULT_MODEL}).",
)
parser.add_argument(
"--kind",
choices=("pr", "issue", "both"),
default="both",
help="Restrict the sweep to PRs or issues only (default: both).",
)
parser.add_argument(
"--only-pr",
type=int,
action="append",
default=[],
help="Limit the PR sweep to these PR numbers (repeat for several).",
)
parser.add_argument(
"--only-issue",
type=int,
action="append",
default=[],
help="Limit the issue sweep to these issue numbers (repeat for several).",
)
parser.add_argument(
"--ignore-existing-marker",
action="store_true",
help=(
"Re-post on PRs/issues that already carry the heads-up marker. "
"Useful for testing the comment wording on a known PR."
),
)
args = parser.parse_args()
cutoff = args.close_on or (
dt.datetime.now(dt.timezone.utc).date() + dt.timedelta(days=DEFAULT_GRACE_DAYS)
)
kinds: tuple[str, ...]
if args.kind == "pr":
kinds = ("pr",)
elif args.kind == "issue":
kinds = ("issue",)
else:
kinds = ("pr", "issue")
only: dict[str, list[int]] = {}
if args.only_pr:
only["pr"] = args.only_pr
if args.only_issue:
only["issue"] = args.only_issue
# The script must NOT hit the LLM in dry-run if no key is set — we still
# want a useful preview that says "skip-no-llm-key" for items that would
# have been judged. Production runs require OPENAI_API_KEY.
if args.close and not os.environ.get("OPENAI_API_KEY"):
parser.error("OPENAI_API_KEY must be set for --close (real-run) mode.")
run(
repo=args.repo,
close=args.close,
cutoff=cutoff,
model=args.model,
kinds=kinds,
only_numbers=only or None,
skip_marker_check=args.ignore_existing_marker,
)
return 0
if __name__ == "__main__":
sys.exit(main())

2
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -23,7 +23,7 @@ jobs:
version: "0.10.9"
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Regenerate JSON Schema
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py

5
.github/workflows/test-linting.yml vendored
View file

@ -132,6 +132,11 @@ jobs:
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, delta vs base)
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
- name: Print OpenAI version
if: steps.changes.outputs.decision != 'skip'
run: |

31
.github/workflows/test-unit-core-utils.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Core Utilities"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
core-utils:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
artifact-name: core-utils

35
.github/workflows/test-unit-enterprise-routing.yml vendored
View file

@ -1,35 +0,0 @@
name: "Unit Tests: Enterprise, Google GenAI & Routing"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
enterprise-routing:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/enterprise
tests/test_litellm/google_genai
tests/test_litellm/router_utils
tests/test_litellm/router_strategy
workers: 2
reruns: 2
artifact-name: enterprise-routing

31
.github/workflows/test-unit-integrations.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Integrations (Callbacks & Logging)"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integrations:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/integrations"
workers: 2
reruns: 3
artifact-name: integrations

47
.github/workflows/test-unit-llm-providers.yml vendored
View file

@ -1,47 +0,0 @@
name: "Unit Tests: LLM Provider Transformations"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
vertex-ai:
name: Vertex AI
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
artifact-name: llm-vertex-ai
other-providers:
name: All Other Providers
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
artifact-name: llm-other-providers

53
.github/workflows/test-unit-misc.yml vendored
View file

@ -1,53 +0,0 @@
name: "Unit Tests: MCP, Secrets, Containers & Misc"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
misc:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/batches
tests/test_litellm/secret_managers
tests/test_litellm/a2a_protocol
tests/test_litellm/anthropic_interface
tests/test_litellm/completion_extras
tests/test_litellm/compression
tests/test_litellm/containers
tests/test_litellm/experimental_mcp_client
tests/test_litellm/models
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/ocr
tests/test_litellm/passthrough
tests/test_litellm/rag
tests/test_litellm/realtime_api
tests/test_litellm/rerank_api
tests/test_litellm/sandbox
tests/test_litellm/test_router
tests/test_litellm/vector_stores
tests/test_litellm/videos
tests/test_litellm/test_*.py
workers: 2
reruns: 2
artifact-name: misc

31
.github/workflows/test-unit-proxy-auth.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Proxy Auth & Key Management"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-auth:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
workers: 2
reruns: 2
artifact-name: proxy-auth

36
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -42,11 +42,10 @@ concurrency:
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
jobs:
# Fast guard — fails the workflow if a test_*.py file under
# tests/proxy_unit_tests/ is not referenced by any matrix entry below.
# The semantic-shard design (no catch-all "remaining" bucket) relies on
# every test file being explicitly assigned; this guard prevents a new
# file from silently dropping out of CI.
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
@ -56,31 +55,8 @@ jobs:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test_*.py is in a matrix shard
run: |
python3 - <<'PY'
import pathlib, sys, yaml
wf = yaml.safe_load(open(".github/workflows/test-unit-proxy-db.yml"))
matrix = wf["jobs"]["proxy-db"]["strategy"]["matrix"]["include"]
referenced = set()
for entry in matrix:
for token in entry["test-path"].split():
if token.startswith("tests/proxy_unit_tests/"):
referenced.add(pathlib.PurePosixPath(token).name)
actual = {p.name for p in pathlib.Path("tests/proxy_unit_tests").iterdir()
if p.name.startswith("test_") and (p.suffix == ".py" or p.is_dir())
and p.name != "test_configs"}
orphans = sorted(actual - referenced)
if orphans:
print("ERROR: the following files/dirs under tests/proxy_unit_tests/")
print(" are not assigned to any shard in test-unit-proxy-db.yml:")
for o in orphans:
print(f" - {o}")
print()
print("Add each to whichever semantic shard it belongs to.")
sys.exit(1)
print(f"OK: all {len(actual)} files assigned to a shard.")
PY
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage

81
.github/workflows/test-unit-proxy-endpoints.yml vendored
View file

@ -1,81 +0,0 @@
name: "Unit Tests: Proxy API Endpoints"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-endpoints:
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails
tests/test_litellm/proxy/management_helpers
tests/test_litellm/proxy/anthropic_endpoints
tests/test_litellm/proxy/google_endpoints
tests/test_litellm/proxy/openai_files_endpoint
tests/test_litellm/proxy/batches_endpoints
tests/test_litellm/proxy/fine_tuning_endpoints
tests/test_litellm/proxy/vector_store_files_endpoints
tests/test_litellm/proxy/video_endpoints
tests/test_litellm/proxy/response_api_endpoints
tests/test_litellm/proxy/image_endpoints
tests/test_litellm/proxy/ocr_endpoints
tests/test_litellm/proxy/vector_store_endpoints
tests/test_litellm/proxy/agent_endpoints
tests/test_litellm/proxy/a2a
tests/test_litellm/proxy/credential_endpoints
tests/test_litellm/proxy/discovery_endpoints
tests/test_litellm/proxy/health_endpoints
tests/test_litellm/proxy/shutdown
tests/test_litellm/proxy/public_endpoints
tests/test_litellm/proxy/prompts
tests/test_litellm/proxy/rag_endpoints
tests/test_litellm/proxy/realtime_endpoints
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
reruns: 2
artifact-name: proxy-endpoints
# Behavior-pinning tests for litellm/proxy/proxy_server.py. Owns its
# own job (not a path on the proxy-endpoints job above) so its budget
# is independent and its coverage artifact is uploaded separately.
# See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc
proxy-server:
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: tests/test_litellm/proxy/proxy_server
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 95
artifact-name: proxy-server

42
.github/workflows/test-unit-proxy-infra.yml vendored
View file

@ -1,42 +0,0 @@
name: "Unit Tests: Proxy Infrastructure"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-infra:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/proxy/db
tests/test_litellm/proxy/middleware
tests/test_litellm/proxy/spend_tracking
tests/test_litellm/proxy/pass_through_endpoints
tests/test_litellm/proxy/_experimental
tests/test_litellm/proxy/experimental
tests/test_litellm/proxy/common_utils
tests/test_litellm/proxy/enterprise_billing
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 2
reruns: 2
artifact-name: proxy-infra

31
.github/workflows/test-unit-responses-caching-types.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Responses, Caching & Types"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
responses-caching-types:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
artifact-name: responses-caching-types

219
.github/workflows/test-unit.yml vendored Normal file
View file

@ -0,0 +1,219 @@
name: "Unit Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One caller for every tests/test_litellm shard, replacing the nine thin workflow
# files that each wrapped a single call to _test-unit-base.yml. Adding a shard is
# now one matrix entry rather than a new file.
#
# `name` is the shard id and nothing else, so each check reports as
# "<shard> / Run tests" exactly as it did when the shard had its own file. Those
# strings are the branch ruleset's required contexts, so they are load-bearing:
# renaming an entry renames a required check and the ruleset stops matching it.
#
# Every entry states its timeouts even when they equal the base workflow's
# defaults. An absent matrix key renders as an empty string, which is not a
# number, so a partially-specified entry would fail the call rather than fall
# back to the default.
#
# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is
# already a matrix and carries a shard-coverage guard that reads that file by
# name. Folding it in here is a follow-up, together with generalising that guard
# into assert_ci_coverage.py.
jobs:
unit:
name: ${{ matrix.shard }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
- shard: core-utils
artifact-name: core-utils
test-path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
timeout-minutes: 20
job-timeout-minutes: 55
- shard: enterprise-routing
artifact-name: enterprise-routing
test-path: >-
tests/test_litellm/enterprise
tests/test_litellm/google_genai
tests/test_litellm/router_utils
tests/test_litellm/router_strategy
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: integrations
artifact-name: integrations
test-path: "tests/test_litellm/integrations"
workers: 2
reruns: 3
timeout-minutes: 20
job-timeout-minutes: 55
- shard: Vertex AI
artifact-name: llm-vertex-ai
test-path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: All Other Providers
artifact-name: llm-other-providers
test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: misc
artifact-name: misc
test-path: >-
tests/test_litellm/batches
tests/test_litellm/secret_managers
tests/test_litellm/a2a_protocol
tests/test_litellm/anthropic_interface
tests/test_litellm/completion_extras
tests/test_litellm/compression
tests/test_litellm/containers
tests/test_litellm/experimental_mcp_client
tests/test_litellm/models
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/ocr
tests/test_litellm/passthrough
tests/test_litellm/rag
tests/test_litellm/realtime_api
tests/test_litellm/rerank_api
tests/test_litellm/sandbox
tests/test_litellm/test_router
tests/test_litellm/vector_stores
tests/test_litellm/videos
tests/test_litellm/test_*.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: proxy-auth
artifact-name: proxy-auth
test-path: >-
tests/test_litellm/proxy/auth
tests/test_litellm/proxy/hooks
tests/test_litellm/proxy/policy_engine
tests/test_litellm/proxy/client
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: proxy-endpoints
artifact-name: proxy-endpoints
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails
tests/test_litellm/proxy/management_helpers
tests/test_litellm/proxy/anthropic_endpoints
tests/test_litellm/proxy/google_endpoints
tests/test_litellm/proxy/openai_files_endpoint
tests/test_litellm/proxy/batches_endpoints
tests/test_litellm/proxy/fine_tuning_endpoints
tests/test_litellm/proxy/vector_store_files_endpoints
tests/test_litellm/proxy/video_endpoints
tests/test_litellm/proxy/response_api_endpoints
tests/test_litellm/proxy/image_endpoints
tests/test_litellm/proxy/ocr_endpoints
tests/test_litellm/proxy/vector_store_endpoints
tests/test_litellm/proxy/agent_endpoints
tests/test_litellm/proxy/a2a
tests/test_litellm/proxy/credential_endpoints
tests/test_litellm/proxy/discovery_endpoints
tests/test_litellm/proxy/health_endpoints
tests/test_litellm/proxy/shutdown
tests/test_litellm/proxy/public_endpoints
tests/test_litellm/proxy/prompts
tests/test_litellm/proxy/rag_endpoints
tests/test_litellm/proxy/realtime_endpoints
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: proxy-server
artifact-name: proxy-server
test-path: "tests/test_litellm/proxy/proxy_server"
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 95
- shard: proxy-infra
artifact-name: proxy-infra
test-path: >-
tests/test_litellm/proxy/db
tests/test_litellm/proxy/middleware
tests/test_litellm/proxy/spend_tracking
tests/test_litellm/proxy/pass_through_endpoints
tests/test_litellm/proxy/_experimental
tests/test_litellm/proxy/experimental
tests/test_litellm/proxy/common_utils
tests/test_litellm/proxy/enterprise_billing
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: responses-caching-types
artifact-name: responses-caching-types
test-path: >-
tests/test_litellm/responses
tests/test_litellm/caching
tests/test_litellm/types
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: ${{ matrix.reruns }}
timeout-minutes: ${{ matrix.timeout-minutes }}
job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
artifact-name: ${{ matrix.artifact-name }}

92
.github/workflows/triage_rollout_heads_up.yml vendored
View file

@ -1,92 +0,0 @@
name: Agent Shin — rollout heads-up (one-shot)
# Fires the 7-day heads-up comment on every open external PR/issue that the
# new triage bot would auto-close. The real sweep is a deliberate one-shot:
# trigger it at rollout via a manual `workflow_dispatch` with `dry_run=false`.
# The script is idempotent (skips items that already carry the
# `<!-- agent-shin:rollout-heads-up -->` marker), so a re-run is harmless.
#
# The automatic push trigger runs DRY-RUN only, so merging the script to
# `litellm_internal_staging` never posts a comment; it just confirms the
# workflow is wired up. Posting real comments requires the manual dispatch,
# which is also the only trigger that exposes `OPENAI_API_KEY`. The heads-up
# is intentionally NOT gated on `AGENT_SHIN_ENABLED`: it has to warn
# contributors while that flag is still off, ahead of the flip that turns on
# auto-closing.
#
# The workflow is a thin shell over `.github/scripts/triage_rollout_heads_up.py`.
# Dry-run vs. real run differ in EXACTLY one CLI flag (`--close`), added only
# on a manual dispatch with `dry_run=false`.
on:
push:
branches:
- litellm_internal_staging
paths:
# The presence of this script on staging IS the rollout merge marker.
# Editing the file later would re-fire the workflow; that's safe because
# the script skips PRs/issues that already have the heads-up marker.
- ".github/scripts/triage_rollout_heads_up.py"
workflow_dispatch:
inputs:
dry_run:
description: "Dry run (true = preview only, false = actually post comments)."
required: false
default: "true"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
heads-up:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage scripts
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run heads-up sweep
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only the manual dispatch (the real-run trigger) needs the LLM key.
# The automatic push trigger runs dry-run and never posts, so it gets
# no key. Mirrors the sibling triage workflows, which expose the key
# only on an enabled/dispatched run rather than unconditionally.
OPENAI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
# The real run is a deliberate manual dispatch with dry_run=false.
# Use the EXACT "false" comparison so any unexpected input value
# fail-closes to dry-run (mirrors the AGENT_SHIN_ENABLED pattern in
# the sibling workflows). The automatic push trigger always stays
# dry-run, so merging the script never posts.
DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}")
if [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${DRY_RUN_INPUT:-true}" = "false" ]; then
ARGS+=(--close)
echo "::notice::Manual rollout dispatch with dry_run=false -> heads-up comments WILL be posted."
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ]; then
echo "::notice::Manual dispatch in dry-run mode -> previewing only, no comments will be posted."
else
echo "::notice::Automatic push trigger -> dry-run preview only. Fire the real rollout sweep with a manual workflow_dispatch (dry_run=false)."
fi
python3 .github/scripts/triage_rollout_heads_up.py "${ARGS[@]}"

20
Makefile
View file

@ -7,6 +7,7 @@
info lint lint-inner lint-dev lint-checks format \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
lint-test-quality lint-test-quality-budget-update \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
lint-install lint-fetch-base bootstrap
@ -35,7 +36,8 @@ help:
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
@echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + basedpyright)"
@echo " make lint-test-quality - Gate the test suite against test-quality-budget.json"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -200,6 +202,11 @@ lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# litellm module-global mutation), counted across tests/ the same delta-vs-base way.
lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/test_quality_gate.py --base origin/litellm_internal_staging
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev lint-fetch-base
@ -221,8 +228,11 @@ lint-ruff-budget-update: install-dev lint-fetch-base
lint-type-discipline-budget-update: install-dev lint-fetch-base
$(UV_RUN) python scripts/type_discipline_gate.py --update
# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update
lint-test-quality-budget-update: install-dev lint-fetch-base
$(UV_RUN) python scripts/test_quality_gate.py --update
# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@ -244,7 +254,7 @@ lint:
lint-inner: lint-install lint-fetch-base
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety
@ -314,7 +324,7 @@ test-unit-helm: install-helm-unittest
# LLM Translation testing targets
test-llm-translation: install-test-deps
@echo "Running LLM translation tests..."
@python .github/workflows/run_llm_translation_tests.py
@python .github/scripts/run_llm_translation_tests.py
test-llm-translation-single: install-test-deps
@echo "Running single LLM translation test file..."

6
litellm/constants.py
View file

@ -243,6 +243,12 @@ AIOHTTP_NEEDS_CLEANUP_CLOSED: Final = (3, 13, 0) <= sys.version_info < (
# https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235
_max_size_env: Final = os.getenv("REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES")
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = int(_max_size_env) if _max_size_env is not None else None
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float(
os.getenv("REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", "20.0")
)
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
# SSL/TLS cipher configuration for faster handshakes
# Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones

300
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -10,17 +10,31 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
"""
import copy
import os
import re
from collections.abc import Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast
from urllib.parse import urlparse
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.prompt_templates.common_utils import (
with_prompt_cache_breakpoint,
)
from litellm.types.integrations.anthropic_cache_control_hook import (
CacheControlInjectionPoint,
CacheControlMessageInjectionPoint,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionCachedContent
from litellm.types.llms.anthropic import AnthropicSystemMessageContent
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionCachedContent,
ChatCompletionTextObject,
PromptCacheBreakpoint,
PromptCacheOptions,
)
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import StandardCallbackDynamicParams
@ -34,6 +48,55 @@ else:
# breakpoints: "A maximum of 4 blocks with cache_control may be provided."
MAX_CACHE_CONTROL_BLOCKS: Final = 4
CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
{"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"}
)
OPENAI_API_HOST: Final = "api.openai.com"
OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE")
def supports_openai_prompt_cache_breakpoint(model: str) -> bool:
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
if model_map_flag is not None:
return model_map_flag
version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower())
if version_match is None:
return False
version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
import litellm
entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
return next((bool(flag) for flag in flags if flag is not None), None)
def targets_openai_api(api_base: object) -> bool:
import litellm
resolved: Final = next(
(value for value in (api_base, litellm.api_base, *map(os.getenv, OPENAI_API_BASE_ENV_VARS)) if value),
None,
)
if not isinstance(resolved, str):
return True
host: Final = urlparse(resolved).hostname
return host is not None and (host == OPENAI_API_HOST or host.endswith(f".{OPENAI_API_HOST}"))
def _carries_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS)
def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
class AnthropicCacheControlHook(CustomPromptManagement):
def get_chat_completion_prompt(
@ -81,13 +144,32 @@ class AnthropicCacheControlHook(CustomPromptManagement):
# provider transform, where each tool_config point appends at most one
# cachePoint to the tools. That block also counts toward Anthropic's
# limit, so reserve a slot for it here to leave room.
reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
openai_dialect: Final = (
stamped_dialect
if isinstance(stamped_dialect, bool)
else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model,
non_default_params.get("custom_llm_provider"),
non_default_params.get("api_base") or non_default_params.get("base_url"),
non_default_params.get("prompt_cache_options"),
)
)
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
processed_messages = self._apply_message_injections(
points=message_points,
messages=processed_messages,
max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
openai_dialect=openai_dialect,
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
):
non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
# Pass through non-message injection points for provider-specific handling
if remaining_points:
@ -97,11 +179,43 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return model, processed_messages, non_default_params
@staticmethod
def _targets_openai_prompt_cache_breakpoint(
model: str | None,
custom_llm_provider: str | None,
api_base: object = None,
prompt_cache_options: object = None,
) -> bool:
if model is None or not supports_openai_prompt_cache_breakpoint(model):
return False
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
return False
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def _resolve_provider(model: str) -> str | None:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
try:
_, provider, _, _ = get_llm_provider(model=model)
except BadRequestError:
return None
return provider
@staticmethod
def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
system_blocks: Final = (
sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
)
return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages)
@staticmethod
def _apply_message_injections(
points: list[CacheControlMessageInjectionPoint],
messages: list[AllMessageValues],
max_blocks: int,
openai_dialect: bool = False,
) -> list[AllMessageValues]:
"""Apply message-level cache control injection points in order.
@ -112,7 +226,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
``max_blocks`` is reached. Injection points are honored in config order,
so earlier points win when slots are scarce.
"""
used_blocks = sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages)
used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
limit_reached = False
for point in points:
@ -134,16 +248,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
continue
messages[target_index] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[target_index], control
messages[target_index], control, openai_dialect
)
used_blocks += 1
if AnthropicCacheControlHook._message_has_cache_control(messages[target_index]):
used_blocks += 1
if limit_reached:
break
if limit_reached:
verbose_logger.warning(
"AnthropicCacheControlHook: Reached the Anthropic limit of %s cache_control blocks. Skipping further injection.",
"AnthropicCacheControlHook: Reached the provider limit of %s cache breakpoints. Skipping further injection.",
MAX_CACHE_CONTROL_BLOCKS,
)
@ -189,16 +304,13 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return []
@staticmethod
def _count_cache_control_blocks(message: AllMessageValues) -> int:
"""Count cache_control breakpoints on a message (message + content level)."""
count = 0
if message.get("cache_control") is not None:
count += 1
def _count_cache_control_blocks(message: object) -> int:
if not isinstance(message, dict):
return 0
count = 1 if _carries_cache_breakpoint(message) else 0
content: Final = message.get("content")
if isinstance(content, list):
for block in content:
if isinstance(block, dict) and block.get("cache_control") is not None:
count += 1
count += sum(1 for block in content if _carries_cache_breakpoint(block))
return count
@staticmethod
@ -208,7 +320,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
@staticmethod
def _safe_insert_cache_control_in_message(
message: AllMessageValues, control: ChatCompletionCachedContent
message: AllMessageValues, control: ChatCompletionCachedContent, openai_dialect: bool = False
) -> AllMessageValues:
"""
Safe way to insert cache control in a message
@ -221,6 +333,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
Per Anthropic's API specification, when using multiple content blocks,
only the last content block can have cache_control.
"""
if openai_dialect:
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(message)
message_content: Final = message.get("content", None)
# 1. if string, insert cache control in the message
@ -232,11 +347,51 @@ class AnthropicCacheControlHook(CustomPromptManagement):
message_content[-1]["cache_control"] = control
return message
@staticmethod
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
if message.get("role") == "assistant":
return message
message_content: Final = message.get("content", None)
if isinstance(message_content, str):
marked: Final = copy.copy(message)
marked["content"] = [
with_prompt_cache_breakpoint(
ChatCompletionTextObject(type="text", text=message_content), PromptCacheBreakpoint(mode="explicit")
)
]
return marked
if isinstance(message_content, list):
target_index: Final = next(
(
index
for index in range(len(message_content) - 1, -1, -1)
if _accepts_prompt_cache_breakpoint(message_content[index])
),
None,
)
if target_index is not None:
message_content[target_index] = with_prompt_cache_breakpoint(
message_content[target_index], PromptCacheBreakpoint(mode="explicit")
)
return message
@staticmethod
def _system_block_with_breakpoint(
block: Mapping[str, object], control: ChatCompletionCachedContent, openai_dialect: bool
) -> Mapping[str, object]:
marker: Final = (
("prompt_cache_breakpoint", PromptCacheBreakpoint(mode="explicit"))
if openai_dialect
else ("cache_control", control)
)
return {**block, marker[0]: marker[1]}
@staticmethod
def apply_to_anthropic_messages_request(
messages: list[dict],
system: str | list | None,
injection_points: list[CacheControlInjectionPoint],
openai_dialect: bool = False,
) -> tuple[list[dict], str | list | None, list[CacheControlInjectionPoint]]:
"""Apply cache control injection for the Anthropic-native v1/messages endpoint.
@ -262,30 +417,32 @@ class AnthropicCacheControlHook(CustomPromptManagement):
else:
remaining_points.append(point)
reserved_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
reserved_blocks: Final = (
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
)
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
used_blocks = sum(
AnthropicCacheControlHook._count_cache_control_blocks(cast(AllMessageValues, msg))
for msg in processed_messages
)
if isinstance(processed_system, list):
used_blocks += sum(
1 for b in processed_system if isinstance(b, dict) and b.get("cache_control") is not None
)
message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
if system_points and processed_system is not None and used_blocks < max_blocks:
if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
system_already_has_cc: Final = isinstance(processed_system, list) and any(
isinstance(b, dict) and b.get("cache_control") is not None for b in processed_system
_carries_cache_breakpoint(b) for b in processed_system
)
if not system_already_has_cc:
control: Final = system_points[0].get("control") or ChatCompletionCachedContent(type="ephemeral")
if isinstance(processed_system, str):
processed_system = [{"type": "text", "text": processed_system, "cache_control": control}]
used_blocks += 1
processed_system = [
AnthropicCacheControlHook._system_block_with_breakpoint(
AnthropicSystemMessageContent(type="text", text=processed_system), control, openai_dialect
)
]
system_blocks += 1
elif len(processed_system) > 0 and isinstance(processed_system[-1], dict):
processed_system[-1] = {**processed_system[-1], "cache_control": control}
used_blocks += 1
processed_system[-1] = AnthropicCacheControlHook._system_block_with_breakpoint(
processed_system[-1], control, openai_dialect
)
system_blocks += 1
for i, msg in enumerate(processed_messages):
content = msg.get("content")
@ -295,7 +452,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
processed_messages = AnthropicCacheControlHook._apply_message_injections(
points=message_points,
messages=cast(list[AllMessageValues], processed_messages),
max_blocks=max_blocks - used_blocks,
max_blocks=max_blocks - system_blocks,
openai_dialect=openai_dialect,
)
return processed_messages, processed_system, remaining_points
@ -315,17 +473,57 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return ChatCompletionCachedContent(type="ephemeral")
@staticmethod
def _stamped_as_judged(points: list[CacheControlInjectionPoint]) -> list[dict[str, object]]:
def _stamped_as_judged(points: Sequence[CacheControlInjectionPoint]) -> Sequence[Mapping[str, object]]:
"""Mark written-back points as having passed the client cache_control judgment.
Builds copies because config-owned point dicts are shared across
requests; mutating them would leak the stamp into future requests.
"""
return [{**point, "_litellm_judged": True} for point in points]
return AnthropicCacheControlHook._stamped(points, "_litellm_judged", True)
@staticmethod
def _judged_configured_points(
points: Sequence[CacheControlInjectionPoint],
messages: list[AllMessageValues],
tools: list[object] | None,
model: str,
custom_llm_provider: str | None,
api_base: object,
prompt_cache_options: object,
) -> Sequence[Mapping[str, object]] | None:
if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools):
return None
return AnthropicCacheControlHook._stamped_with_dialect(
points, model, custom_llm_provider, api_base, prompt_cache_options
)
@staticmethod
def _stamped_with_dialect(
points: Sequence[CacheControlInjectionPoint],
model: str,
custom_llm_provider: str | None,
api_base: object,
prompt_cache_options: object,
) -> Sequence[Mapping[str, object]]:
if not supports_openai_prompt_cache_breakpoint(model):
return points
return AnthropicCacheControlHook._stamped(
points,
"_litellm_openai_dialect",
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, prompt_cache_options
),
)
@staticmethod
def _stamped(
points: Sequence[CacheControlInjectionPoint], key: str, value: object
) -> Sequence[Mapping[str, object]]:
return [{**point, key: value} for point in points]
@staticmethod
def _should_stand_down(
points: list[CacheControlInjectionPoint],
points: Sequence[CacheControlInjectionPoint],
messages: list[AllMessageValues],
system: str | list | None,
tools: list | None,
@ -359,11 +557,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
carry the mark either at the top level (Anthropic shape) or nested under
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
"""
if any(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages):
if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
return True
if isinstance(system, list):
if any(isinstance(block, dict) and block.get("cache_control") is not None for block in system):
return True
if tools is not None:
return any(
isinstance(tool, dict)
@ -438,6 +633,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
tools: list | None = None,
enable_prompt_caching: bool | None = None,
api_base: object = None,
) -> None:
"""For /chat/completions: resolve the injection points the request should carry.
@ -452,10 +648,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
unchanged.
"""
if non_default_params.get("cache_control_injection_points"):
if AnthropicCacheControlHook._should_stand_down(
non_default_params["cache_control_injection_points"], messages, None, tools
):
judged: Final = AnthropicCacheControlHook._judged_configured_points(
non_default_params["cache_control_injection_points"],
messages,
tools,
model,
custom_llm_provider,
api_base,
non_default_params.get("prompt_cache_options"),
)
if judged is None:
non_default_params.pop("cache_control_injection_points")
else:
non_default_params["cache_control_injection_points"] = judged
return
points: Final = AnthropicCacheControlHook.get_default_injection_points(
messages=messages,
@ -476,6 +681,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
model: str | None = None,
custom_llm_provider: str | None = None,
tools: list[dict] | None = None,
api_base: str | None = None,
) -> tuple[list[dict], str | list | None]:
"""Extract cache_control_injection_points from kwargs and apply if present.
@ -513,11 +719,21 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if not injection_points:
return messages, system
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
)
breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
messages=messages,
system=system,
injection_points=injection_points,
openai_dialect=openai_dialect,
)
if (
openai_dialect
and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
):
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
if remaining:
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)
return messages, system

6
litellm/integrations/arize/arize_phoenix_prompt_manager.py
View file

@ -359,10 +359,10 @@ class ArizePhoenixPromptManager(CustomPromptManagement):
"""
Determine if prompt management should run based on the prompt_id.
For Arize Phoenix, we always return True and handle the prompt loading
in the _compile_prompt_helper method.
Arize Phoenix needs a prompt_id to compile, so it declines requests without one;
prompt loading itself happens in the _compile_prompt_helper method.
"""
return True
return prompt_id is not None
def _compile_prompt_helper(
self,

2
litellm/integrations/prompt_management_base.py
View file

@ -165,7 +165,7 @@ class PromptManagementBase(ABC):
ignore_prompt_manager_optional_params: bool | None = False,
) -> tuple[str, list[AllMessageValues], dict]:
if prompt_id is None:
raise ValueError("prompt_id is required for Prompt Management Base class")
return model, messages, non_default_params
if not self.should_run_prompt_management(
prompt_id=prompt_id,
prompt_spec=prompt_spec,

9
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -12,6 +12,8 @@ from collections.abc import Mapping
from pathlib import Path
from typing import Final
CLI_TOKEN_FRESHNESS_BUFFER_SECONDS: Final = 360
def get_cli_token_file_path() -> str:
"""Get the path to the CLI token file"""
@ -72,13 +74,18 @@ def get_litellm_gateway_api_key(
return token_data["key"]
def is_cli_token_fresh(token_data: Mapping[str, object], buffer_hours: float = 0.1) -> bool:
def is_cli_token_fresh(
token_data: Mapping[str, object], buffer_hours: float = CLI_TOKEN_FRESHNESS_BUFFER_SECONDS / 3600
) -> bool:
"""Check whether a cached CLI token (as stored in token.json) is still
within its expiration window. Used by `lite auth print-token` to fail
fast, without a network round trip, once the cached token is past
`LITELLM_CLI_JWT_EXPIRATION_HOURS`."""
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
expires_at: Final = token_data.get("expires_at")
if isinstance(expires_at, (int, float)):
return time.time() < expires_at - buffer_hours * 3600
timestamp: Final = token_data.get("timestamp")
if not isinstance(timestamp, (int, float)):
return False

30
litellm/litellm_core_utils/litellm_logging.py
View file

@ -832,8 +832,8 @@ class Logging(LiteLLMLoggingBaseClass):
eg. AnthropicCacheControlHook and BedrockKnowledgeBaseHook both don't require a `prompt_id` to be passed in, they are triggered by dynamic params
"""
for param in non_default_params:
if param in DynamicPromptManagementParamLiteral.list_all_params():
for param in DynamicPromptManagementParamLiteral.list_all_params():
if non_default_params.get(param):
return True
#############################################################################
@ -966,6 +966,23 @@ class Logging(LiteLLMLoggingBaseClass):
return None
@staticmethod
def _prompt_manager_runs_without_prompt_id(
logger: CustomLogger,
prompt_spec: PromptSpec | None,
dynamic_callback_params: StandardCallbackDynamicParams | None,
) -> bool:
if not isinstance(logger, CustomPromptManagement):
return False
try:
return logger.should_run_prompt_management(
prompt_id=None,
prompt_spec=prompt_spec,
dynamic_callback_params=dynamic_callback_params or StandardCallbackDynamicParams(),
)
except Exception:
return False
def get_custom_logger_for_prompt_management(
self,
model: str,
@ -1016,8 +1033,13 @@ class Logging(LiteLLMLoggingBaseClass):
callback_type=CustomPromptManagement
)
if prompt_management_loggers:
logger: Final = prompt_management_loggers[0]
for logger in prompt_management_loggers:
if prompt_id is None and not self._prompt_manager_runs_without_prompt_id(
logger=logger,
prompt_spec=prompt_spec,
dynamic_callback_params=dynamic_callback_params,
):
continue
self.model_call_details["prompt_integration"] = logger.__class__.__name__
return logger

12
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -10,7 +10,7 @@ from collections.abc import Iterable, Mapping, Sequence
from itertools import groupby
from os import PathLike
from pathlib import Path
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
from openai.types.chat.chat_completion_custom_tool_param import (
CustomFormatGrammar,
@ -1325,6 +1325,16 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
return False
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
if marker is None:
return target
marked: Final = {**target, "prompt_cache_breakpoint": marker} # mutable-ok: API message payload
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
"""
Filters a value from a dictionary

31
litellm/litellm_core_utils/realtime_errors.py Normal file
View file

@ -0,0 +1,31 @@
"""Loud-failure helpers for the realtime WebSocket paths.
A realtime caller that only gets a bare close frame has nothing to act on, so
every failure surfaces as an OpenAI-style ``error`` event plus a close frame
whose reason names the failure. Close reasons are capped at
``WEBSOCKET_CLOSE_REASON_MAX_BYTES``: RFC 6455 control frames carry at most 125
bytes, two of which hold the status code, and a longer reason makes the close
frame itself fail, which is how a loud failure turns back into a silent one.
"""
import json
from typing import Final
from litellm.types.realtime import RealtimeErrorDetail, RealtimeErrorEvent
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
def realtime_error_event(message: str, error_type: str) -> str:
detail: Final[RealtimeErrorDetail] = {"type": error_type, "message": message}
event: Final[RealtimeErrorEvent] = {"type": "error", "error": detail}
return json.dumps(event)
def websocket_close_reason(message: str, fallback: str) -> str:
encoded: Final = message.encode("utf-8")
if not encoded:
return fallback
if len(encoded) <= WEBSOCKET_CLOSE_REASON_MAX_BYTES:
return message
return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore")

10
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -484,10 +484,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if "output_config" in extra_kwargs:
request_data["output_config"] = extra_kwargs["output_config"]
custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(request_data)
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
)
if openai_request is None:
raise ValueError("Failed to translate request to OpenAI format")
@ -526,6 +530,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if key not in excluded_keys and key not in completion_kwargs and value is not None:
completion_kwargs[key] = value
explicit_prompt_cache_key: Final = extra_kwargs.get("prompt_cache_key")
if explicit_prompt_cache_key is not None:
completion_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
# Normalize reasoning_effort based on model capabilities
# (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported)
# Must run BEFORE _route_openai_thinking, which prepends "responses/"

58
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -2,10 +2,12 @@ import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast
import litellm
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
prompt_cache_key_from_user_id,
)
# OpenAI has a 64-character limit for function/tool names
@ -13,6 +15,7 @@ from litellm.llms.anthropic.experimental_pass_through.utils import (
OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
def truncate_tool_name(name: str) -> str:
@ -61,6 +64,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
@ -148,7 +152,7 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs
self, kwargs, *, custom_llm_provider: str | None = None
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -179,7 +183,10 @@ class AnthropicAdapter:
(
translated_body,
tool_name_mapping,
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(anthropic_message_request=request_body)
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
)
return translated_body, tool_name_mapping
@ -245,6 +252,9 @@ class AnthropicAdapter:
return anthropic_wrapper.anthropic_sse_wrapper()
_BlockT: Final = TypeVar("_BlockT", bound=Mapping[str, object])
class LiteLLMAnthropicMessagesAdapter:
def __init__(self):
pass
@ -308,6 +318,12 @@ class LiteLLMAnthropicMessagesAdapter:
# Fallback for non-dict objects (shouldn't happen in practice)
cast(dict[str, object], target)["cache_control"] = cache_control
@staticmethod
def _add_prompt_cache_breakpoint_if_present(source: object, target: _BlockT) -> _BlockT:
if isinstance(source, dict) and "prompt_cache_breakpoint" in source:
return with_prompt_cache_breakpoint(target, source["prompt_cache_breakpoint"])
return target
def translatable_anthropic_params(self) -> list[str]:
"""
Which anthropic params, we need to translate to the openai format.
@ -368,7 +384,9 @@ class LiteLLMAnthropicMessagesAdapter:
if content.get("type") == "text":
text_obj = ChatCompletionTextObject(type="text", text=content.get("text", ""))
self._add_cache_control_if_applicable(content, text_obj, model)
new_user_content_list.append(text_obj)
new_user_content_list.append(
self._add_prompt_cache_breakpoint_if_present(content, text_obj)
)
elif content.get("type") == "image":
# Convert Anthropic image format to OpenAI format
source = content.get("source", {})
@ -378,7 +396,9 @@ class LiteLLMAnthropicMessagesAdapter:
image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url)
image_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj)
self._add_cache_control_if_applicable(content, image_obj, model)
new_user_content_list.append(image_obj)
new_user_content_list.append(
self._add_prompt_cache_breakpoint_if_present(content, image_obj)
)
elif content.get("type") == "document":
# Convert Anthropic document format (PDF, etc.) to OpenAI format
source = content.get("source", {})
@ -869,7 +889,7 @@ class LiteLLMAnthropicMessagesAdapter:
continue
text_obj = ChatCompletionTextObject(type="text", text=text)
self._add_cache_control_if_applicable(block, text_obj, model)
text_parts.append(text_obj)
text_parts.append(self._add_prompt_cache_breakpoint_if_present(block, text_obj))
return ChatCompletionSystemMessage(role="system", content=text_parts) if text_parts else None
def _add_system_message_to_messages(
@ -900,23 +920,41 @@ class LiteLLMAnthropicMessagesAdapter:
"text": block.get("text", ""),
}
self._add_cache_control_if_applicable(block, text_block, model_name)
openai_system_content.append(text_block)
openai_system_content.append(self._add_prompt_cache_breakpoint_if_present(block, text_block))
if openai_system_content:
new_messages.insert(
0,
ChatCompletionSystemMessage(role="system", content=openai_system_content),
)
@staticmethod
def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool:
if not model or not custom_llm_provider:
return False
if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND:
return False
supported_params: Final = litellm.get_supported_openai_params(
model=model, custom_llm_provider=custom_llm_provider
)
return "prompt_cache_key" in (supported_params or ())
def _translate_metadata_to_openai(
self,
anthropic_message_request: AnthropicMessagesRequest,
new_kwargs: ChatCompletionRequest,
*,
custom_llm_provider: str | None = None,
) -> None:
"""Translate metadata fields from Anthropic request to OpenAI request."""
if "metadata" in anthropic_message_request:
metadata: Final = anthropic_message_request["metadata"]
if metadata and "user_id" in metadata:
new_kwargs["user"] = metadata["user_id"]
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
if prompt_cache_key is not None and self._supports_prompt_cache_key(
anthropic_message_request.get("model"), custom_llm_provider
):
new_kwargs["prompt_cache_key"] = prompt_cache_key
if "litellm_metadata" in anthropic_message_request:
# metadata will be passed to litellm.acompletion(), it's a litellm_param
@ -1069,7 +1107,10 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs[k] = v
def translate_anthropic_to_openai(
self, anthropic_message_request: AnthropicMessagesRequest
self,
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format.
@ -1103,6 +1144,7 @@ class LiteLLMAnthropicMessagesAdapter:
self._translate_metadata_to_openai(
anthropic_message_request=anthropic_message_request,
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
)
## CONVERT TOOL CHOICE
self._translate_tool_choice_to_openai(

4
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -230,7 +230,7 @@ async def anthropic_messages(
)
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
)
original_stream: Final = stream or kwargs.get("_websearch_interception_converted_stream", False)
@ -422,7 +422,7 @@ def anthropic_messages_handler(
)
messages, system = AnthropicCacheControlHook.maybe_inject_cache_control(
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools
messages, system, kwargs, model=model, custom_llm_provider=custom_llm_provider, tools=tools, api_base=api_base
)
metadata = validate_anthropic_api_metadata(metadata)

7
litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py
View file

@ -105,7 +105,8 @@ def _build_responses_kwargs(
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
excluded: Final = {"anthropic_messages"}
for key, value in _forwarded_kwargs(extra_kwargs).items():
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
for key, value in forwarded_kwargs.items():
if key == "litellm_logging_obj" and value is not None:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObject,
@ -121,6 +122,10 @@ def _build_responses_kwargs(
elif key not in excluded and key not in responses_kwargs and value is not None:
responses_kwargs[key] = value
explicit_prompt_cache_key: Final = forwarded_kwargs.get("prompt_cache_key")
if explicit_prompt_cache_key is not None:
responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
return responses_kwargs

55
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -12,12 +12,14 @@ from typing import Any, Final, cast
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking_budget,
)
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
prompt_cache_key_from_user_id,
)
from litellm.types.llms.anthropic import (
AllAnthropicPassThroughMessageValues,
@ -82,7 +84,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
@staticmethod
def _translate_midturn_system_content_to_responses(
content: str | Iterable[AnthropicSystemMessageContent],
) -> list[dict[str, str]]: # mutable-ok: API message payload
) -> list[dict[str, object]]: # mutable-ok: API message payload
"""Convert in-sequence system content to Responses input-text parts."""
if isinstance(content, str):
return (
@ -91,7 +93,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if not isinstance(content, list):
return [] # mutable-ok: API message payload
return [ # mutable-ok: API message payload
{"type": "input_text", "text": text} # mutable-ok: API message payload
with_prompt_cache_breakpoint(
{"type": "input_text", "text": text}, block.get("prompt_cache_breakpoint")
) # mutable-ok: API message payload
for block in content
if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload
]
@ -146,11 +150,20 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
continue
btype = block.get("type")
if btype == "text":
user_parts.append({"type": "input_text", "text": block.get("text", "")})
user_parts.append(
with_prompt_cache_breakpoint(
{"type": "input_text", "text": block.get("text", "")},
block.get("prompt_cache_breakpoint"),
)
)
elif btype == "image":
url = self._translate_anthropic_image_source_to_url(cast(dict, block.get("source", {})))
if url:
user_parts.append({"type": "input_image", "image_url": url})
user_parts.append(
with_prompt_cache_breakpoint(
{"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
)
)
elif btype == "tool_result":
tool_use_id = block.get("tool_use_id", "")
inner = block.get("content")
@ -376,19 +389,36 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
anthropic_request["messages"],
)
input_items: Final = self.translate_messages_to_responses_input(messages_list)
system: Final = anthropic_request.get("system")
developer_parts: Final = (
self._translate_midturn_system_content_to_responses(system)
if isinstance(system, list)
and any(isinstance(block, dict) and block.get("prompt_cache_breakpoint") is not None for block in system)
else ()
)
if developer_parts:
input_items.insert(
0,
{ # mutable-ok: API message payload
"type": "message",
"role": "developer",
"content": developer_parts,
},
)
responses_kwargs: Final[dict[str, Any]] = {
"model": model,
"input": self.translate_messages_to_responses_input(messages_list),
"input": input_items,
}
# system -> instructions
system: Final = anthropic_request.get("system")
if system:
if system and not developer_parts:
if isinstance(system, str):
responses_kwargs["instructions"] = system
elif isinstance(system, list):
text_parts = [b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"]
responses_kwargs["instructions"] = "\n".join(filter(None, text_parts))
responses_kwargs["instructions"] = "\n".join(
filter(None, (b.get("text", "") for b in system if isinstance(b, dict) and b.get("type") == "text"))
)
# max_tokens -> max_output_tokens
max_tokens: Final = anthropic_request.get("max_tokens")
@ -452,10 +482,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if openai_cm is not None:
responses_kwargs["context_management"] = openai_cm
# metadata user_id -> user
# metadata user_id -> user and prompt_cache_key
metadata: Final = anthropic_request.get("metadata")
if isinstance(metadata, dict) and "user_id" in metadata:
responses_kwargs["user"] = str(metadata["user_id"])[:64]
prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"])
if prompt_cache_key is not None:
responses_kwargs["prompt_cache_key"] = prompt_cache_key
return responses_kwargs

9
litellm/llms/anthropic/experimental_pass_through/utils.py
View file

@ -1,8 +1,17 @@
import os
from typing import Final
import litellm
from litellm.types.utils import ModelInfo
OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH: Final = 64
def prompt_cache_key_from_user_id(user_id: object) -> str | None:
if user_id is None:
return None
return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
def is_reasoning_auto_summary_enabled() -> bool:
"""Check whether the default 'summary: detailed' injection is enabled (opt-in)."""

14
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -20,6 +20,7 @@ from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -5976,8 +5977,19 @@ class BaseLLMHTTPHandler:
await websocket.close(code=e.status_code, reason=_redact_string(str(e)))
except Exception as e:
verbose_logger.exception("Error connecting to backend: %s", e)
redacted_error: Final = _redact_string(str(e))
try:
await websocket.close(code=1011, reason=_redact_string(f"Internal server error: {e}"))
await websocket.send_text(realtime_error_event(redacted_error, error_type="server_error"))
except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below
verbose_logger.debug("Could not send realtime error event to client; closing anyway")
try:
await websocket.close(
code=1011,
reason=websocket_close_reason(
_redact_string(f"Internal server error: {e}"),
fallback="Internal server error",
),
)
except RuntimeError as close_error:
if "already completed" in str(close_error) or "websocket.close" in str(close_error):
# The WebSocket is already closed or the response is completed, so we can ignore this error

2
litellm/main.py
View file

@ -507,6 +507,7 @@ async def acompletion(
custom_llm_provider=cast(str | None, custom_llm_provider), # cast-ok: read from untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@ -5171,6 +5172,7 @@ def completion(
custom_llm_provider=cast(str | None, kwargs.get("custom_llm_provider")), # cast-ok: untyped kwargs
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=kwargs.get("api_base") or base_url,
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (

4
litellm/model_prices_and_context_window_backup.json
View file

@ -25368,6 +25368,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -25430,6 +25431,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -25492,6 +25494,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -25554,6 +25557,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,

11
litellm/proxy/_experimental/mcp_server/bridge_token_flow.py
View file

@ -15,6 +15,7 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HE
from litellm.types.mcp_server.mcp_server_manager import MCPServer
if TYPE_CHECKING:
from litellm.models.user import LiteLLM_UserTable
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _BridgeAuthorizationCode
from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (
EnvelopeIdentity,
@ -181,7 +182,13 @@ async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResol
async def _reload_active_user_by_id(user_id: str) -> "_KeyResolutionFailure | None":
"""Re-validate a live litellm user by id, returning ``None`` when the user is active or a precise
"""``None`` when the user is live, else the precise failure ``load_active_user_by_id`` found."""
loaded: Final = await load_active_user_by_id(user_id)
return loaded if isinstance(loaded, str) else None
async def load_active_user_by_id(user_id: str) -> "LiteLLM_UserTable | _KeyResolutionFailure":
"""Load a live litellm user by id, returning the record when the user is active or a precise
failure otherwise. The interactive DCR client authenticates via SSO, so its refresh envelope seals a
user subject; renewing it must re-check the user is still live (present and not SCIM-deactivated) so a
deactivated user cannot keep refreshing, mirroring how admission re-validates the same user subject on
@ -226,7 +233,7 @@ async def _reload_active_user_by_id(user_id: str) -> "_KeyResolutionFailure | No
return "no_active_key"
if isinstance(user_object.metadata, dict) and user_object.metadata.get("scim_active") is False:
return "no_active_key"
return None
return user_object
async def _key_owner_scim_deactivated(key: "UserAPIKeyAuth") -> bool:

54
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -47,8 +47,12 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
aggregate_token,
complete_connect_flow,
is_gateway_dcr_client_id,
is_proxy_api_resource,
native_client_auth_contract,
native_client_authorize,
register_aggregate_client,
relative_request_url,
revoke_refresh_token,
)
from litellm.proxy._experimental.mcp_server.oauth_utils import (
TOKEN_NO_CACHE_HEADERS,
@ -58,6 +62,10 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import (
validate_trusted_redirect_uri,
well_known_root_suffix,
)
from litellm.proxy._experimental.mcp_server.proxy_api_credentials import (
lookup_consent_teams,
mint_proxy_credential,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
@ -1663,6 +1671,18 @@ async def authorize(
)
if mcp_server_name is None and client_id and is_gateway_dcr_client_id(client_id):
if is_proxy_api_resource(request, resource):
return await native_client_authorize(
request=request,
client_id=client_id,
redirect_uri=redirect_uri,
state=state,
code_challenge=code_challenge,
code_challenge_method=code_challenge_method,
response_type=response_type,
session_user_id=_session_cookie_user_id(request),
lookup_consent_teams=lookup_consent_teams,
)
return aggregate_authorize(
request=request,
client_id=client_id,
@ -1764,6 +1784,7 @@ async def token_endpoint(
reload_user=_reload_active_user_by_id,
cache=user_api_key_cache,
resource=resource,
mint_proxy_credential=mint_proxy_credential,
)
lookup_name: Final = mcp_server_name or client_id
@ -1793,12 +1814,19 @@ async def token_endpoint(
@router.post("/authorize/complete")
async def authorize_complete(request: Request, flow: str = Form(...), delivery: str | None = Form(None)):
async def authorize_complete(
request: Request,
flow: str = Form(...),
delivery: str | None = Form(None),
team_id: str | None = Form(None),
decision: str | None = Form(None),
) -> Response:
"""Finish an aggregate connect flow: mint the gateway authorization code for the
signed-in user and hand it back to the DCR client, by 303 redirect (default) or, for
a loopback client on a different machine, as a copyable callback URL
(``delivery=manual``). POST plus the per-flow HttpOnly cookie set at /authorize; an
anonymous or bad-flow request just 400s."""
anonymous or bad-flow request just 400s. The native-client consent page adds
``decision`` (approve or deny) and the ``team_id`` the credential is attributed to."""
from litellm.proxy.proxy_server import user_api_key_cache # noqa: PLC0415 # circular import at module load
return await complete_connect_flow(
@ -1807,9 +1835,31 @@ async def authorize_complete(request: Request, flow: str = Form(...), delivery:
session_user_id=_session_cookie_user_id(request),
cache=user_api_key_cache,
delivery=delivery,
team_id=team_id,
decision=decision,
)
@router.post("/revoke")
async def revoke_endpoint(request: Request, token: str = Form(...), client_id: str = Form(...)) -> Response:
"""RFC 7009 revocation for the gateway's refresh tokens (``lite logout``): 200 for a known
client whatever the token's state, 503 when the shared single-use record cannot be written;
access tokens expire on their own."""
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # circular import at module load
master_key,
user_api_key_cache,
)
return await revoke_refresh_token(token=token, client_id=client_id, master_key=master_key, cache=user_api_key_cache)
@router.get("/.well-known/litellm-cli-auth")
async def native_client_auth_discovery(request: Request) -> JSONResponse:
"""The versioned contract a native client (``lite login --pkce``, or a CLI in any other
language) reads to sign a user in through the browser and obtain a proxy credential."""
return JSONResponse(native_client_auth_contract(request), headers=TOKEN_NO_CACHE_HEADERS)
# Per RFC 6749 §4.1.2.1, an IdP that rejects an OAuth authorization request
# redirects back to the configured redirect URI with ``error`` /
# ``error_description`` / ``error_uri`` query params and no ``code``. The MCP

565
litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py
View file

@ -42,15 +42,16 @@ import hmac
import html
import secrets
from base64 import urlsafe_b64encode
from collections.abc import Awaitable, Callable, Mapping
from collections.abc import Awaitable, Callable, Iterable, Mapping
from datetime import datetime, timezone
from typing import Final, Literal, TypeVar
from types import MappingProxyType
from typing import Final, Literal, Protocol, TypeVar
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from fastapi import HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from typing_extensions import assert_never
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
@ -70,6 +71,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.session_credent
from litellm.proxy._experimental.mcp_server.outbound_credentials.session_token import (
SESSION_REFRESH_TTL_SECONDS,
MintedSessionToken,
SessionAudience,
SessionKeys,
SessionPrincipal,
mint_session_refresh_token,
@ -79,6 +81,9 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.common_utils.html_forms.native_client_consent import (
render_native_client_consent_page,
)
from litellm.types.mcp_server.mcp_server_manager import MCPServer
GATEWAY_DCR_CLIENT_ID_PREFIX: Final = "llm_dcrc_"
@ -144,6 +149,47 @@ ReloadUser = Callable[[str], Awaitable[ReloadUserFailure | None]]
``None`` means the user is active; ``unavailable`` is a retryable DB outage; anything
else fails the grant closed."""
PROXY_API_AUDIENCE: Final[SessionAudience] = "proxy_api"
"""The audience a native client (``lite login --pkce``, a Go CLI) asks for by sending the
proxy base URL itself as the RFC 8707 ``resource``: the grant then mints the proxy-API CLI
credential that LLM routes accept, instead of the MCP-only session pair."""
ProxyCredentialMintFailure = Literal[ReloadUserFailure, "not_a_member", "team_required"]
class MintedProxyCredential(BaseModel):
model_config = ConfigDict(frozen=True)
key: str = Field(min_length=1)
expires_in: int = Field(gt=0)
user_id: str = Field(min_length=1)
team_id: str | None = None
class MintProxyCredential(Protocol):
"""Injected proxy-API credential minter ``(user_id, team_id)``: reloads the user live,
checks team membership, refuses a teamless grant for a user who has teams to pick from,
and mints the same credential ``lite login`` mints."""
def __call__(
self, user_id: str, team_id: str | None, /
) -> Awaitable[MintedProxyCredential | ProxyCredentialMintFailure]: ...
class ConsentTeam(BaseModel):
model_config = ConfigDict(frozen=True)
team_id: str = Field(min_length=1)
team_alias: str | None = None
class LookupConsentTeams(Protocol):
"""Injected lookup of the teams a signed-in user may bind a proxy-API credential to."""
def __call__(self, user_id: str, /) -> Awaitable[tuple[ConsentTeam, ...] | ReloadUserFailure]: ...
async def _refuse_proxy_credential(user_id: str, team_id: str | None) -> ProxyCredentialMintFailure:
return "unresolvable"
class GatewayDcrClient(BaseModel):
"""The registration record sealed into a gateway DCR ``client_id``.
@ -173,6 +219,7 @@ class _ConnectFlow(BaseModel):
jti: str = Field(min_length=1)
exp: int
resource_server_id: str | None = None
audience: SessionAudience | None = None
class _GatewayAuthCode(BaseModel):
@ -190,6 +237,8 @@ class _GatewayAuthCode(BaseModel):
iat: int
exp: int
resource_server_id: str | None = None
audience: SessionAudience | None = None
team_id: str | None = None
def is_gateway_dcr_client_id(client_id: str | None) -> bool:
@ -318,9 +367,9 @@ def _cookie_path_and_secure(request: Request) -> tuple[str, bool]:
return parsed.path or "/", parsed.scheme == "https"
def _append_query_params(url: str, params: dict[str, str]) -> str:
def _append_query_params(url: str, params: Iterable[tuple[str, str]]) -> str:
parsed: Final = urlparse(url)
query: Final = parse_qsl(parsed.query, keep_blank_values=True) + list(params.items())
query: Final = (*parse_qsl(parsed.query, keep_blank_values=True), *params)
return urlunparse(parsed._replace(query=urlencode(query)))
@ -392,6 +441,155 @@ def aggregate_authorize(
section 4.1.2.1 an unvalidated redirect URI must not receive an error redirect, and
once the client is at fault there is no trusted place to send the browser.
"""
rejected: Final = _rejected_authorize_request(
client_id, redirect_uri, state, code_challenge, code_challenge_method, response_type
)
if rejected is not None:
return rejected
base_url: Final = get_request_base_url(request)
if session_user_id is None:
return _login_redirect(base_url, request)
scoped_server: Final = resolve_scoped_resource_server(request, resource)
handle: Final = secrets.token_urlsafe(24)
flow: Final = _new_connect_flow(
session_user_id=session_user_id,
client_id=client_id,
redirect_uri=redirect_uri,
state=state,
code_challenge=code_challenge or "",
resource_server_id=scoped_server.server_id if scoped_server is not None else None,
audience=None,
)
connect_url: Final = _append_query_params(
f"{base_url}/ui/connect",
(("connect_flow", handle), ("connect_client", _origin_only(redirect_uri))),
)
response: Final = RedirectResponse(connect_url, status_code=303)
_set_flow_cookie(response, request, handle, flow)
return response
async def native_client_authorize(
request: Request,
client_id: str,
redirect_uri: str,
state: str,
code_challenge: str | None,
code_challenge_method: str | None,
response_type: str | None,
session_user_id: str | None,
lookup_consent_teams: LookupConsentTeams,
) -> Response:
"""The authorize verb for a native client that named the proxy API itself as its
RFC 8707 ``resource``: the same client, redirect, PKCE, and sign-in checks as the
aggregate verb plus a loopback-only redirect (the credential this grant mints is the
user's personal proxy key, which belongs on their own machine and never behind a hosted
callback), then the consent page rendered right here (no connect-page interlude, since
there is no per-server vaulting to do) with the flow sealed into the per-flow cookie
and its handle carried only in the form, never in a URL."""
rejected: Final = _rejected_authorize_request(
client_id, redirect_uri, state, code_challenge, code_challenge_method, response_type
)
if rejected is not None:
return rejected
if not is_loopback_redirect_host(urlparse(redirect_uri)):
return _oauth_error(400, "invalid_request", "a proxy-API grant may only redirect to a loopback address")
base_url: Final = get_request_base_url(request)
if session_user_id is None:
return _login_redirect(base_url, request)
teams: Final = await lookup_consent_teams(session_user_id)
if not isinstance(teams, tuple):
return _consent_lookup_failure_response(teams)
handle: Final = secrets.token_urlsafe(24)
flow: Final = _new_connect_flow(
session_user_id=session_user_id,
client_id=client_id,
redirect_uri=redirect_uri,
state=state,
code_challenge=code_challenge or "",
resource_server_id=None,
audience=PROXY_API_AUDIENCE,
)
page: Final = render_native_client_consent_page(
client_origin=_origin_only(redirect_uri),
user_id=session_user_id,
teams=tuple((team.team_id, team.team_alias or team.team_id) for team in teams),
flow_handle=handle,
complete_url=f"{base_url}/authorize/complete",
)
response: Final = HTMLResponse(page, headers=_CONSENT_PAGE_HEADERS)
_set_flow_cookie(response, request, handle, flow)
return response
_CONSENT_PAGE_HEADERS: Final = MappingProxyType(
{
**TOKEN_NO_CACHE_HEADERS,
"X-Frame-Options": "DENY",
"Content-Security-Policy": "frame-ancestors 'none'",
}
)
NATIVE_CLIENT_AUTH_CONTRACT_VERSION: Final = 1
"""The version a native client checks before trusting the rest of the discovery document.
Bump it only when an existing field changes meaning or goes away; adding fields is free."""
class NativeClientAuthContract(TypedDict):
contract_version: ReadOnly[int]
issuer: ReadOnly[str]
authorization_endpoint: ReadOnly[str]
token_endpoint: ReadOnly[str]
registration_endpoint: ReadOnly[str]
revocation_endpoint: ReadOnly[str]
resource: ReadOnly[str]
response_types_supported: ReadOnly[tuple[str, ...]]
grant_types_supported: ReadOnly[tuple[str, ...]]
code_challenge_methods_supported: ReadOnly[tuple[str, ...]]
token_endpoint_auth_methods_supported: ReadOnly[tuple[str, ...]]
revocation_endpoint_auth_methods_supported: ReadOnly[tuple[str, ...]]
def native_client_auth_contract(request: Request) -> NativeClientAuthContract:
"""The versioned discovery document at ``/.well-known/litellm-cli-auth``: everything a
native client (in any language) needs to run the sign-in without reading LiteLLM
source. ``resource`` is the exact value to send as the RFC 8707 ``resource`` parameter
on authorize and token requests so the grant is issued for the proxy API."""
base_url: Final = get_request_base_url(request)
contract: Final[NativeClientAuthContract] = {
"contract_version": NATIVE_CLIENT_AUTH_CONTRACT_VERSION,
"issuer": base_url,
"authorization_endpoint": f"{base_url}/authorize",
"token_endpoint": f"{base_url}/token",
"registration_endpoint": f"{base_url}/register",
"revocation_endpoint": f"{base_url}/revoke",
"resource": base_url,
"response_types_supported": ("code",),
"grant_types_supported": ("authorization_code", "refresh_token"),
"code_challenge_methods_supported": ("S256",),
"token_endpoint_auth_methods_supported": ("none",),
"revocation_endpoint_auth_methods_supported": ("none",),
}
return contract
def is_proxy_api_resource(request: Request, resource: str | None) -> bool:
"""True when the RFC 8707 ``resource`` names the proxy itself (its base URL), which is
how a native client asks for the proxy-API audience rather than an MCP session."""
if resource is None:
return False
canonical: Final = canonical_resource_uri(resource)
return canonical is not None and canonical == canonicalize_url_identity(get_request_base_url(request))
def _rejected_authorize_request(
client_id: str,
redirect_uri: str,
state: str,
code_challenge: str | None,
code_challenge_method: str | None,
response_type: str | None,
) -> Response | None:
client: Final = open_gateway_dcr_client(client_id)
if client is None:
return _oauth_error(400, "invalid_client", "unknown or malformed client_id")
@ -407,14 +605,25 @@ def aggregate_authorize(
)
if len(state) > MAX_STATE_LENGTH:
return _oauth_error(400, "invalid_request", f"state must be at most {MAX_STATE_LENGTH} characters")
base_url: Final = get_request_base_url(request)
if session_user_id is None:
login_url: Final = f"{base_url}/sso/key/generate?{urlencode({'return_to': relative_request_url(request)})}"
return RedirectResponse(login_url, status_code=303)
return None
def _login_redirect(base_url: str, request: Request) -> Response:
return_to: Final = urlencode((("return_to", relative_request_url(request)),))
return RedirectResponse(f"{base_url}/sso/key/generate?{return_to}", status_code=303)
def _new_connect_flow(
session_user_id: str,
client_id: str,
redirect_uri: str,
state: str,
code_challenge: str,
resource_server_id: str | None,
audience: SessionAudience | None,
) -> _ConnectFlow:
now: Final = datetime.now(timezone.utc)
scoped_server: Final = resolve_scoped_resource_server(request, resource)
handle: Final = secrets.token_urlsafe(24)
flow: Final = _ConnectFlow(
return _ConnectFlow(
user_id=session_user_id,
client_id=client_id,
redirect_uri=redirect_uri,
@ -422,13 +631,12 @@ def aggregate_authorize(
code_challenge=code_challenge,
jti=secrets.token_urlsafe(24),
exp=int(now.timestamp()) + CONNECT_FLOW_TTL_SECONDS,
resource_server_id=scoped_server.server_id if scoped_server is not None else None,
resource_server_id=resource_server_id,
audience=audience,
)
connect_url: Final = _append_query_params(
f"{base_url}/ui/connect",
{"connect_flow": handle, "connect_client": _origin_only(redirect_uri)},
)
response: Final = RedirectResponse(connect_url, status_code=303)
def _set_flow_cookie(response: Response, request: Request, handle: str, flow: _ConnectFlow) -> None:
path, secure = _cookie_path_and_secure(request)
response.set_cookie(
key=_flow_cookie_name(handle),
@ -439,7 +647,18 @@ def aggregate_authorize(
httponly=True,
samesite="lax",
)
return response
def _consent_lookup_failure_response(failure: ReloadUserFailure) -> Response:
match failure:
case "unavailable":
return _oauth_error(503, "temporarily_unavailable", "the gateway database is unavailable; retry")
case "unresolvable":
return _oauth_error(500, "server_error", "the gateway is not configured to resolve users")
case "no_active_key":
return _oauth_error(403, "access_denied", "the signed-in user is not active")
case _:
assert_never(failure)
def _origin_only(url: str) -> str:
@ -455,6 +674,8 @@ async def complete_connect_flow(
session_user_id: str | None,
cache: DualCache,
delivery: str | None = None,
team_id: str | None = None,
decision: str | None = None,
) -> Response:
"""The deliberate finish step of the connect flow: mint the gateway authorization
code and send the browser back to the client.
@ -479,9 +700,16 @@ async def complete_connect_flow(
party. Unknown ``delivery`` values are rejected rather than defaulted: a client that
asked for manual delivery and got a dead redirect instead would silently lose its
code.
``decision`` and ``team_id`` come from the native-client consent page. ``"deny"``
burns the flow and sends the client ``error=access_denied`` so it stops waiting;
``team_id`` is sealed into the code only for proxy-API flows, where it picks which of
the user's teams the minted credential is attributed to.
"""
if delivery not in (None, "redirect", "manual"):
return _oauth_error(400, "invalid_request", "delivery must be 'redirect' or 'manual'")
if decision not in (None, "approve", "deny"):
return _oauth_error(400, "invalid_request", "decision must be 'approve' or 'deny'")
sealed_flow: Final = request.cookies.get(_flow_cookie_name(flow_handle))
if sealed_flow is None:
return _oauth_error(400, "invalid_request", "unknown or expired connect flow")
@ -495,10 +723,34 @@ async def complete_connect_flow(
return _oauth_error(401, "login_required", "sign in to LiteLLM to finish connecting")
if session_user_id != flow.user_id:
return _oauth_error(403, "access_denied", "the signed-in user does not match this connect flow")
if not await _SingleUseGuard(cache).claim(
f"{_USED_FLOW_CACHE_PREFIX}{flow.jti}", CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
):
return _oauth_error(400, "invalid_request", "this connect flow was already completed; restart the connection")
flow_refusal: Final = _claim_refusal(
await _SingleUseGuard(cache).claim(
f"{_USED_FLOW_CACHE_PREFIX}{flow.jti}", CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
),
replayed=_oauth_error(
400, "invalid_request", "this connect flow was already completed; restart the connection"
),
)
if flow_refusal is not None:
return flow_refusal
response: Final = (
_denied_flow_response(flow) if decision == "deny" else _approved_flow_response(flow, delivery, team_id, now)
)
path, secure = _cookie_path_and_secure(request)
response.delete_cookie(key=_flow_cookie_name(flow_handle), path=path, secure=secure, httponly=True, samesite="lax")
return response
def _state_param(flow: _ConnectFlow) -> tuple[tuple[str, str], ...]:
return (("state", flow.state),) if flow.state else ()
def _denied_flow_response(flow: _ConnectFlow) -> Response:
params: Final = (("error", "access_denied"), *_state_param(flow))
return RedirectResponse(_append_query_params(flow.redirect_uri, params), status_code=303)
def _approved_flow_response(flow: _ConnectFlow, delivery: str | None, team_id: str | None, now: datetime) -> Response:
manual_delivery: Final = delivery == "manual" and is_loopback_redirect_host(urlparse(flow.redirect_uri))
code_ttl: Final = MANUAL_DELIVERY_AUTH_CODE_TTL_SECONDS if manual_delivery else GATEWAY_AUTH_CODE_TTL_SECONDS
code: Final = _seal(
@ -512,16 +764,14 @@ async def complete_connect_flow(
iat=int(now.timestamp()),
exp=int(now.timestamp()) + code_ttl,
resource_server_id=flow.resource_server_id,
audience=flow.audience,
team_id=(team_id or None) if flow.audience == PROXY_API_AUDIENCE else None,
),
)
params: Final = {"code": code, **({"state": flow.state} if flow.state else {})}
callback_url: Final = _append_query_params(flow.redirect_uri, params)
response: Final[Response] = (
_manual_delivery_response(callback_url) if manual_delivery else RedirectResponse(callback_url, status_code=303)
)
path, secure = _cookie_path_and_secure(request)
response.delete_cookie(key=_flow_cookie_name(flow_handle), path=path, secure=secure, httponly=True, samesite="lax")
return response
callback_url: Final = _append_query_params(flow.redirect_uri, (("code", code), *_state_param(flow)))
if manual_delivery:
return _manual_delivery_response(callback_url)
return RedirectResponse(callback_url, status_code=303)
def _manual_delivery_response(callback_url: str) -> Response:
@ -562,6 +812,27 @@ def _pkce_verifier_matches(code_verifier: str, code_challenge: str) -> bool:
return hmac.compare_digest(computed, code_challenge.encode("utf-8"))
ClaimOutcome = Literal["first", "replayed", "unavailable"]
_CLAIM_UNAVAILABLE_DESCRIPTION: Final = "the single-use record is unavailable right now; try again shortly"
def _claim_refusal(outcome: ClaimOutcome, replayed: Response) -> Response | None:
"""A claim that is not the first caller's is refused, but the two reasons must stay apart on the
wire: a replay is the grant's own 4xx, while a shared backend that could not record the claim is
a 503 (RFC 7009 section 2.2.1, RFC 6749 section 5.2 ``temporarily_unavailable``), so the client
keeps the still-valid token and retries instead of being told it was already used."""
match outcome:
case "first":
return None
case "replayed":
return replayed
case "unavailable":
return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
case _:
assert_never(outcome)
class _SingleUseGuard:
"""Atomic single-use claim for a one-time id (an auth-code, connect-flow ``jti``, or refresh-token
``jti``) over the injected proxy cache.
@ -585,9 +856,10 @@ class _SingleUseGuard:
def __init__(self, cache: DualCache) -> None:
self._cache = cache
async def claim(self, key: str, ttl_seconds: int) -> bool:
"""Atomically claim ``key``. ``True`` iff this caller is the first (increment to 1); ``False``
on a replay (>1) or when the claim could not be recorded in the shared backend (fail closed)."""
async def claim(self, key: str, ttl_seconds: int) -> ClaimOutcome:
"""Atomically claim ``key``. ``"first"`` iff this caller is the first (increment to 1),
``"replayed"`` on a replay (>1), and ``"unavailable"`` when the claim could not be recorded in
the shared backend, which every caller treats as a refusal (fail closed)."""
from litellm.proxy.proxy_server import redis_usage_cache # noqa: PLC0415 # circular import at module load
# Resolve the shared authority HERE rather than trusting the injected cache: callers pass
@ -606,11 +878,11 @@ class _SingleUseGuard:
verbose_logger.warning(
"mcp gateway single-use claim: shared cache backend unavailable, failing closed: %s", e
)
return False
return count == 1
return "unavailable"
return "first" if count == 1 else "replayed"
# No shared backend configured (single-replica): the in-memory increment is authoritative.
count = await self._cache.async_increment_cache(key, 1, ttl=ttl_seconds, local_only=True)
return count == 1
return "first" if count == 1 else "replayed"
def _session_token_pair(principal: SessionPrincipal, keys: SessionKeys, now: datetime) -> Response:
@ -630,6 +902,37 @@ def _session_token_pair(principal: SessionPrincipal, keys: SessionKeys, now: dat
)
class _ProxyCredentialTokenResponse(TypedDict):
access_token: ReadOnly[str]
token_type: ReadOnly[Literal["Bearer"]]
expires_in: ReadOnly[int]
refresh_token: ReadOnly[str]
user_id: ReadOnly[str]
team_id: ReadOnly[str | None]
def _proxy_credential_response(
minted: MintedProxyCredential, principal: SessionPrincipal, keys: SessionKeys, now: datetime
) -> Response:
"""The proxy-API token response: the access token is the very credential ``lite
login`` stores (accepted on every proxy route with user and team attribution), and
the refresh token is a gateway-sealed rotating token bound to the team the credential
was minted for, so a renewal keeps the team the user consented to."""
bound_principal: Final = principal.model_copy(update=MappingProxyType({"team_id": minted.team_id}))
refresh: Final = mint_session_refresh_token(bound_principal, keys, now)
if not isinstance(refresh, MintedSessionToken):
return _oauth_error(500, "server_error", "failed to mint the session credential")
body: Final[_ProxyCredentialTokenResponse] = {
"access_token": minted.key,
"token_type": "Bearer",
"expires_in": minted.expires_in,
"refresh_token": refresh.token.get_secret_value(),
"user_id": minted.user_id,
"team_id": minted.team_id,
}
return JSONResponse(status_code=200, content=body, headers=TOKEN_NO_CACHE_HEADERS)
def _reload_failure_response(failure: ReloadUserFailure) -> Response:
"""Map the live-user revalidation failure onto its OAuth error, exhaustively, so a new
``ReloadUserFailure`` member is a type error here rather than silently 400ing."""
@ -644,6 +947,22 @@ def _reload_failure_response(failure: ReloadUserFailure) -> Response:
assert_never(failure)
def _mint_failure_response(failure: ProxyCredentialMintFailure) -> Response:
match failure:
case "not_a_member":
return _oauth_error(
400, "invalid_grant", "the user is no longer a member of the team this grant was issued for"
)
case "team_required":
return _oauth_error(
400, "invalid_grant", "this user belongs to a team; sign in again and pick the team for this credential"
)
case "unavailable" | "unresolvable" | "no_active_key":
return _reload_failure_response(failure)
case _:
assert_never(failure)
def _resource_conflicts_with_scope(
request: Request, resource: str | None, sealed_resource_server_id: str | None
) -> bool:
@ -670,15 +989,26 @@ async def aggregate_token(
reload_user: ReloadUser,
cache: DualCache,
resource: str | None = None,
mint_proxy_credential: MintProxyCredential = _refuse_proxy_credential,
) -> Response:
"""The aggregate token verb: authorization_code and refresh_token grants for the
identity-only session pair. Every path re-validates the litellm user live before
minting, so a deactivated user cannot obtain or renew a session."""
identity-only session pair, or for the proxy-API credential when the grant was issued
with that audience. Every path re-validates the litellm user live before minting, so a
deactivated user cannot obtain or renew a session."""
if master_key is None:
verbose_logger.error("mcp_gateway_dcr token grant rejected: no master_key configured")
return _oauth_error(500, "server_error", "the gateway has no master key configured")
keys: Final = session_keys_from_master_key(master_key)
now: Final = datetime.now(timezone.utc)
issue: Final = _GrantIssuer(
request=request,
resource=resource,
keys=keys,
now=now,
reload_user=reload_user,
mint_proxy_credential=mint_proxy_credential,
guard=_SingleUseGuard(cache),
)
if grant_type == "authorization_code":
return await _authorization_code_grant(
request=request,
@ -687,10 +1017,8 @@ async def aggregate_token(
client_id=client_id,
code_verifier=code_verifier,
resource=resource,
keys=keys,
now=now,
reload_user=reload_user,
guard=_SingleUseGuard(cache),
issue=issue,
)
if grant_type == "refresh_token":
return await _refresh_token_grant(
@ -700,12 +1028,78 @@ async def aggregate_token(
resource=resource,
keys=keys,
now=now,
reload_user=reload_user,
guard=_SingleUseGuard(cache),
issue=issue,
)
return _oauth_error(400, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token")
class _GrantIssuer:
"""The tail every grant shares once its own proof (code + PKCE, or a refresh token)
has checked out: revalidate the user live, claim the single-use marker, mint. The
claim comes AFTER revalidation and minting so a transient DB 503 never burns a
still-valid code or refresh token, and fails closed when it cannot be recorded."""
def __init__(
self,
request: Request,
resource: str | None,
keys: SessionKeys,
now: datetime,
reload_user: ReloadUser,
mint_proxy_credential: MintProxyCredential,
guard: _SingleUseGuard,
) -> None:
self._request: Final = request
self._resource: Final = resource
self._keys: Final = keys
self._now: Final = now
self._reload_user: Final = reload_user
self._mint_proxy_credential: Final = mint_proxy_credential
self._guard: Final = guard
async def __call__(
self, principal: SessionPrincipal, claim_key: str, claim_ttl_seconds: int, replayed: str
) -> Response:
match principal.audience:
case None:
return await self._issue_session_pair(principal, claim_key, claim_ttl_seconds, replayed)
case "proxy_api":
return await self._issue_proxy_credential(principal, claim_key, claim_ttl_seconds, replayed)
case _:
assert_never(principal.audience)
async def _issue_session_pair(
self, principal: SessionPrincipal, claim_key: str, claim_ttl_seconds: int, replayed: str
) -> Response:
failure: Final = await self._reload_user(principal.user_id)
if failure is not None:
return _reload_failure_response(failure)
refusal: Final = await self._claim_refusal(claim_key, claim_ttl_seconds, replayed)
if refusal is not None:
return refusal
return _session_token_pair(principal, self._keys, self._now)
async def _issue_proxy_credential(
self, principal: SessionPrincipal, claim_key: str, claim_ttl_seconds: int, replayed: str
) -> Response:
if self._resource is not None and not is_proxy_api_resource(self._request, self._resource):
return _oauth_error(
400, "invalid_target", "resource does not match the proxy API this grant was issued for"
)
minted: Final = await self._mint_proxy_credential(principal.user_id, principal.team_id)
if not isinstance(minted, MintedProxyCredential):
return _mint_failure_response(minted)
refusal: Final = await self._claim_refusal(claim_key, claim_ttl_seconds, replayed)
if refusal is not None:
return refusal
return _proxy_credential_response(minted, principal, self._keys, self._now)
async def _claim_refusal(self, claim_key: str, claim_ttl_seconds: int, replayed: str) -> Response | None:
return _claim_refusal(
await self._guard.claim(claim_key, claim_ttl_seconds), replayed=_oauth_error(400, "invalid_grant", replayed)
)
async def _authorization_code_grant(
request: Request,
code: str | None,
@ -713,10 +1107,8 @@ async def _authorization_code_grant(
client_id: str,
code_verifier: str | None,
resource: str | None,
keys: SessionKeys,
now: datetime,
reload_user: ReloadUser,
guard: _SingleUseGuard,
issue: _GrantIssuer,
) -> Response:
if not code or not redirect_uri or not code_verifier:
return _oauth_error(400, "invalid_request", "code, redirect_uri, and code_verifier are required")
@ -733,23 +1125,19 @@ async def _authorization_code_grant(
return _oauth_error(400, "invalid_target", "resource does not match the scope this code was issued for")
if not _pkce_verifier_matches(code_verifier, parsed.code_challenge):
return _oauth_error(400, "invalid_grant", "PKCE verification failed")
# Revalidate the user BEFORE claiming the code, so a transient DB outage (a retryable
# 503) does not consume a still-valid code and force the client to restart sign-in.
failure: Final = await reload_user(parsed.user_id)
if failure is not None:
return _reload_failure_response(failure)
# Atomic single-use claim is the gate: on a concurrent double-redeem exactly one caller
# wins, and a claim that cannot be recorded fails closed. The marker's TTL derives from
# the code's own remaining lifetime so it outlives whichever lifetime the code was minted with.
if not await guard.claim(
f"{_USED_CODE_CACHE_PREFIX}{parsed.jti}",
parsed.exp - int(now.timestamp()) + _CLAIM_TTL_BUFFER_SECONDS,
):
return _oauth_error(400, "invalid_grant", "the authorization code was already used")
return _session_token_pair(
SessionPrincipal(user_id=parsed.user_id, client_id=client_id, resource_server_id=parsed.resource_server_id),
keys,
now,
# The marker's TTL derives from the code's own remaining lifetime so it outlives
# whichever lifetime the code was minted with.
return await issue(
SessionPrincipal(
user_id=parsed.user_id,
client_id=client_id,
resource_server_id=parsed.resource_server_id,
audience=parsed.audience,
team_id=parsed.team_id,
),
claim_key=f"{_USED_CODE_CACHE_PREFIX}{parsed.jti}",
claim_ttl_seconds=parsed.exp - int(now.timestamp()) + _CLAIM_TTL_BUFFER_SECONDS,
replayed="the authorization code was already used",
)
@ -760,8 +1148,7 @@ async def _refresh_token_grant(
resource: str | None,
keys: SessionKeys,
now: datetime,
reload_user: ReloadUser,
guard: _SingleUseGuard,
issue: _GrantIssuer,
) -> Response:
if not refresh_token:
return _oauth_error(400, "invalid_request", "refresh_token is required")
@ -770,16 +1157,38 @@ async def _refresh_token_grant(
return _oauth_error(400, "invalid_grant", "the refresh token is invalid for this client")
if _resource_conflicts_with_scope(request, resource, opened.principal.resource_server_id):
return _oauth_error(400, "invalid_target", "resource does not match the scope this token was issued for")
failure: Final = await reload_user(opened.principal.user_id)
if failure is not None:
return _reload_failure_response(failure)
# Refresh-token rotation (OAuth 2.0 Security BCP section 4.13): the presented refresh token is
# single-use. Claim its jti before issuing the replacement pair, so a captured or replayed
# refresh token cannot mint a second pair after the legitimate holder rotated. Claimed AFTER
# user revalidation so a transient DB 503 does not burn a still-valid token; a claim that
# cannot be recorded fails closed, exactly like the authorization-code path.
if not await guard.claim(
f"{_USED_REFRESH_CACHE_PREFIX}{opened.jti}", SESSION_REFRESH_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
):
return _oauth_error(400, "invalid_grant", "the refresh token was already used")
return _session_token_pair(opened.principal, keys, now)
# single-use, so a captured or replayed refresh token cannot mint a second pair after the
# legitimate holder rotated.
return await issue(
opened.principal,
claim_key=f"{_USED_REFRESH_CACHE_PREFIX}{opened.jti}",
claim_ttl_seconds=SESSION_REFRESH_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS,
replayed="the refresh token was already used",
)
async def revoke_refresh_token(token: str, client_id: str, master_key: str | None, cache: DualCache) -> Response:
"""RFC 7009 revocation for the gateway's refresh tokens: burn the presented token's
``jti`` so neither the holder nor a thief can rotate it again. Access tokens are
stateless and expire on their own (the proxy-API credential within
``CLI_JWT_EXPIRATION_HOURS``), so per RFC 7009 section 2.2 an unrecognized or already
dead token still answers 200; only an unknown client is refused. A live token whose
burn could not be recorded in the shared backend answers 503 (section 2.2.1), so the
client knows the token still stands and retries instead of reporting a logout that
never happened."""
if not is_gateway_dcr_client_id(client_id) or open_gateway_dcr_client(client_id) is None:
return _oauth_error(401, "invalid_client", "unknown or malformed client_id")
if master_key is None:
verbose_logger.error("mcp_gateway_dcr revoke rejected: no master_key configured")
return _oauth_error(500, "server_error", "the gateway has no master key configured")
keys: Final = session_keys_from_master_key(master_key)
now: Final = datetime.now(timezone.utc)
opened: Final = open_session_refresh_bearer(token, keys, now, expected_client_id=client_id)
if isinstance(opened, SessionRefreshOpened):
burned: Final = await _SingleUseGuard(cache).claim(
f"{_USED_REFRESH_CACHE_PREFIX}{opened.jti}", SESSION_REFRESH_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
)
if burned == "unavailable":
return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
return Response(content="{}", media_type="application/json", headers=TOKEN_NO_CACHE_HEADERS)

19
litellm/proxy/_experimental/mcp_server/outbound_credentials/session_token.py
View file

@ -76,6 +76,13 @@ SessionTokenKind = Literal["session", "session_refresh"]
on open, so a signature-valid token of one kind cannot be replayed as the other even if its
wire prefix is swapped (the prefix is not part of the signed payload; this claim is)."""
SessionAudience = Literal["proxy_api"]
"""The non-MCP audience a session REFRESH token can be minted for. ``None`` (the default and
the only value ever on an MCP wire) means the aggregate MCP gateway; ``"proxy_api"`` means the
refresh grant re-mints the proxy-API CLI credential instead of an MCP session pair. The audience
is read only from the signed claims, never from the request, so a token of one audience can
never be redeemed as the other."""
class SessionPrincipal(BaseModel):
"""The litellm user a session token identifies and the DCR client it was issued to.
@ -97,6 +104,8 @@ class SessionPrincipal(BaseModel):
user_id: str = Field(min_length=1)
client_id: str = Field(min_length=1)
resource_server_id: str | None = None
audience: SessionAudience | None = None
team_id: str | None = None
class SessionKeys(BaseModel):
@ -194,6 +203,8 @@ class _SessionClaims(BaseModel):
user_id: str = Field(min_length=1)
client_id: str = Field(min_length=1)
resource_server_id: str | None = None
audience: SessionAudience | None = None
team_id: str | None = None
def is_session_token(candidate: str) -> bool:
@ -295,6 +306,8 @@ def _mint(
user_id=principal.user_id,
client_id=principal.client_id,
resource_server_id=principal.resource_server_id,
audience=principal.audience,
team_id=principal.team_id,
)
token: Final = prefix + jwt.encode(
claims.model_dump(exclude_none=True), keys.signing_key.get_secret_value(), algorithm=_SESSION_JWT_ALGORITHM
@ -333,7 +346,11 @@ def _open(
return SessionExpired()
return OpenedSessionToken(
principal=SessionPrincipal(
user_id=claims.user_id, client_id=claims.client_id, resource_server_id=claims.resource_server_id
user_id=claims.user_id,
client_id=claims.client_id,
resource_server_id=claims.resource_server_id,
audience=claims.audience,
team_id=claims.team_id,
),
jti=claims.jti,
)

90
litellm/proxy/_experimental/mcp_server/proxy_api_credentials.py Normal file
View file

@ -0,0 +1,90 @@
"""The proxy-API side of the native-client sign-in: turning a consented OAuth grant into
the same per-user credential ``lite login`` stores, so the bearer a CLI obtains through
the browser flow is accepted on every proxy route with user and team attribution."""
from __future__ import annotations
from collections.abc import Sequence
from typing import Final
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
from litellm.proxy._experimental.mcp_server.bridge_token_flow import load_active_user_by_id
from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
ConsentTeam,
MintedProxyCredential,
ProxyCredentialMintFailure,
ReloadUserFailure,
)
from litellm.proxy._types import LiteLLM_UserTable
from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken
from litellm.proxy.management_endpoints.ui_sso import (
CliSsoTeamDetail,
fetch_cli_sso_team_details,
selected_cli_sso_team_detail,
)
async def lookup_consent_teams(user_id: str) -> tuple[ConsentTeam, ...] | ReloadUserFailure:
user: Final = await load_active_user_by_id(user_id)
if isinstance(user, str):
return user
details: Final = await _team_details(user.teams)
if details is None:
return "unavailable"
return tuple(
ConsentTeam(team_id=detail.team_id, team_alias=detail.team_alias)
for detail in details
if detail.team_id is not None
)
async def mint_proxy_credential(
user_id: str, team_id: str | None
) -> MintedProxyCredential | ProxyCredentialMintFailure:
"""Mint the ``lite login`` credential for a consented grant. Membership is checked
live, so a team the user left between consent and redemption (or between refreshes)
refuses the grant instead of minting a credential attributed to a team they are no
longer on. The team is exactly the one the consent page sealed into the grant; nothing
is picked on the user's behalf here, so a refresh can never move the credential, and a
grant that names no team is refused for a user with a live team to pick from (the same
rule ``lite login`` applies), so a user cannot step outside their teams' attribution by
posting the consent form without one. Memberships whose team rows are gone count as no
team at all, the way ``lite login`` treats them, so they can never lock a user out. The
user row handed to the minter carries no team list, exactly like ``lite login``'s, so
the minter's own first-team fallback stays inert."""
user: Final = await load_active_user_by_id(user_id)
if isinstance(user, str):
return user
if user.user_role is None:
return "no_active_key"
if team_id is not None and team_id not in user.teams:
return "not_a_member"
details: Final = await _team_details(user.teams) if user.teams else ()
if details is None:
return "unavailable"
if team_id is None and any(detail.team_id is not None for detail in details):
return "team_required"
selected: Final = selected_cli_sso_team_detail(details, team_id)
if selected is None:
return "not_a_member"
key: Final = ExperimentalUIJWTToken.get_cli_jwt_auth_token(
user_info=LiteLLM_UserTable(user_id=user.user_id, user_role=user.user_role, models=user.models),
team_id=team_id,
team_alias=selected.team_alias,
team_models=selected.team_models,
team_model_aliases=selected.team_model_aliases,
)
return MintedProxyCredential(
key=key,
expires_in=CLI_JWT_EXPIRATION_HOURS * 3600,
user_id=user.user_id,
team_id=team_id,
)
async def _team_details(teams: Sequence[str]) -> tuple[CliSsoTeamDetail, ...] | None:
from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # rebound after startup, so read it per call
if prisma_client is None:
return None
return await fetch_cli_sso_team_details(prisma_client, teams)

2
litellm/proxy/_lazy_features.py
View file

@ -155,10 +155,12 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/.well-known/oauth-",
"/.well-known/openid-configuration",
"/.well-known/jwks.json",
"/.well-known/litellm-cli-auth",
"/authorize",
"/token",
"/callback",
"/register",
"/revoke",
),
# Catches the /{mcp_server_name}/authorize|token|register variants.
path_suffixes=("/authorize", "/token", "/register"),

7
litellm/proxy/client/README.md
View file

@ -339,9 +339,10 @@ sequenceDiagram
The CLI provides these authentication commands:
- **`lite login`** - Start SSO authentication flow
- **`lite logout`** - Clear stored authentication token
- **`lite login --pkce`** - Sign in through the system browser with OAuth authorization code + PKCE; the key renews itself with a refresh token
- **`lite logout`** - Clear stored authentication token (and revoke a `--pkce` refresh token on the proxy)
- **`lite whoami`** - Show current authentication status
- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); fails once the token has expired
- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first and fails once a classic token has expired
### Authentication Flow Steps
@ -377,7 +378,7 @@ Authentication tokens are stored in `~/.litellm/token.json` with restricted file
}
```
The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
### Usage

4
litellm/proxy/client/cli/README.md
View file

@ -501,13 +501,13 @@ To pin the model, pass the agent's own model flag (for example `lite claude --mo
The token minted by `lite login` is a short-lived, per-session agent credential, not a managed virtual key. It is scoped to the user and team you authenticated as, inherits that user's and team's models and budgets, and is enforced on the proxy exactly like a virtual key on the same team (guardrails, routing, logging, spend). Spend is tracked against the shared team and user budgets, so running several agents (or logging in more than once) does not hand each session its own separate budget; they all draw down the same team/user allowance. There is no separate per-session cap, so sustained agent use is not capped at a small chat-session limit.
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, and `lite opencode` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, and `lite opencode` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the daily re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
### Route Every Claude Code Session Through the Proxy
`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it.
Two things need to already be true: you've run `lite login`, since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you.
Two things need to already be true: you've run `lite login` (or `lite login --pkce`, whose key the helper renews on its own), since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you.
```bash
lite login

167
litellm/proxy/client/cli/commands/auth.py
View file

@ -11,7 +11,7 @@ import click
import requests
from rich.console import Console
from rich.table import Table
from typing_extensions import NotRequired, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh
@ -22,6 +22,15 @@ from .claude_settings import (
ClaudeSettingsError,
write_claude_settings,
)
from .pkce_login import (
Http,
PkceFailure,
RevocationUnavailable,
fresh_api_key,
pkce_token_record,
revoke_stored_credential,
run_pkce_login,
)
from .private_json import write_private_json
@ -34,6 +43,13 @@ class CliTokenData(TypedDict):
auth_header_name: str
jwt_token: str
timestamp: float
expires_at: ReadOnly[NotRequired[float]]
refresh_token: ReadOnly[NotRequired[str]]
client_id: ReadOnly[NotRequired[str]]
token_endpoint: ReadOnly[NotRequired[str]]
revocation_endpoint: ReadOnly[NotRequired[str]]
resource: ReadOnly[NotRequired[str]]
team_id: ReadOnly[NotRequired[str | None]]
class CliTeam(TypedDict, total=False):
@ -46,6 +62,8 @@ class CliTeam(TypedDict, total=False):
class CliContextObj(TypedDict):
base_url: str
base_url_explicit: NotRequired[bool]
api_key: ReadOnly[NotRequired[str | None]]
api_key_from_token_file: ReadOnly[NotRequired[bool]]
class CliPollData(TypedDict, total=False):
@ -79,10 +97,7 @@ class CliAuthResult(TypedDict):
# Token storage utilities
def get_token_file_path() -> str:
"""Get the path to store the authentication token"""
home_dir: Final = Path.home()
config_dir: Final = home_dir / ".litellm"
config_dir.mkdir(exist_ok=True)
return str(config_dir / "token.json")
return str(Path.home() / ".litellm" / "token.json")
def save_token(token_data: CliTokenData) -> None:
@ -115,11 +130,23 @@ def get_stored_api_key(expected_base_url: str | None = None) -> str | None:
If expected_base_url is provided, the key is only returned when it was
originally issued for that URL. This prevents credential leakage when the
CLI is pointed at a different (possibly malicious) server.
CLI is pointed at a different (possibly malicious) server. A key obtained by
``lite login --pkce`` is refreshed here once it nears expiry.
"""
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
token_data: Final = load_token()
if token_data is None:
return None
if expected_base_url is not None and token_data.get("base_url") != expected_base_url.rstrip("/"):
return None
return fresh_api_key(token_data, save_token, requests.Session(), reload=load_token, warn=_warn)
return get_litellm_gateway_api_key(expected_base_url=expected_base_url)
def _warn(message: str) -> None:
click.echo(message, err=True)
def _login_command(renews: bool) -> str:
return "lite login --pkce" if renews else "lite login"
# Team selection utilities
@ -645,6 +672,41 @@ def _configure_claude_code(base_url: str) -> None:
click.echo("Your other Claude Code settings were left untouched. Restart Claude Code to pick this up.")
def _finish_login(base_url: str, api_key: str, config_claude: bool) -> None:
from litellm.proxy.client.cli.interface import show_commands
click.echo("\nLogin successful!")
click.echo(f"JWT Token: {api_key[:20]}...")
click.echo("You can now use the CLI without specifying --api-key")
if config_claude:
_configure_claude_code(base_url)
click.echo("\n" + "=" * 60)
show_commands()
def _replace_stored_token(record: CliTokenData, http: Http) -> None:
previous: Final = load_token()
save_token(record)
if previous is None:
return
revocation: Final = revoke_stored_credential(previous, http)
if revocation is not None:
click.echo(
f"Could not revoke the previous login's refresh token on the proxy ({revocation.reason}); "
"it expires on its own."
)
def _pkce_login(base_url: str, config_claude: bool) -> None:
http: Final = requests.Session()
credential: Final = run_pkce_login(base_url, http, echo=click.echo)
if isinstance(credential, PkceFailure):
click.echo(f"Authentication failed: {credential.reason}")
return
_replace_stored_token(pkce_token_record(base_url, credential), http)
_finish_login(base_url, credential.access_token, config_claude)
@click.command(name="login")
@click.option(
"--config-claude",
@ -655,16 +717,28 @@ def _configure_claude_code(base_url: str) -> None:
"Unrelated settings are preserved."
),
)
@click.option(
"--pkce",
is_flag=True,
default=False,
help=(
"Sign in with OAuth authorization code + PKCE through your system browser (loopback redirect), "
"with a refresh token that renews the key automatically. Requires a proxy that serves "
"/.well-known/litellm-cli-auth."
),
)
@click.pass_context
def login(ctx: click.Context, config_claude: bool):
def login(ctx: click.Context, config_claude: bool, pkce: bool) -> None:
"""Login to LiteLLM proxy using SSO authentication"""
from litellm.constants import LITELLM_CLI_SOURCE_IDENTIFIER
from litellm.proxy.client.cli.interface import show_commands
ctx_obj: Final[CliContextObj] = ctx.obj
base_url: Final = ctx_obj["base_url"]
try:
if pkce:
_pkce_login(base_url, config_claude)
return
cli_sso_flow: Final = _start_cli_sso_flow(base_url=base_url)
key_id: Final = cli_sso_flow["login_id"]
poll_secret: Final = cli_sso_flow["poll_secret"]
@ -691,7 +765,7 @@ def login(ctx: click.Context, config_claude: bool):
# Save token data. base_url is stored so we can verify origin
# before reusing the key on a subsequent CLI invocation.
save_token(
_replace_stored_token(
{
"base_url": base_url.rstrip("/"),
"key": api_key,
@ -701,19 +775,11 @@ def login(ctx: click.Context, config_claude: bool):
"auth_header_name": "Authorization",
"jwt_token": "",
"timestamp": time.time(),
}
},
requests.Session(),
)
click.echo("\nLogin successful!")
click.echo(f"JWT Token: {api_key[:20]}...")
click.echo("You can now use the CLI without specifying --api-key")
if config_claude:
_configure_claude_code(base_url)
# Show available commands after successful login
click.echo("\n" + "=" * 60)
show_commands()
_finish_login(base_url, api_key, config_claude)
return
else:
click.echo("Authentication timed out. Please try again.")
@ -738,7 +804,20 @@ def login(ctx: click.Context, config_claude: bool):
@click.command(name="logout")
def logout():
"""Logout and clear stored authentication"""
clear_token()
token_data: Final = load_token()
revocation: Final = revoke_stored_credential(token_data, requests.Session()) if token_data is not None else None
match revocation:
case RevocationUnavailable(reason=reason):
raise click.ClickException(
f"The proxy could not record the revocation ({reason}). Nothing was cleared; run `lite logout` again shortly."
)
case PkceFailure(reason=reason):
clear_token()
click.echo(f"Could not revoke the refresh token on the proxy ({reason}); it expires on its own.")
case None:
clear_token()
case _:
assert_never(revocation)
click.echo("Logged out successfully. Authentication token cleared.")
@ -750,8 +829,9 @@ def print_token(ctx: click.Context):
Designed to be used as Claude Code's `apiKeyHelper`
(https://docs.claude.com/en/docs/claude-code/settings): stdout must
contain only the token, so all diagnostics go to stderr. The token
expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); once
expired, run `lite login` again.
expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); a
`lite login --pkce` token renews itself here first, and once a token
has expired for good, run the same `lite login` command again.
"""
token_data: Final = load_token()
if not token_data:
@ -763,19 +843,23 @@ def print_token(ctx: click.Context):
# actually issued this token for -- that's the whole point of not
# needing a wrapper command.
ctx_obj: Final[CliContextObj] = ctx.obj
if ctx_obj.get("base_url_explicit"):
base_url: Final = ctx_obj["base_url"]
if token_data.get("base_url") != base_url.rstrip("/"):
click.echo("Not authenticated for this server. Run 'lite login'.", err=True)
sys.exit(1)
issued_for_this_server: Final = token_data.get("base_url") == ctx_obj.get("base_url", "").rstrip("/")
if ctx_obj.get("base_url_explicit") and not issued_for_this_server:
click.echo("Not authenticated for this server. Run 'lite login'.", err=True)
sys.exit(1)
if not is_cli_token_fresh(token_data):
renews: Final = "refresh_token" in token_data
if not is_cli_token_fresh(token_data) and not renews:
click.echo("Token expired. Run 'lite login' again.", err=True)
sys.exit(1)
api_key: Final = token_data.get("key")
api_key: Final = (
ctx_obj.get("api_key")
if issued_for_this_server and ctx_obj.get("api_key_from_token_file")
else fresh_api_key(token_data, save_token, requests.Session(), reload=load_token, warn=_warn)
)
if not api_key:
click.echo("No token available. Run 'lite login'.", err=True)
click.echo(f"Key expired. Run '{_login_command(renews)}' again.", err=True)
sys.exit(1)
click.echo(api_key)
@ -794,16 +878,29 @@ def whoami():
click.echo(f"User Email: {token_data.get('user_email', 'Unknown')}")
click.echo(f"User ID: {token_data.get('user_id', 'Unknown')}")
click.echo(f"User Role: {token_data.get('user_role', 'Unknown')}")
team_id: Final = token_data.get("team_id")
if team_id:
click.echo(f"Team ID: {team_id}")
# Check if token is still valid (basic timestamp check)
timestamp: Final = token_data.get("timestamp", 0)
age_hours: Final = (time.time() - timestamp) / 3600
click.echo(f"Token age: {age_hours:.1f} hours")
if age_hours > CLI_JWT_EXPIRATION_HOURS:
expires_at: Final = token_data.get("expires_at")
if isinstance(expires_at, (int, float)):
click.echo(_key_expiry_line(expires_at, renews="refresh_token" in token_data))
elif age_hours > CLI_JWT_EXPIRATION_HOURS:
click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
def _key_expiry_line(expires_at: float, renews: bool) -> str:
remaining_hours: Final = (expires_at - time.time()) / 3600
if remaining_hours <= 0:
return f"Key expired. Run '{_login_command(renews)}' again"
status: Final = f"Key expires in: {remaining_hours:.1f} hours"
return f"{status}, renewed on next use" if renews else status
@click.group(name="auth")
def auth_group():
"""Manage CLI authentication (apiKeyHelper support, etc.)"""

573
litellm/proxy/client/cli/commands/pkce_login.py Normal file
View file

@ -0,0 +1,573 @@
"""Browser sign-in for ``lite login --pkce``: OAuth 2.1 authorization code + PKCE S256
against the proxy's own authorization server, as a public client on a loopback redirect.
The proxy publishes everything this needs at ``/.well-known/litellm-cli-auth``, so a CLI
in any other language can run the same steps from that document alone."""
from __future__ import annotations
import hashlib
import secrets
import socket
import threading
import time
import webbrowser
from base64 import urlsafe_b64encode
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from http.server import BaseHTTPRequestHandler, HTTPServer
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol
from urllib.parse import parse_qs, urlencode, urlparse
import requests
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.cli_token_utils import CLI_TOKEN_FRESHNESS_BUFFER_SECONDS
if TYPE_CHECKING:
from .auth import CliTokenData
CLI_AUTH_DISCOVERY_PATH: Final = "/.well-known/litellm-cli-auth"
CALLBACK_PATH: Final = "/callback"
LOGIN_TIMEOUT_SECONDS: Final = 300
_HTTP_TIMEOUT_SECONDS: Final = 15
_CLIENT_NAME: Final = "litellm-cli"
class CliAuthContract(BaseModel):
model_config = ConfigDict(frozen=True)
contract_version: Literal[1]
issuer: str
authorization_endpoint: str
token_endpoint: str
registration_endpoint: str
revocation_endpoint: str
resource: str
code_challenge_methods_supported: tuple[str, ...]
class _RegisteredClient(BaseModel):
model_config = ConfigDict(frozen=True)
client_id: str = Field(min_length=1)
class _TokenResponse(BaseModel):
model_config = ConfigDict(frozen=True)
access_token: str = Field(min_length=1)
expires_in: int = Field(gt=0)
refresh_token: str = Field(min_length=1)
user_id: str | None = None
team_id: str | None = None
@dataclass(frozen=True, slots=True)
class PkceFailure:
reason: str
@dataclass(frozen=True, slots=True)
class RevocationUnavailable:
reason: str
@dataclass(frozen=True, slots=True)
class PkceCredential:
access_token: str
refresh_token: str
expires_at: float
client_id: str
token_endpoint: str
revocation_endpoint: str
resource: str
user_id: str | None
team_id: str | None
@dataclass(frozen=True, slots=True)
class CallbackCode:
code: str
@dataclass(frozen=True, slots=True)
class CallbackDenied:
error: str
description: str | None
CallbackOutcome = CallbackCode | CallbackDenied
class Http(Protocol):
def get(self, url: str, *, timeout: float) -> requests.Response: ...
def post(
self,
url: str,
*,
data: Mapping[str, str] | None = None,
json: Mapping[str, object] | None = None,
timeout: float,
allow_redirects: bool,
) -> requests.Response: ...
class LoopbackServer(HTTPServer):
"""The OS-assigned loopback listener the browser is sent back to. Only the response
carrying the pending sign-in's ``state`` settles it; anything else (a stray request, a
stale tab, an attacker poking the port) gets a 400 and the wait continues. A connection
that opens and then sends nothing is dropped after ``connection_timeout_seconds`` so it
cannot hold the single-threaded wait past its deadline."""
def __init__(self, expected_state: str, connection_timeout_seconds: float = 5) -> None:
super().__init__(("127.0.0.1", 0), _CallbackHandler)
self.expected_state: Final = expected_state
self.connection_timeout_seconds: Final = connection_timeout_seconds
self.outcome: CallbackOutcome | None = None
self.timeout = 1
@property
def redirect_uri(self) -> str:
return f"http://127.0.0.1:{self.server_address[1]}{CALLBACK_PATH}"
def get_request(self) -> tuple[socket.socket, object]:
accepted: Final[tuple[socket.socket, object]] = super().get_request()
accepted[0].settimeout(self.connection_timeout_seconds)
return accepted
def wait(
self, timeout_seconds: float, clock: Callable[[], float] = time.monotonic
) -> CallbackOutcome | PkceFailure:
deadline: Final = clock() + timeout_seconds
while self.outcome is None:
if clock() >= deadline:
return PkceFailure("timed out waiting for the browser sign-in to finish")
self.handle_request()
return self.outcome
class _CallbackHandler(BaseHTTPRequestHandler):
server: LoopbackServer # pyright: ignore[reportIncompatibleVariableOverride] # only ever constructed by LoopbackServer
def do_GET(self) -> None:
parsed: Final = urlparse(self.path)
if parsed.path != CALLBACK_PATH:
self._respond(404, "Not found.")
return
params: Final = parse_qs(parsed.query)
if _first(params, "state") != self.server.expected_state:
self._respond(400, "This response does not belong to the pending sign-in; still waiting.")
return
error: Final = _first(params, "error")
if error is not None:
self.server.outcome = CallbackDenied(error=error, description=_first(params, "error_description"))
self._respond(200, "Sign-in was not approved. You can close this window.")
return
code: Final = _first(params, "code")
if code is None:
self._respond(400, "The sign-in response carried no authorization code; still waiting.")
return
self.server.outcome = CallbackCode(code=code)
self._respond(200, "Signed in to LiteLLM. You can close this window and return to the terminal.")
def log_message(self, format: str, *args: object) -> None:
return
def _respond(self, status: int, text: str) -> None:
body: Final = text.encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def _first(params: Mapping[str, Sequence[str]], key: str) -> str | None:
values: Final = params.get(key)
return values[0] if values else None
def discover_cli_auth(base_url: str, http: Http) -> CliAuthContract | PkceFailure:
url: Final = f"{base_url.rstrip('/')}{CLI_AUTH_DISCOVERY_PATH}"
try:
response: Final = http.get(url, timeout=_HTTP_TIMEOUT_SECONDS)
except requests.RequestException as exc:
return PkceFailure(f"could not reach {url}: {exc}")
if response.status_code != 200:
return PkceFailure(
f"{url} answered {response.status_code}; this proxy version does not support `lite login --pkce`"
)
try:
contract: Final = CliAuthContract.model_validate(response.json())
except (ValueError, ValidationError) as exc:
return PkceFailure(f"{url} returned an unsupported discovery document: {exc}")
if "S256" not in contract.code_challenge_methods_supported:
return PkceFailure("the proxy does not support PKCE S256")
if _canonical_url(contract.issuer) != _canonical_url(base_url):
return PkceFailure(f"{url} is issued for {contract.issuer}, not {base_url}; pass that address as --base-url")
foreign: Final = _endpoints_outside(contract, _origin(base_url))
if foreign:
return PkceFailure(
f"{url} names endpoints outside {base_url} ({', '.join(foreign)}); refusing to send credentials there"
)
return contract
def _endpoints_outside(contract: CliAuthContract, origin: str | None) -> tuple[str, ...]:
endpoints: Final = (
contract.authorization_endpoint,
contract.token_endpoint,
contract.registration_endpoint,
contract.revocation_endpoint,
contract.resource,
)
return tuple(endpoint for endpoint in endpoints if origin is None or _origin(endpoint) != origin)
def _origin(url: str) -> str | None:
"""``scheme://host:port`` with the default port made explicit, so the same server spelled
two ways (``https://llm.example.com`` and ``https://LLM.example.com:443/``) compares equal
and two different servers never do."""
parsed: Final = urlparse(url)
try:
port: Final = parsed.port
except ValueError:
return None
if parsed.scheme not in ("http", "https") or not parsed.hostname:
return None
host: Final = f"[{parsed.hostname}]" if ":" in parsed.hostname else parsed.hostname
return f"{parsed.scheme}://{host}:{port or (443 if parsed.scheme == 'https' else 80)}"
def _canonical_url(url: str) -> str | None:
"""The origin plus the path with its trailing slash dropped: the RFC 8414 section 3.3
identity check, so a document can only ever be accepted for the proxy it was fetched from."""
origin: Final = _origin(url)
return None if origin is None else f"{origin}{urlparse(url).path.rstrip('/')}"
class _ClientRegistration(TypedDict):
client_name: ReadOnly[str]
redirect_uris: ReadOnly[tuple[str, ...]]
grant_types: ReadOnly[tuple[str, ...]]
response_types: ReadOnly[tuple[str, ...]]
token_endpoint_auth_method: ReadOnly[Literal["none"]]
def _form(**fields: str) -> Mapping[str, str]:
return MappingProxyType(fields)
def _refused_redirect(request_name: str, response: requests.Response) -> PkceFailure | None:
"""Every POST to the proxy is sent with ``allow_redirects=False``: a 307 or 308 would make
``requests`` replay the form, code and verifier or refresh token included, wherever ``Location``
points, past the origin check discovery passed."""
if not 300 <= response.status_code < 400:
return None
return PkceFailure(
f"{request_name} redirected to {response.headers.get('Location', 'another address')}; refusing to follow it"
)
def register_client(contract: CliAuthContract, redirect_uri: str, http: Http) -> str | PkceFailure:
registration: Final[_ClientRegistration] = {
"client_name": _CLIENT_NAME,
"redirect_uris": (redirect_uri,),
"grant_types": ("authorization_code", "refresh_token"),
"response_types": ("code",),
"token_endpoint_auth_method": "none",
}
try:
response: Final = http.post(
contract.registration_endpoint, json=registration, timeout=_HTTP_TIMEOUT_SECONDS, allow_redirects=False
)
except requests.RequestException as exc:
return PkceFailure(f"client registration failed: {exc}")
redirected: Final = _refused_redirect("client registration", response)
if redirected is not None:
return redirected
if response.status_code not in (200, 201):
return PkceFailure(f"client registration failed with {response.status_code}: {_error_detail(response)}")
try:
return _RegisteredClient.model_validate(response.json()).client_id
except (ValueError, ValidationError) as exc:
return PkceFailure(f"client registration returned an unexpected body: {exc}")
def pkce_pair() -> tuple[str, str]:
verifier: Final = secrets.token_urlsafe(64)
digest: Final = hashlib.sha256(verifier.encode("ascii")).digest()
return verifier, urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")
def authorize_url(contract: CliAuthContract, client_id: str, redirect_uri: str, state: str, code_challenge: str) -> str:
query: Final = urlencode(
_form(
response_type="code",
client_id=client_id,
redirect_uri=redirect_uri,
state=state,
code_challenge=code_challenge,
code_challenge_method="S256",
resource=contract.resource,
)
)
return f"{contract.authorization_endpoint}?{query}"
def redeem_code(
contract: CliAuthContract,
client_id: str,
redirect_uri: str,
code: str,
code_verifier: str,
http: Http,
now: Callable[[], float] = time.time,
) -> PkceCredential | PkceFailure:
return _token_request(
token_endpoint=contract.token_endpoint,
revocation_endpoint=contract.revocation_endpoint,
resource=contract.resource,
client_id=client_id,
form=_form(
grant_type="authorization_code",
code=code,
redirect_uri=redirect_uri,
client_id=client_id,
code_verifier=code_verifier,
resource=contract.resource,
),
http=http,
now=now,
)
def refresh_credential(
token_endpoint: str,
revocation_endpoint: str,
resource: str,
client_id: str,
refresh_token: str,
http: Http,
now: Callable[[], float] = time.time,
) -> PkceCredential | PkceFailure:
return _token_request(
token_endpoint=token_endpoint,
revocation_endpoint=revocation_endpoint,
resource=resource,
client_id=client_id,
form=_form(grant_type="refresh_token", refresh_token=refresh_token, client_id=client_id, resource=resource),
http=http,
now=now,
)
def _token_request(
token_endpoint: str,
revocation_endpoint: str,
resource: str,
client_id: str,
form: Mapping[str, str],
http: Http,
now: Callable[[], float],
) -> PkceCredential | PkceFailure:
try:
response: Final = http.post(token_endpoint, data=form, timeout=_HTTP_TIMEOUT_SECONDS, allow_redirects=False)
except requests.RequestException as exc:
return PkceFailure(f"token request failed: {exc}")
redirected: Final = _refused_redirect("token request", response)
if redirected is not None:
return redirected
if response.status_code != 200:
return PkceFailure(f"token request failed with {response.status_code}: {_error_detail(response)}")
try:
token: Final = _TokenResponse.model_validate(response.json())
except (ValueError, ValidationError) as exc:
return PkceFailure(f"token endpoint returned an unexpected body: {exc}")
return PkceCredential(
access_token=token.access_token,
refresh_token=token.refresh_token,
expires_at=now() + token.expires_in,
client_id=client_id,
token_endpoint=token_endpoint,
revocation_endpoint=revocation_endpoint,
resource=resource,
user_id=token.user_id,
team_id=token.team_id,
)
def revoke_credential(
revocation_endpoint: str, client_id: str, refresh_token: str, http: Http
) -> PkceFailure | RevocationUnavailable | None:
try:
response: Final = http.post(
revocation_endpoint,
data=_form(token=refresh_token, token_type_hint="refresh_token", client_id=client_id),
timeout=_HTTP_TIMEOUT_SECONDS,
allow_redirects=False,
)
except requests.RequestException as exc:
return PkceFailure(f"revocation request failed: {exc}")
redirected: Final = _refused_redirect("revocation request", response)
if redirected is not None:
return redirected
if response.status_code == 503:
return RevocationUnavailable(f"revocation failed with 503: {_error_detail(response)}")
if response.status_code != 200:
return PkceFailure(f"revocation failed with {response.status_code}: {_error_detail(response)}")
return None
_ERROR_BODY: Final = TypeAdapter(Mapping[str, object])
def _error_detail(response: requests.Response) -> str:
try:
body: Final = _ERROR_BODY.validate_json(response.content)
except ValidationError:
return response.text[:200]
return str(body.get("error_description") or body.get("error") or body.get("detail") or body)[:200]
def run_pkce_login(
base_url: str,
http: Http,
open_browser: Callable[[str], object] = webbrowser.open,
echo: Callable[[str], None] = print,
timeout_seconds: float = LOGIN_TIMEOUT_SECONDS,
) -> PkceCredential | PkceFailure:
contract: Final = discover_cli_auth(base_url, http)
if isinstance(contract, PkceFailure):
return contract
state: Final = secrets.token_urlsafe(32)
verifier, challenge = pkce_pair()
with LoopbackServer(state) as server:
client_id: Final = register_client(contract, server.redirect_uri, http)
if isinstance(client_id, PkceFailure):
return client_id
url: Final = authorize_url(contract, client_id, server.redirect_uri, state, challenge)
echo(f"Opening browser to: {url}")
echo("Approve the sign-in in your browser. Waiting...")
threading.Thread(target=open_browser, args=(url,), name="lite-login-browser", daemon=True).start()
outcome: Final = server.wait(timeout_seconds)
match outcome:
case PkceFailure():
return outcome
case CallbackDenied():
return PkceFailure(f"sign-in was not approved ({outcome.error}): {outcome.description or 'no details'}")
case CallbackCode():
return redeem_code(contract, client_id, server.redirect_uri, outcome.code, verifier, http)
def pkce_token_record(base_url: str, credential: PkceCredential) -> CliTokenData:
record: Final[CliTokenData] = {
"base_url": base_url.rstrip("/"),
"key": credential.access_token,
"user_id": credential.user_id or "cli-user",
"user_email": "unknown",
"user_role": "cli",
"auth_header_name": "Authorization",
"jwt_token": "",
"timestamp": time.time(),
"expires_at": credential.expires_at,
"refresh_token": credential.refresh_token,
"client_id": credential.client_id,
"token_endpoint": credential.token_endpoint,
"revocation_endpoint": credential.revocation_endpoint,
"resource": credential.resource,
"team_id": credential.team_id,
}
return record
def _ignore_warning(_message: str) -> None:
return None
def fresh_api_key(
token_data: Mapping[str, object],
save: Callable[[CliTokenData], None],
http: Http,
*,
reload: Callable[[], Mapping[str, object] | None],
now: Callable[[], float] = time.time,
warn: Callable[[str], None] = _ignore_warning,
) -> str | None:
"""The stored key, refreshed first when it is about to expire and a refresh token is
on file. The refresh fires at the same moment ``is_cli_token_fresh`` stops calling the
key fresh, so a command that checks freshness and then asks for the key never disagrees
with itself. The rotated pair is saved before the new key is returned, so a crash after
this point never strands the CLI with a burned refresh token. A refresh that fails
reads the record again, because a sibling ``lite`` process may have rotated the pair
first, in which case the key it saved for this same proxy is the live one; when no sibling
did, the reason the proxy gave goes to ``warn`` so a revoked or refused refresh token is
never a silent failure. A record without ``expires_at`` (the classic ``lite login``
credential) is returned as stored."""
key: Final = token_data.get("key")
if not isinstance(key, str) or not key:
return None
expires_at: Final = token_data.get("expires_at")
if not isinstance(expires_at, (int, float)):
return key
if now() < expires_at - CLI_TOKEN_FRESHNESS_BUFFER_SECONDS:
return key
still_valid: Final = key if now() < expires_at else None
refresh_inputs: Final = _refresh_inputs(token_data)
if refresh_inputs is None:
return still_valid
refreshed: Final = refresh_credential(*refresh_inputs, http=http, now=now)
if isinstance(refreshed, PkceFailure):
sibling_key: Final = _key_rotated_by_a_sibling(reload(), token_data, now())
if sibling_key is None:
warn(f"Could not renew the key: {refreshed.reason}")
return sibling_key or still_valid
base_url: Final = token_data.get("base_url")
save(pkce_token_record(base_url if isinstance(base_url, str) else "", refreshed))
return refreshed.access_token
_CREDENTIAL_IDENTITY_FIELDS: Final = ("base_url", "token_endpoint", "resource", "user_id", "team_id")
def _key_rotated_by_a_sibling(
record: Mapping[str, object] | None, token_data: Mapping[str, object], now: float
) -> str | None:
"""The key a sibling process saved, but only when it continues this very credential:
same proxy, same token endpoint, same resource, same user and team, and not yet expired.
A concurrent ``lite login`` against a different proxy, or as someone else on this one,
replaces the same file, and its key must never be sent as this credential."""
if record is None or record.get("refresh_token") == token_data.get("refresh_token"):
return None
if any(record.get(field) != token_data.get(field) for field in _CREDENTIAL_IDENTITY_FIELDS):
return None
expires_at: Final = record.get("expires_at")
if not isinstance(expires_at, (int, float)) or now >= expires_at:
return None
key: Final = record.get("key")
return key if isinstance(key, str) and key else None
def _refresh_inputs(token_data: Mapping[str, object]) -> tuple[str, str, str, str, str] | None:
values: Final = tuple(
token_data.get(field)
for field in ("token_endpoint", "revocation_endpoint", "resource", "client_id", "refresh_token")
)
if not all(isinstance(value, str) and value for value in values):
return None
token_endpoint, revocation_endpoint, resource, client_id, refresh_token = values
return str(token_endpoint), str(revocation_endpoint), str(resource), str(client_id), str(refresh_token)
def revoke_stored_credential(
token_data: Mapping[str, object], http: Http
) -> PkceFailure | RevocationUnavailable | None:
refresh_inputs: Final = _refresh_inputs(token_data)
if refresh_inputs is None:
return None
_, revocation_endpoint, _, client_id, refresh_token = refresh_inputs
return revoke_credential(revocation_endpoint, client_id, refresh_token, http)

35
litellm/proxy/client/cli/commands/up.py
View file

@ -17,7 +17,7 @@ from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh
from .agents import AgentRunError, resolve_api_key, verify_proxy_key
from .auth import load_token, login
from .auth import CliContextObj, get_stored_api_key, load_token, login
from .claude_settings import (
BACKUP_PATH,
CLAUDE_SETTINGS_PATH,
@ -103,22 +103,41 @@ def restore_claude_settings(settings_path: Path | None = None, backup_path: Path
return record
def _usable_login(api_key: str | None) -> bool:
if api_key is None:
return False
token_data: Final = load_token()
return token_data is not None and is_cli_token_fresh(token_data)
def _key_resolved_on_the_way_in(ctx_obj: CliContextObj, base_url: str) -> str | None:
if ctx_obj.get("api_key_from_token_file"):
return ctx_obj.get("api_key")
return get_stored_api_key(expected_base_url=base_url)
def _stored_login_is_pkce() -> bool:
token_data: Final = load_token()
return token_data is not None and "refresh_token" in token_data
def _ensure_fresh_login(ctx: click.Context) -> None:
base_url: Final = ctx.obj["base_url"].rstrip("/")
token_data = load_token()
if token_data and token_data.get("base_url") == base_url and is_cli_token_fresh(token_data):
ctx_obj: Final[CliContextObj] = ctx.obj
base_url: Final = ctx_obj["base_url"].rstrip("/")
if _usable_login(_key_resolved_on_the_way_in(ctx_obj, base_url)):
return
pkce: Final = _stored_login_is_pkce()
login_command: Final = "lite login --pkce" if pkce else "lite login"
if not sys.stdin.isatty():
raise UpError(
"No fresh LiteLLM login found for this proxy. Run `lite login` first (apiKeyHelper "
f"No fresh LiteLLM login found for this proxy. Run `{login_command}` first (apiKeyHelper "
"reads this token on every Claude Code request)."
)
click.echo("No fresh LiteLLM login found for this proxy; starting login...")
ctx.invoke(login)
token_data = load_token()
if not token_data or token_data.get("base_url") != base_url or not is_cli_token_fresh(token_data):
ctx.invoke(login, pkce=pkce)
if not _usable_login(get_stored_api_key(expected_base_url=base_url)):
raise UpError("Login did not produce a usable token; cannot start `lite up`.")

9
litellm/proxy/client/cli/main.py
View file

@ -93,11 +93,12 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s
# If no API key provided via flag or environment variable, try to load from saved token.
# Pass base_url so we only use the stored key when it was issued for this server.
if api_key is None:
api_key = get_stored_api_key(expected_base_url=base_url)
api_key_from_token_file: Final = api_key is None
resolved_api_key: Final = get_stored_api_key(expected_base_url=base_url) if api_key_from_token_file else api_key
ctx.obj["base_url"] = base_url
ctx.obj["api_key"] = api_key
ctx.obj["api_key"] = resolved_api_key
ctx.obj["api_key_from_token_file"] = api_key_from_token_file
# `--base-url` defaults to localhost:4000 for local dev convenience, but
# apiKeyHelper is invoked bare (no flags) -- commands that must work
# unattended (print-token) need to tell "user didn't say" apart from
@ -107,7 +108,7 @@ def cli(ctx: click.Context, show_version: bool, base_url: str | None, api_key: s
ctx.obj["base_url_explicit"] = base_url_provided or bool(stored_base_url)
if show_version:
print_version(base_url, api_key)
print_version(base_url, resolved_api_key)
ctx.exit()
# If no subcommand was invoked, start interactive mode

91
litellm/proxy/common_utils/html_forms/native_client_consent.py Normal file
View file

@ -0,0 +1,91 @@
from collections.abc import Sequence
from html import escape
from typing import Final
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
def render_native_client_consent_page(
*,
client_origin: str,
user_id: str,
teams: Sequence[tuple[str, str]],
flow_handle: str,
complete_url: str,
) -> str:
"""The consent page a native client's sign-in lands on: who is signed in, which
loopback client asked, which team the credential is attributed to, and an explicit
Approve or Deny that POSTs back to ``complete_url``. Every value is client- or
user-influenced and HTML-escaped; the flow handle travels only in the form body."""
return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="referrer" content="no-referrer">
<title>Authorize CLI access - LiteLLM</title>
<style>
body {{
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
background-color: #f8fafc;
margin: 0;
padding: 20px;
display: flex;
justify-content: center;
align-items: center;
min-height: 100vh;
color: #1e293b;
}}
.container {{
background-color: #fff;
padding: 40px;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
width: 450px;
max-width: 100%;
}}
h1 {{ margin: 0 0 16px; font-size: 24px; font-weight: 600; }}
p {{ margin: 0 0 12px; line-height: 1.5; }}
code {{ background: #f1f5f9; padding: 2px 6px; border-radius: 4px; }}
label {{ display: block; margin: 16px 0 6px; font-weight: 600; }}
select {{ width: 100%; padding: 8px; border: 1px solid #cbd5e1; border-radius: 6px; font-size: 14px; }}
.actions {{ display: flex; gap: 12px; margin-top: 24px; }}
button {{ flex: 1; padding: 10px; border-radius: 6px; font-size: 15px; cursor: pointer; border: 1px solid #cbd5e1; }}
.approve {{ background: #2563eb; color: #fff; border-color: #2563eb; }}
.deny {{ background: #fff; color: #1e293b; }}
</style>
</head>
<body>
<div class="container">
<h1>Authorize CLI access</h1>
<p>A command-line client at <code>{escape(client_origin)}</code> wants to call LiteLLM as <strong>{escape(user_id)}</strong>.</p>
<p>Approving issues it a personal credential that expires within {CLI_JWT_EXPIRATION_HOURS} hours. <code>lite logout</code> stops it from being renewed. Only approve if you started this sign-in yourself.</p>
<form method="post" action="{escape(complete_url)}">
<input type="hidden" name="flow" value="{escape(flow_handle)}">
{_team_field(teams)}
<div class="actions">
<button type="submit" name="decision" value="deny" class="deny">Deny</button>
<button type="submit" name="decision" value="approve" class="approve">Approve</button>
</div>
</form>
</div>
</body>
</html>
"""
def _team_field(teams: Sequence[tuple[str, str]]) -> str:
if not teams:
return ""
if len(teams) == 1:
team_id, team_label = teams[0]
return (
f'<input type="hidden" name="team_id" value="{escape(team_id)}">'
f"<p>Requests are attributed to team <strong>{escape(team_label)}</strong>.</p>"
)
options: Final = "".join(
f'<option value="{escape(team_id)}">{escape(team_label)}</option>' for team_id, team_label in teams
)
return (
f'<label for="team_id">Attribute requests to team</label><select id="team_id" name="team_id">{options}</select>'
)

22
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -270,7 +270,7 @@ class _TeamRowGrants(BaseModel):
litellm_model_table: _TeamModelAliasTable | None = None
class _CliSsoTeamDetail(BaseModel):
class CliSsoTeamDetail(BaseModel):
"""The per-team snapshot cached in the CLI SSO flow and echoed to the CLI on poll."""
team_id: str | None = None
@ -279,8 +279,8 @@ class _CliSsoTeamDetail(BaseModel):
team_model_aliases: Mapping[str, str] | None = None
_CLI_SSO_TEAM_DETAILS_ADAPTER: Final = TypeAdapter(tuple[_CliSsoTeamDetail, ...])
_TEAMLESS_CLI_SSO_TEAM_DETAIL: Final = _CliSsoTeamDetail(team_models=())
_CLI_SSO_TEAM_DETAILS_ADAPTER: Final = TypeAdapter(tuple[CliSsoTeamDetail, ...])
_TEAMLESS_CLI_SSO_TEAM_DETAIL: Final = CliSsoTeamDetail(team_models=())
class _CustomSsoCall(Protocol):
@ -2192,10 +2192,10 @@ async def _build_cli_sso_user_defined_values(
)
def _cli_sso_team_detail(team_row: Mapping[str, object]) -> _CliSsoTeamDetail:
def _cli_sso_team_detail(team_row: Mapping[str, object]) -> CliSsoTeamDetail:
team: Final = _TeamRowGrants.model_validate(team_row)
alias_table: Final = team.litellm_model_table
return _CliSsoTeamDetail(
return CliSsoTeamDetail(
team_id=team.team_id,
team_alias=team.team_alias,
team_models=team.models,
@ -2203,10 +2203,10 @@ def _cli_sso_team_detail(team_row: Mapping[str, object]) -> _CliSsoTeamDetail:
)
async def _fetch_cli_sso_team_details(
async def fetch_cli_sso_team_details(
prisma_client: PrismaClient,
teams: Sequence[str],
) -> tuple[_CliSsoTeamDetail, ...] | None:
) -> tuple[CliSsoTeamDetail, ...] | None:
"""``None`` means the lookup itself failed, which is not the same as the user having no teams."""
if not teams:
return ()
@ -2221,7 +2221,7 @@ async def _fetch_cli_sso_team_details(
return tuple(_cli_sso_team_detail(team_row.model_dump()) for team_row in prisma_teams)
def _cli_sso_session_teams(team_details: Sequence[_CliSsoTeamDetail]) -> list[str]:
def _cli_sso_session_teams(team_details: Sequence[CliSsoTeamDetail]) -> list[str]:
"""The teams a login may bind to: only those whose row still exists.
A team deleted out from under a membership, which is what deleting an organization
@ -2231,7 +2231,7 @@ def _cli_sso_session_teams(team_details: Sequence[_CliSsoTeamDetail]) -> list[st
return [detail.team_id for detail in team_details if detail.team_id is not None]
def _selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> _CliSsoTeamDetail | None:
def selected_cli_sso_team_detail(team_details: object, team_id: str | None) -> CliSsoTeamDetail | None:
"""``None`` means the team's grants are unknown. An empty grant is a real value meaning unrestricted,
so an unknown one must not be minted as empty."""
if team_id is None:
@ -2282,7 +2282,7 @@ async def _complete_cli_sso_callback_session(
if hasattr(user_info, "teams") and user_info.teams:
teams = user_info.teams if isinstance(user_info.teams, list) else []
team_details: Final = await _fetch_cli_sso_team_details(prisma_client=prisma_client, teams=teams)
team_details: Final = await fetch_cli_sso_team_details(prisma_client=prisma_client, teams=teams)
if team_details is None:
raise HTTPException(
status_code=500,
@ -2483,7 +2483,7 @@ async def cli_poll_key(
# If no team_id provided and user has 0 or 1 team, use first team (or None)
team_id = user_teams[0] if len(user_teams) > 0 else None
selected_team: Final = _selected_cli_sso_team_detail(
selected_team: Final = selected_cli_sso_team_detail(
team_details=user_team_details,
team_id=team_id,
)

173
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -9,8 +9,9 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc.
import json
import os
import re
from collections.abc import Callable
from types import MappingProxyType
from typing import Annotated, Any, Final, cast
from typing import TYPE_CHECKING, Annotated, Any, Final, cast
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket
@ -55,11 +56,15 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
)
from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
from .passthrough_endpoint_router import PassthroughEndpointRouter
if TYPE_CHECKING:
from litellm.router import Router
vertex_llm_base: Final = VertexBase()
router: Final = APIRouter()
openai_passthrough_router: Final = APIRouter()
@ -2373,6 +2378,112 @@ async def cursor_proxy_route(
return received_value
VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON: Final = (
"Vertex AI auth failed: set a use_in_pass_through vertex model, default_vertex_config, or DEFAULT_VERTEXAI_* env"
)
VERTEX_PUBLISHER_MODEL_PREFIX: Final = "publishers/google/models/"
VERTEX_PUBLISHERS_SEGMENT: Final = "publishers/"
def _vertex_publisher_model_suffix(model: str) -> str:
"""
Turn whatever the client named into the ``publishers/<publisher>/models/<id>`` tail of a Vertex resource name.
Clients send bare ids, LiteLLM ids (``vertex_ai/gemini-live-2.5-flash``), and the Live SDK's ``models/<id>``,
and a publisher model id never contains a slash, so anything ahead of the last one is addressing, not identity
"""
publishers_at: Final = model.find(VERTEX_PUBLISHERS_SEGMENT)
if publishers_at != -1:
return model[publishers_at:]
return f"{VERTEX_PUBLISHER_MODEL_PREFIX}{model.rsplit('/', 1)[-1]}"
def _get_llm_router() -> "Router | None":
from litellm.proxy.proxy_server import llm_router
return llm_router
def _resolve_vertex_live_credentials(
vertex_project: str | None,
vertex_location: str | None,
model: str | None,
) -> VertexPassThroughCredentials | None:
"""
Resolution order: an explicit project/location registration, then ``default_vertex_config`` (which the proxy
fills from the ``DEFAULT_VERTEXAI_*`` env vars whenever the yaml leaves it out), then any DB model entry
flagged ``use_in_pass_through``.
DB entries come last on purpose: an operator who set a global default already said which project
pass-through traffic should bill to, and this route silently ignoring that would be the worse surprise
"""
keyed: Final = passthrough_endpoint_router.get_vertex_credentials(
project_id=vertex_project,
location=vertex_location,
)
if keyed is not None and keyed.vertex_project is not None:
return keyed
from_deployments: Final = passthrough_endpoint_router.get_vertex_credentials_from_router_deployments(model=model)
if from_deployments is not None:
return from_deployments
if keyed is not None:
return keyed
passthrough_endpoint_router.set_default_vertex_config()
return passthrough_endpoint_router.get_vertex_credentials(
project_id=vertex_project,
location=vertex_location,
)
def _build_vertex_live_setup_model_rewriter(
vertex_project: str | None,
vertex_location: str | None,
llm_router: "Router | None",
) -> Callable[[str], str] | None:
"""
Rewrite the ``setup`` frame's model into the full Vertex resource path the Live API requires.
Clients address the gateway the way they address LiteLLM (bare id or model alias); Vertex reads anything
that is not a ``projects/...`` path as a project name and closes the socket
"""
if vertex_project is None or vertex_location is None:
return None
def rewrite(setup_model: str) -> str:
if setup_model.startswith("projects/"):
return setup_model
aliased: Final = _resolve_alias_to_upstream_model(setup_model, llm_router)
return f"projects/{vertex_project}/locations/{vertex_location}/{_vertex_publisher_model_suffix(aliased)}"
return rewrite
def _resolve_alias_to_upstream_model(setup_model: str, llm_router: "Router | None") -> str:
"""
The Live SDK wraps whatever the caller typed as ``models/<name>``, so a gateway alias arrives prefixed
"""
if llm_router is None:
return setup_model
candidates: Final = (setup_model, setup_model.rsplit("/", 1)[-1])
upstream: Final = next(
(
deployment["litellm_params"].get("model")
for deployment in (llm_router.get_model_list() or ())
if deployment.get("model_name") in candidates
),
None,
)
if upstream is None:
return setup_model
try:
_, provider, _, _ = litellm.get_llm_provider(model=upstream)
except litellm.exceptions.BadRequestError:
return upstream
return upstream.removeprefix(f"{provider}/")
async def vertex_ai_live_websocket_passthrough(
websocket: WebSocket,
model: str | None = None,
@ -2396,51 +2507,38 @@ async def vertex_ai_live_websocket_passthrough(
await websocket.accept()
incoming_headers: Final = dict(websocket.headers)
vertex_credentials_config = passthrough_endpoint_router.get_vertex_credentials(
project_id=vertex_project,
location=vertex_location,
vertex_credentials_config: Final = _resolve_vertex_live_credentials(
vertex_project=vertex_project,
vertex_location=vertex_location,
model=model,
)
if vertex_credentials_config is None:
# Attempt to load defaults from environment/config if not already initialised
passthrough_endpoint_router.set_default_vertex_config()
vertex_credentials_config = passthrough_endpoint_router.get_vertex_credentials(
project_id=vertex_project,
location=vertex_location,
)
resolved_project = vertex_project
resolved_location: str | None = vertex_location
credentials_value: str | None = None
if vertex_credentials_config is not None:
resolved_project = resolved_project or vertex_credentials_config.vertex_project
temp_location: Final = resolved_location or vertex_credentials_config.vertex_location
# Ensure resolved_location is a string
if isinstance(temp_location, dict) or temp_location is not None:
resolved_location = str(temp_location)
else:
resolved_location = None
credentials_value = (
str(vertex_credentials_config.vertex_credentials)
if vertex_credentials_config.vertex_credentials is not None
else None
)
configured_project: Final = vertex_project or (
vertex_credentials_config.vertex_project if vertex_credentials_config is not None else None
)
configured_location: Final = vertex_location or (
vertex_credentials_config.vertex_location if vertex_credentials_config is not None else None
)
credentials_value: Final = (
vertex_credentials_config.vertex_credentials if vertex_credentials_config is not None else None
)
try:
resolved_location = resolved_location or (vertex_llm_base.get_default_vertex_location())
if model:
resolved_location = vertex_llm_base.get_vertex_region(
vertex_region=resolved_location,
resolved_location: Final = (
vertex_llm_base.get_vertex_region(
vertex_region=configured_location or vertex_llm_base.get_default_vertex_location(),
model=model,
)
if model
else configured_location or vertex_llm_base.get_default_vertex_location()
)
(
access_token,
resolved_project,
) = await vertex_llm_base._ensure_access_token_async(
credentials=credentials_value,
project_id=resolved_project,
project_id=configured_project,
custom_llm_provider="vertex_ai_beta",
)
except Exception as e:
@ -2453,7 +2551,7 @@ async def vertex_ai_live_websocket_passthrough(
request_data={},
)
if websocket.client_state != WebSocketState.DISCONNECTED:
await websocket.close(code=1011, reason="Vertex AI authentication failed")
await websocket.close(code=1011, reason=VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON)
return
host_location: Final = resolved_location or vertex_llm_base.get_default_vertex_location()
@ -2485,6 +2583,11 @@ async def vertex_ai_live_websocket_passthrough(
forward_headers=False,
endpoint="/vertex_ai/live",
accept_websocket=False,
setup_model_rewriter=_build_vertex_live_setup_model_rewriter(
vertex_project=resolved_project,
vertex_location=resolved_location,
llm_router=_get_llm_router(),
),
)

96
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -5,7 +5,7 @@ import json
import posixpath
import traceback
from base64 import b64encode
from collections.abc import AsyncGenerator, Callable, Mapping
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
from datetime import datetime
from itertools import groupby
from typing import Any, Final, TypedDict, cast
@ -32,11 +32,15 @@ from websockets.exceptions import (
ConnectionClosedOK,
InvalidStatus,
)
from websockets.frames import Close, CloseCode
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
from litellm.constants import (
MAXIMUM_TRACEBACK_LINES_TO_LOG,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
@ -1890,6 +1894,72 @@ def create_websocket_passthrough_route(
return websocket_endpoint_func
def _rewrite_vertex_live_setup_model(text_data: str, setup_model_rewriter: Callable[[str], str] | None) -> str:
"""
Rewrite the model of a Vertex AI Live ``setup`` frame, leaving every other frame byte-identical
"""
if setup_model_rewriter is None:
return text_data
try:
message: Final = json.loads(text_data)
except json.JSONDecodeError:
return text_data
if not isinstance(message, dict):
return text_data
setup: Final = message.get("setup")
if not isinstance(setup, dict):
return text_data
setup_model: Final = setup.get("model")
if not isinstance(setup_model, str):
return text_data
rewritten_model: Final = setup_model_rewriter(setup_model)
if rewritten_model == setup_model:
return text_data
return json.dumps({**message, "setup": {**setup, "model": rewritten_model}}) # mutable-ok: one-shot json payload
def _truncated_close_reason(reason: str) -> str:
"""
Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character
"""
encoded: Final = reason.encode("utf-8")
if len(encoded) <= WEBSOCKET_CLOSE_REASON_MAX_BYTES:
return reason
return encoded[:WEBSOCKET_CLOSE_REASON_MAX_BYTES].decode("utf-8", errors="ignore")
SENDABLE_CLOSE_CODES: Final = frozenset(CloseCode) - frozenset(
{CloseCode.NO_STATUS_RCVD, CloseCode.ABNORMAL_CLOSURE, CloseCode.TLS_HANDSHAKE}
)
def _client_socket_is_open(websocket: WebSocket) -> bool:
"""
Starlette tracks the two halves separately and raises on a second close, so both have to still be live
"""
return (
websocket.client_state != WebSocketState.DISCONNECTED
and websocket.application_state != WebSocketState.DISCONNECTED
)
def _upstream_close_to_relay(task_results: Iterable[object]) -> Close | None:
"""
The upstream close worth telling the client about: anything other than a plain, reasonless normal close.
Codes outside ``SENDABLE_CLOSE_CODES`` and the private range never travel on the wire (1006 for a socket that
died without a close frame, 1005 for one that sent no code), so relaying them would build an invalid frame
"""
upstream_close: Final = next((result for result in task_results if isinstance(result, Close)), None)
if upstream_close is None:
return None
if upstream_close.code == 1000 and upstream_close.reason == "":
return None
if upstream_close.code not in SENDABLE_CLOSE_CODES and not 3000 <= upstream_close.code < 5000:
return None
return upstream_close
async def websocket_passthrough_request(
websocket: WebSocket,
target: str,
@ -1899,6 +1969,7 @@ async def websocket_passthrough_request(
endpoint: str | None = None,
cost_per_request: float | None = None,
accept_websocket: bool = True,
setup_model_rewriter: Callable[[str], str] | None = None,
):
"""
WebSocket passthrough request handler.
@ -1911,6 +1982,7 @@ async def websocket_passthrough_request(
forward_headers: Whether to forward incoming headers
endpoint: The endpoint path (for logging purposes)
cost_per_request: Optional field - cost per request to the target endpoint
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import proxy_logging_obj
@ -2100,7 +2172,7 @@ async def websocket_passthrough_request(
)
# Not a JSON message or doesn't contain setup data
await upstream_ws.send(text_data)
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
elif bytes_data is not None:
await upstream_ws.send(bytes_data)
except asyncio.CancelledError:
@ -2111,8 +2183,8 @@ async def websocket_passthrough_request(
)
await upstream_ws.close()
async def forward_upstream_to_client() -> None:
"""Forward messages from upstream to client WebSocket"""
async def forward_upstream_to_client() -> Close | None:
"""Forward messages from upstream to client WebSocket, returning the upstream's close frame"""
try:
# Wait for the first response from upstream
raw_response = await upstream_ws.recv(decode=False)
@ -2177,6 +2249,7 @@ async def websocket_passthrough_request(
except (ConnectionClosedOK, ConnectionClosedError) as e:
verbose_proxy_logger.debug("Upstream WebSocket connection closed: %s", e)
return e.rcvd
except asyncio.CancelledError:
verbose_proxy_logger.debug("asyncio.CancelledError in forward_upstream_to_client")
raise
@ -2209,6 +2282,13 @@ async def websocket_passthrough_request(
if exception is not None:
raise exception
upstream_close: Final = _upstream_close_to_relay(task.result() for task in done)
if upstream_close is not None and _client_socket_is_open(websocket):
await websocket.close(
code=upstream_close.code,
reason=_truncated_close_reason(upstream_close.reason),
)
end_time: Final = datetime.now()
# Update passthrough logging payload with response data
@ -2294,7 +2374,7 @@ async def websocket_passthrough_request(
),
)
if websocket.client_state != WebSocketState.DISCONNECTED:
if _client_socket_is_open(websocket):
await websocket.close(
code=getattr(exc, "status_code", 1011),
reason="Upstream connection rejected",
@ -2322,10 +2402,10 @@ async def websocket_passthrough_request(
),
)
if websocket.client_state != WebSocketState.DISCONNECTED:
if _client_socket_is_open(websocket):
await websocket.close(code=1011, reason="WebSocket passthrough error")
finally:
if websocket.client_state != WebSocketState.DISCONNECTED:
if _client_socket_is_open(websocket):
await websocket.close()

92
litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py
View file

@ -1,3 +1,4 @@
import json
from collections.abc import Callable
from typing import TYPE_CHECKING, Final
@ -10,7 +11,7 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
from litellm.types.router import LiteLLMParamsTypedDict
from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict
if TYPE_CHECKING:
from litellm.router import Router
@ -27,6 +28,15 @@ def _get_str_value(values: dict[str, object] | None, key: str) -> str | None:
return value if isinstance(value, str) else None
def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str | None:
"""
A hashable stand-in for a credential, so two deployments can be compared for holding the same one
"""
if isinstance(credentials, dict):
return json.dumps(credentials, sort_keys=True)
return credentials
class PassthroughEndpointRouter:
"""
Use this class to Get credentials for pass-through endpoints
@ -120,6 +130,86 @@ class PassthroughEndpointRouter:
return None
return provider
def get_vertex_credentials_from_router_deployments(self, model: str | None) -> VertexPassThroughCredentials | None:
"""
Resolve vertex pass-through credentials from the live router deployments flagged ``use_in_pass_through``.
``deployment_key_to_vertex_credentials`` is only reachable when the caller names a project and location,
which WebSocket clients never do, so DB-stored deployments need this lookup to be usable at all.
With no model to go on, only deployments that agree on a project, a location, and a credential answer:
guessing between two Vertex projects would mint a token for one and later send the other one's model name
"""
llm_router: Final = self.llm_router_getter()
if llm_router is None:
return None
resolved: Final = tuple(
(deployment, credentials)
for deployment in (llm_router.get_model_list() or ())
if (credentials := self._resolve_vertex_deployment_credentials(deployment["litellm_params"])) is not None
)
matched: Final = next(
(
credentials
for deployment, credentials in resolved
if model is not None and self._deployment_matches_model(deployment, model)
),
None,
)
if matched is not None:
return matched
targets: Final = frozenset(
(
credentials.vertex_project,
credentials.vertex_location,
_credential_identity(credentials.vertex_credentials),
)
for _, credentials in resolved
)
if len(targets) != 1:
return None
return resolved[0][1]
def _resolve_vertex_deployment_credentials(
self, litellm_params: LiteLLMParamsTypedDict
) -> VertexPassThroughCredentials | None:
if litellm_params.get("use_in_pass_through") is not True:
return None
if self._get_deployment_provider(litellm_params) != "vertex_ai":
return None
credential_name: Final = litellm_params.get("litellm_credential_name")
credential_values: Final = (
CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else None
)
vertex_project: Final = _get_str_value(credential_values, "vertex_project") or litellm_params.get(
"vertex_project"
)
vertex_location: Final = _get_str_value(credential_values, "vertex_location") or litellm_params.get(
"vertex_location"
)
stored_credentials: Final = (
credential_values.get("vertex_credentials") if credential_values is not None else None
)
vertex_credentials: Final = (
stored_credentials if isinstance(stored_credentials, (str, dict)) else None
) or litellm_params.get("vertex_credentials")
if vertex_project is None or vertex_location is None:
return None
return VertexPassThroughCredentials(
vertex_project=vertex_project,
vertex_location=vertex_location,
vertex_credentials=vertex_credentials,
)
@staticmethod
def _deployment_matches_model(deployment: DeploymentTypedDict, model: str) -> bool:
upstream_model: Final = deployment["litellm_params"].get("model")
return model in (
deployment.get("model_name"),
upstream_model,
upstream_model.split("/", 1)[-1] if upstream_model is not None else None,
)
def _get_vertex_env_vars(self) -> VertexPassThroughCredentials:
"""
Helper to get vertex pass through config from environment variables

21
litellm/proxy/proxy_server.py
View file

@ -222,7 +222,7 @@ from functools import lru_cache
import litellm
import litellm._redis
from litellm import Router
from litellm._logging import verbose_proxy_logger, verbose_router_logger
from litellm._logging import _redact_string, verbose_proxy_logger, verbose_router_logger
from litellm.caching.caching import DualCache, RedisCache
from litellm.caching.redis_cluster_cache import RedisClusterCache
from litellm.constants import (
@ -259,6 +259,10 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.realtime_errors import (
realtime_error_event,
websocket_close_reason,
)
from litellm.litellm_core_utils.sensitive_data_masker import (
SensitiveDataMasker,
mask_sensitive_keys,
@ -10993,9 +10997,20 @@ async def realtime_websocket_endpoint(
except websockets.exceptions.InvalidStatusCode as e:
verbose_proxy_logger.exception("Invalid status code")
await websocket.close(code=e.status_code, reason="Invalid status code")
except Exception:
except Exception as e:
verbose_proxy_logger.exception("Internal server error")
await websocket.close(code=1011, reason="Internal server error")
redacted_error: Final = _redact_string(str(e))
try:
await websocket.send_text(realtime_error_event(redacted_error, error_type="server_error"))
except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below
verbose_proxy_logger.debug("Could not send realtime error event to client; closing anyway")
try:
await websocket.close(
code=1011,
reason=websocket_close_reason(redacted_error, fallback="Internal server error"),
)
except Exception: # noqa: BLE001 # the lower layer may have closed the socket already; closing twice is not an error
verbose_proxy_logger.debug("Could not close realtime client websocket; it is already gone")
######################################################################

55
litellm/realtime_api/main.py
View file

@ -1,15 +1,21 @@
"""Abstraction function for OpenAI's realtime API"""
import asyncio
import os
from typing import Any, Final, cast
from typing import Any, Final, Literal, cast
import litellm
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout
from litellm.constants import (
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
request_timeout,
)
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.xai.common_utils import XAIModelInfo
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES, VertexAccessTokenResolver
from litellm.types.realtime import (
RealtimeClientSecretRequest,
RealtimeExpiresAfter,
@ -281,6 +287,41 @@ async def arealtime_calls(
)
async def vertex_access_token_resolver(
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
) -> tuple[str, str]:
return await vertex_llm_base._ensure_access_token_async(
credentials=credentials,
project_id=project_id,
custom_llm_provider=custom_llm_provider,
)
async def _resolve_vertex_access_token_bounded(
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
resolver: VertexAccessTokenResolver,
timeout_seconds: float,
) -> tuple[str, str]:
try:
return await asyncio.wait_for(
resolver(
credentials=credentials,
project_id=project_id,
custom_llm_provider="vertex_ai",
),
timeout=timeout_seconds,
)
except asyncio.TimeoutError as e:
raise ValueError(
"Vertex AI realtime: timed out fetching Google OAuth access token after "
f"{timeout_seconds}s; check network egress from the proxy "
"to the OAuth token endpoint (oauth2.googleapis.com)"
) from e
@wrapper_client
async def _arealtime(
model: str,
@ -478,10 +519,11 @@ async def _arealtime(
(
access_token,
resolved_project,
) = await vertex_llm_base._ensure_access_token_async(
) = await _resolve_vertex_access_token_bounded(
credentials=vertex_credentials,
project_id=vertex_project,
custom_llm_provider="vertex_ai",
resolver=vertex_access_token_resolver,
timeout_seconds=REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
)
vertex_realtime_config: Final = VertexAIRealtimeConfig(
@ -559,10 +601,11 @@ async def _realtime_health_check(
(
access_token,
resolved_project,
) = await vertex_llm_base._ensure_access_token_async(
) = await _resolve_vertex_access_token_bounded(
credentials=VertexBase.safe_get_vertex_ai_credentials(vertex_model_params),
project_id=VertexBase.safe_get_vertex_ai_project(vertex_model_params),
custom_llm_provider="vertex_ai",
resolver=vertex_access_token_resolver,
timeout_seconds=REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
)
vertex_realtime_config: Final = VertexAIRealtimeConfig(
access_token=access_token,

31
litellm/responses/utils.py
View file

@ -37,24 +37,45 @@ def normalize_responses_api_stream_options(
return ResponsesAPIStreamOptions(include_obfuscation=include_obfuscation)
def _is_chat_text_part(part: object) -> bool:
return isinstance(part, dict) and part.get("type") == "text"
def _as_input_text_part(part: object) -> object:
if isinstance(part, dict) and part.get("type") == "text":
return {**part, "type": "input_text"} # mutable-ok: fresh part so the caller's block keeps its chat type
return part
class ResponsesAPIRequestUtils:
"""Helper utils for constructing ResponseAPI requests"""
@staticmethod
def shape_prompt_managed_message_for_responses(message: object) -> object:
if not isinstance(message, dict) or message.get("role") == "assistant":
return message
content: object = message.get("content")
if not isinstance(content, list) or not any(_is_chat_text_part(part) for part in content):
return message
shaped_content: Final = [_as_input_text_part(part) for part in content] # mutable-ok: Responses-shaped copy
return {**message, "content": shaped_content} # mutable-ok: copy, the hook's message stays untouched
@staticmethod
def merge_prompt_management_input(
original_input: str | ResponseInputParam,
client_input: list[AllMessageValues],
merged_input: list[AllMessageValues],
) -> list[object]:
shape: Final = ResponsesAPIRequestUtils.shape_prompt_managed_message_for_responses
if isinstance(original_input, str):
return [*merged_input]
return [shape(message) for message in merged_input]
original_items: Final = tuple(original_input)
client_item_ids: Final = frozenset(id(item) for item in client_input)
message_positions = tuple(index for index, item in enumerate(original_items) if id(item) in client_item_ids)
if len(message_positions) == len(original_items):
return [*merged_input]
return [shape(message) for message in merged_input]
if not message_positions:
verbose_logger.warning(
"Prompt management hook returned messages without Responses API input messages; merged messages were ignored"
@ -69,7 +90,7 @@ class ResponsesAPIRequestUtils:
if corresponding_messages:
merged_by_position: Final = dict(zip(message_positions, merged_input))
return [
merged_by_position[index] if index in merged_by_position else item
shape(merged_by_position[index]) if index in merged_by_position else item
for index, item in enumerate(original_items)
]
@ -82,14 +103,14 @@ class ResponsesAPIRequestUtils:
for index, position in enumerate(message_positions)
}
trailing_items: Final = original_items[message_positions[-1] + 1 :]
return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), merged)] + list(
return [item for merged in merged_input for item in (*prefixes.get(id(merged), ()), shape(merged))] + list(
trailing_items
)
verbose_logger.warning(
"Prompt management hook replaced Responses API messages; non-message input items were dropped"
)
return [*merged_input]
return [shape(message) for message in merged_input]
@staticmethod
def merge_client_forwarded_headers(

4
litellm/types/integrations/anthropic_cache_control_hook.py
View file

@ -1,6 +1,6 @@
from typing import Literal
from typing_extensions import NotRequired, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.types.llms.openai import ChatCompletionCachedContent
@ -13,6 +13,7 @@ class CacheControlMessageInjectionPoint(TypedDict):
index: int | str | None # Optional: target by specific index
control: ChatCompletionCachedContent | None
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
class CacheControlToolConfigInjectionPoint(TypedDict):
@ -21,6 +22,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
location: Literal["tool_config"]
control: ChatCompletionCachedContent | None
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint

4
litellm/types/llms/anthropic.py
View file

@ -9,6 +9,7 @@ from .openai import (
ChatCompletionCachedContent,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
PromptCacheBreakpoint,
)
@ -201,6 +202,7 @@ class AnthropicMessagesTextParam(TypedDict, total=False):
type: Required[Literal["text"]]
text: Required[str]
cache_control: dict | ChatCompletionCachedContent | None
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
class AnthropicMessagesToolUseParam(TypedDict, total=False):
@ -261,6 +263,7 @@ class AnthropicMessagesImageParam(TypedDict, total=False):
type: Required[Literal["image"]]
source: Required[AnthropicContentParamSource | AnthropicContentParamSourceFileId | AnthropicContentParamSourceUrl]
cache_control: dict | ChatCompletionCachedContent | None
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
class CitationsObject(TypedDict):
@ -347,6 +350,7 @@ class AnthropicSystemMessageContent(TypedDict, total=False):
type: str
text: str
cache_control: dict | ChatCompletionCachedContent | None
prompt_cache_breakpoint: ReadOnly[PromptCacheBreakpoint]
class AnthropicMessagesSystemMessageParam(TypedDict, total=False):

12
litellm/types/llms/openai.py
View file

@ -71,6 +71,7 @@ from pydantic import (
)
from typing_extensions import (
NotRequired,
ReadOnly,
Required,
TypedDict,
override,
@ -510,6 +511,15 @@ class ChatCompletionCachedContent(TypedDict):
ttl: NotRequired[Literal["5m", "1h"]]
class PromptCacheBreakpoint(TypedDict):
mode: ReadOnly[Literal["explicit"]]
class PromptCacheOptions(TypedDict, total=False):
mode: ReadOnly[Literal["implicit", "explicit"]]
ttl: ReadOnly[Literal["30m"]]
class ChatCompletionThinkingBlock(TypedDict, total=False):
type: Required[Literal["thinking"]]
thinking: str
@ -917,6 +927,7 @@ class ChatCompletionRequest(TypedDict, total=False):
seed: int
service_tier: str
safety_identifier: str
prompt_cache_key: str # writable-ok: the /v1/messages adapter assigns it after construction
stop: str | list[str]
stream_options: dict
temperature: float
@ -1148,6 +1159,7 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False):
max_tool_calls: int | None
prompt_cache_key: str | None
prompt_cache_retention: str | None
prompt_cache_options: ReadOnly[PromptCacheOptions | None]
stream_options: ResponsesAPIStreamOptions | None
top_logprobs: int | None
partial_images: int | None # Number of partial images to generate (1-3) for streaming image generation

13
litellm/types/llms/vertex_ai.py
View file

@ -1,5 +1,5 @@
from enum import Enum
from typing import Any, Final, Literal
from typing import Any, Final, Literal, Protocol
from typing_extensions import (
Required,
@ -747,6 +747,17 @@ class VertexVideoGenerationResponse(TypedDict, total=False):
VERTEX_CREDENTIALS_TYPES = str | dict[str, str]
class VertexAccessTokenResolver(Protocol):
"""Resolves a Google OAuth access token and the project id it belongs to."""
async def __call__(
self,
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
custom_llm_provider: Literal["vertex_ai", "vertex_ai_beta", "gemini"],
) -> tuple[str, str]: ...
class VertexPartnerProvider(str, Enum):
mistralai = "mistralai"
llama = "llama"

12
litellm/types/realtime.py
View file

@ -1,7 +1,7 @@
from typing import Any, Literal
from pydantic import BaseModel
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from .llms.openai import (
OpenAIRealtimeEvents,
@ -152,3 +152,13 @@ class RealtimeTranscriptionSessionResponse(BaseModel):
model_config = {"extra": "allow"}
client_secret: dict[str, Any] | None = None
class RealtimeErrorDetail(TypedDict):
type: ReadOnly[str]
message: ReadOnly[str]
class RealtimeErrorEvent(TypedDict):
type: ReadOnly[Literal["error"]]
error: ReadOnly[RealtimeErrorDetail]

1
litellm/types/utils.py
View file

@ -141,6 +141,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_tool_choice: bool | None
supports_assistant_prefill: bool | None
supports_prompt_caching: bool | None
supports_prompt_cache_breakpoint: ReadOnly[bool | None]
supports_computer_use: bool | None
supports_audio_input: bool | None
supports_embedding_image_input: bool | None

11
litellm/utils.py
View file

@ -2560,6 +2560,14 @@ def supports_prompt_caching(model: str, custom_llm_provider: str | None = None)
)
def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
return _supports_factory(
model=model,
custom_llm_provider=custom_llm_provider,
key="supports_prompt_cache_breakpoint",
)
def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool:
"""
Check if the given model supports computer use and return a boolean value.
@ -5473,6 +5481,7 @@ def _get_model_info_helper(
supports_tool_choice=None,
supports_assistant_prefill=None,
supports_prompt_caching=None,
supports_prompt_cache_breakpoint=None,
supports_computer_use=None,
supports_pdf_input=None,
)
@ -5712,6 +5721,7 @@ def _get_model_info_helper(
supports_tool_choice=_model_info.get("supports_tool_choice", None),
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
supports_audio_input=_model_info.get("supports_audio_input", None),
supports_audio_output=_model_info.get("supports_audio_output", None),
supports_pdf_input=_model_info.get("supports_pdf_input", None),
@ -5846,6 +5856,7 @@ def get_model_info(
supports_function_calling: Optional[bool]
supports_tool_choice: Optional[bool]
supports_prompt_caching: Optional[bool]
supports_prompt_cache_breakpoint: Optional[bool]
supports_audio_input: Optional[bool]
supports_audio_output: Optional[bool]
supports_pdf_input: Optional[bool]

4
model_prices_and_context_window.json
View file

@ -25368,6 +25368,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -25430,6 +25431,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -25492,6 +25494,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -25554,6 +25557,7 @@
"supports_none_reasoning_effort": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,

3
model_prices_and_context_window.schema.json
View file

@ -664,6 +664,9 @@
"supports_pdf_input": {
"type": "boolean"
},
"supports_prompt_cache_breakpoint": {
"type": "boolean"
},
"supports_prompt_caching": {
"type": "boolean"
},

1
scripts/budget_ratchet_check.py
View file

@ -44,6 +44,7 @@ DEFAULT_BUDGETS: tuple[str, ...] = (
"ruff-strict-budget.json",
"type-discipline-budget.json",
"basedpyright-code-budget.json",
"test-quality-budget.json",
)
GRADUATION_CONFIGS = MappingProxyType({"ruff-strict-budget.json": "ruff.toml"})

492
scripts/check_test_quality.py Normal file
View file

@ -0,0 +1,492 @@
#!/usr/bin/env python3
"""Test-quality checker: the test-suite smells no linter enforces.
Sibling of scripts/check_type_discipline.py, same output contract
(``path:line: CODE message``) and same stdlib-only constraint, aimed at the test
tree instead of the package. Each rule is a shape the testing-strategy audit
measured and named; scripts/test_quality_gate.py caps the codebase total of each
one against test-quality-budget.json so the counts can only ratchet down.
Rules
-----
TQ001 A collectible test function whose body contains no assertion of any kind:
no `assert` statement, no `pytest.raises`/`warns`/`deprecated_call`/`fail`,
and no `assert*` method call (mock's `assert_called_once`, unittest's
`assertEqual`, `numpy.testing.assert_allclose`). Such a test passes as long
as the code under it does not raise, so it pins nothing and cannot fail for
the reason anyone would want it to. Assert the observable output instead.
The whole function subtree counts, nested helper definitions included, so a
test that asserts inside a locally-defined async helper passes.
TQ002 Mock-echo: a test that patches something and whose every assertion only
inspects the mock that replaced it (`assert_called_once_with`, `.called`,
`.call_args`, `.call_count`, `.mock_calls`). The test restates the
implementation back at itself: it verifies that the code called what the
code calls, so it survives any refactor that keeps the call and breaks the
behavior. Assert what the caller observes -- the returned value, the
rebuilt response, the raised exception -- and fake at the HTTP boundary
(respx / MockTransport) rather than patching litellm internals.
A test with no assertions at all is TQ001, never TQ002.
TQ003 `sys.path.insert(...)` inside the test tree. pytest's rootdir handling and
the installed package already make `litellm` importable, so these are
no-ops carried by copy-paste; the ones that are not no-ops make the test's
imports depend on the working directory it happens to run from.
TQ004 Raw `os.environ[...] = ...` assignment. The write outlives the test and
leaks into whatever runs next in the same process, which is how a suite
acquires an ordering dependency. Use `monkeypatch.setenv`, which is undone
at teardown.
TQ005 `litellm.<attr> = ...` module-global mutation. The SDK's module globals are
process-wide, so this is the same leak as TQ004 one level up, and it is
what the 491-line save/restore conftest exists to paper over. Inject the
dependency or use a fixture that restores it.
Every rule is suppressible with `# test-quality-ok: <reason>` on the reported
line, following the repo's `*-ok: <reason>` convention. A suppression without a
reason does not suppress.
What counts as an assertion
---------------------------
An `assert` statement; `pytest.raises` / `warns` / `deprecated_call` / `fail`,
qualified or bare (`skip` and `xfail` are deliberately excluded, since they abort
the test rather than pin a behaviour); and any callable whose name starts with
`assert`, qualified (`m.assert_called_once`, `self.assertEqual`,
`np.testing.assert_allclose`) or bare (`assert_auth_denied(...)`, the shape the
e2e harness uses). A test also counts as asserting when it reaches an assertion
through a function defined in the same module, followed transitively, because
extracting the assertions into a shared helper is good factoring rather than a
test that pins nothing. A helper imported from another module is not followed, so
a test whose only assertions live across a module boundary still reports TQ001
and needs a suppression.
What counts as mock inspection (TQ002)
--------------------------------------
An `assert_`-prefixed call, which is mock's own family, or a reference to
`called` / `call_args` / `call_args_list` / `call_count` / `mock_calls` and their
await-counterparts. unittest's `assertEqual` has no underscore after "assert" and
so is never mistaken for one. A patch is installed by any call or decorator whose
name is `patch` or `patch.object` / `patch.dict` / `patch.multiple`, which covers
`unittest.mock` however it was imported as well as pytest-mock's `mocker.patch`.
Scope
-----
Only files under the test roots passed on the command line are examined, and
TQ001/TQ002 only look at functions pytest would collect: a `test_`-prefixed
function at module level, or a `test_`-prefixed method of a `Test`-prefixed
class that defines no `__init__`.
Usage
-----
python check_test_quality.py tests/
Exit code 1 if any violation is found. Stdlib only.
"""
from __future__ import annotations
import ast
import io
import re
import sys
import tokenize
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, NamedTuple
TEST_FUNCTION_PREFIX: Final = "test_"
TEST_CLASS_PREFIX: Final = "Test"
MIN_REASON_LEN: Final = 3
SUPPRESSION_TOKEN: Final = "test-quality-ok"
SUPPRESSION_RE: Final = re.compile(r"#\s*test-quality-ok(?::\s*(?P<reason>.*))?")
PYTEST_ASSERTION_HELPERS: Final = frozenset(("raises", "warns", "deprecated_call", "fail"))
MOCK_INSPECTION_ATTRIBUTES: Final = frozenset((
"called", "call_args", "call_args_list", "call_count", "mock_calls",
"await_args", "await_args_list", "await_count", "awaited",
))
MOCK_ASSERTION_PREFIX: Final = "assert_"
PATCH_MEMBERS: Final = frozenset(("object", "dict", "multiple"))
FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef
class Violation(NamedTuple):
path: Path
line: int
code: str
message: str
def render(self) -> str:
return f"{self.path}:{self.line}: {self.code} {self.message}"
def _dotted_name(node: ast.expr) -> str:
"""`a.b.c` for an attribute chain rooted in a plain name, else ""."""
if isinstance(node, ast.Name):
return node.id
if isinstance(node, ast.Attribute):
root: Final = _dotted_name(node.value)
return f"{root}.{node.attr}" if root else ""
return ""
def suppressed_lines(source: str) -> frozenset[int]:
"""Lines carrying `# test-quality-ok: <reason>` with a reason of usable length."""
try:
tokens: Final = tuple(tokenize.generate_tokens(io.StringIO(source).readline))
except (tokenize.TokenError, IndentationError, SyntaxError):
return frozenset()
return frozenset(
token.start[0]
for token in tokens
if token.type == tokenize.COMMENT
and (match := SUPPRESSION_RE.search(token.string)) is not None
and len((match.group("reason") or "").strip()) >= MIN_REASON_LEN
)
def _is_collectible_class(node: ast.ClassDef) -> bool:
"""pytest collects `Test`-prefixed classes that define no constructor."""
if not node.name.startswith(TEST_CLASS_PREFIX):
return False
return not any(
isinstance(child, ast.FunctionDef) and child.name == "__init__"
for child in node.body
)
def iter_test_functions(tree: ast.Module) -> Iterator[FunctionNode]:
"""Every function pytest would collect from this module, in source order."""
for node in tree.body:
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
if node.name.startswith(TEST_FUNCTION_PREFIX):
yield node
elif isinstance(node, ast.ClassDef) and _is_collectible_class(node):
yield from (
child
for child in node.body
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef))
and child.name.startswith(TEST_FUNCTION_PREFIX)
)
def _is_pytest_assertion_call(call: ast.Call) -> bool:
func: Final = call.func
if isinstance(func, ast.Attribute):
return func.attr in PYTEST_ASSERTION_HELPERS
if isinstance(func, ast.Name):
return func.id in PYTEST_ASSERTION_HELPERS
return False
def _is_assertion_helper_call(call: ast.Call) -> bool:
"""Any `assert*` callable: `x.assertEqual(...)`, `m.assert_called_once()`,
`np.testing.assert_allclose(...)`, and the bare shared helpers the e2e harness
uses (`assert_auth_denied(result, ...)`)."""
func: Final = call.func
if isinstance(func, ast.Attribute):
return func.attr.startswith("assert")
return isinstance(func, ast.Name) and func.id.startswith("assert")
def iter_assertions(function: FunctionNode) -> Iterator[ast.stmt | ast.Call]:
"""Every node in the function that pins a behaviour, nested definitions included."""
for node in ast.walk(function):
if isinstance(node, ast.Assert):
yield node
elif isinstance(node, ast.Call) and (
_is_pytest_assertion_call(node) or _is_assertion_helper_call(node)
):
yield node
class CallTarget(NamedTuple):
"""A call that might resolve to a function defined in this module: either a bare
name, looked up among the module-level functions, or a `self.` attribute, looked
up among the enclosing class's own methods."""
through_self: bool
name: str
@dataclass(frozen=True, slots=True)
class Scope:
"""What one function can reach by name. Keeping methods per-class is what stops
two same-named helpers in different classes from resolving to each other."""
module_level: Mapping[str, FunctionNode]
methods: Mapping[str, FunctionNode]
def resolve(self, target: CallTarget) -> FunctionNode | None:
source: Final = self.methods if target.through_self else self.module_level
return source.get(target.name)
def _call_target(func: ast.expr) -> CallTarget | None:
if isinstance(func, ast.Name):
return CallTarget(through_self=False, name=func.id)
if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) and func.value.id == "self":
return CallTarget(through_self=True, name=func.attr)
return None
def _call_targets(function: FunctionNode) -> frozenset[CallTarget]:
return frozenset(
target
for node in ast.walk(function)
if isinstance(node, ast.Call)
for target in (_call_target(node.func),)
if target is not None
)
def _functions_in(body: Iterable[ast.stmt]) -> Mapping[str, FunctionNode]:
return MappingProxyType({
node.name: node
for node in body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
})
def build_scopes(tree: ast.Module) -> Mapping[FunctionNode, Scope]:
"""Every function in the module paired with what it can reach by name. A
module-level function sees only module-level functions; a method also sees its
own class's methods, and no other class's."""
module_level: Final = _functions_in(tree.body)
module_scope: Final = Scope(module_level=module_level, methods=MappingProxyType({}))
class_scopes: Final = tuple(
(node, Scope(module_level=module_level, methods=_functions_in(node.body)))
for node in tree.body
if isinstance(node, ast.ClassDef)
)
return MappingProxyType({
**{function: module_scope for function in module_level.values()},
**{
function: scope
for node, scope in class_scopes
for function in scope.methods.values()
},
})
def _reaches_assertion(
function: FunctionNode,
scopes: Mapping[FunctionNode, Scope],
seen: frozenset[FunctionNode],
) -> bool:
if function in seen:
return False
if any(iter_assertions(function)):
return True
scope: Final = scopes.get(function)
if scope is None:
return False
return any(
_reaches_assertion(callee, scopes, seen | frozenset((function,)))
for target in _call_targets(function)
for callee in (scope.resolve(target),)
if callee is not None
)
def asserts_through_helpers(
function: FunctionNode, scopes: Mapping[FunctionNode, Scope]
) -> bool:
"""Whether the test reaches an assertion through a function defined in this
module, followed transitively. Extracting the assertions into a shared helper is
good factoring rather than a test that pins nothing, so following one is what
keeps TQ001 honest."""
scope: Final = scopes.get(function)
if scope is None:
return False
return any(
_reaches_assertion(callee, scopes, frozenset((function,)))
for target in _call_targets(function)
for callee in (scope.resolve(target),)
if callee is not None
)
def _is_patch_installer(dotted: str) -> bool:
"""`patch`, `mock.patch`, `mocker.patch`, `patch.object`, `mock.patch.dict`, ..."""
parts: Final = dotted.split(".")
if parts[-1] == "patch":
return True
return len(parts) >= 2 and parts[-2] == "patch" and parts[-1] in PATCH_MEMBERS
def _installs_patch(function: FunctionNode) -> bool:
decorators: Final = tuple(
_dotted_name(d.func) if isinstance(d, ast.Call) else _dotted_name(d)
for d in function.decorator_list
)
if any(name and _is_patch_installer(name) for name in decorators):
return True
return any(
_is_patch_installer(_dotted_name(node.func))
for node in ast.walk(function)
if isinstance(node, ast.Call) and _dotted_name(node.func)
)
def _only_inspects_a_mock(node: ast.stmt | ast.Call) -> bool:
"""True when this assertion reads a mock's call record and nothing else."""
if isinstance(node, ast.Call):
func = node.func
return isinstance(func, ast.Attribute) and func.attr.startswith(MOCK_ASSERTION_PREFIX)
return any(
isinstance(child, ast.Attribute)
and (
child.attr in MOCK_INSPECTION_ATTRIBUTES
or child.attr.startswith(MOCK_ASSERTION_PREFIX)
)
for child in ast.walk(node)
)
def iter_assertion_violations(path: Path, tree: ast.Module) -> Iterator[Violation]:
scopes: Final = build_scopes(tree)
for function in iter_test_functions(tree):
assertions: Final = tuple(iter_assertions(function))
if not assertions and asserts_through_helpers(function, scopes):
continue
if not assertions:
yield Violation(
path,
function.lineno,
"TQ001",
f"test `{function.name}` asserts nothing, so it can only fail by raising; "
f"assert the observable output (suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
)
elif _installs_patch(function) and all(map(_only_inspects_a_mock, assertions)):
yield Violation(
path,
function.lineno,
"TQ002",
f"test `{function.name}` patches something and only asserts that the mock was "
f"called, which restates the implementation; assert what the caller observes "
f"(suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
)
def iter_sys_path_violations(path: Path, tree: ast.Module) -> Iterator[Violation]:
for node in ast.walk(tree):
if isinstance(node, ast.Call) and _dotted_name(node.func) == "sys.path.insert":
yield Violation(
path,
node.lineno,
"TQ003",
"sys.path.insert in a test; pytest's rootdir and the installed package already "
f"make litellm importable (suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
)
def _environ_subscript_targets(target: ast.expr) -> Iterator[ast.Subscript]:
if isinstance(target, ast.Tuple):
for element in target.elts:
yield from _environ_subscript_targets(element)
return
if isinstance(target, ast.Subscript) and _dotted_name(target.value) in ("os.environ", "environ"):
yield target
def iter_environ_violations(path: Path, tree: ast.Module) -> Iterator[Violation]:
for node in ast.walk(tree):
targets: Final = (
node.targets if isinstance(node, ast.Assign)
else (node.target,) if isinstance(node, (ast.AugAssign, ast.AnnAssign))
else ()
)
for target in targets:
for subscript in _environ_subscript_targets(target):
yield Violation(
path,
subscript.lineno,
"TQ004",
"raw os.environ write leaks into every test that runs after this one; "
f"use monkeypatch.setenv (suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
)
def _litellm_attribute_targets(target: ast.expr) -> Iterator[ast.Attribute]:
if isinstance(target, ast.Tuple):
for element in target.elts:
yield from _litellm_attribute_targets(element)
return
if isinstance(target, ast.Attribute) and _dotted_name(target.value) == "litellm":
yield target
def iter_global_mutation_violations(path: Path, tree: ast.Module) -> Iterator[Violation]:
for node in ast.walk(tree):
targets: Final = (
node.targets if isinstance(node, ast.Assign)
else (node.target,) if isinstance(node, (ast.AugAssign, ast.AnnAssign))
else ()
)
for target in targets:
for attribute in _litellm_attribute_targets(target):
yield Violation(
path,
attribute.lineno,
"TQ005",
f"litellm.{attribute.attr} is a process-wide global; writing it here is what the "
"save/restore conftest exists to undo, so inject the dependency or use a fixture "
f"(suppress: `# {SUPPRESSION_TOKEN}: <reason>`)",
)
def check_file(path: Path) -> tuple[Violation, ...]:
try:
source: Final = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as exc:
return (Violation(path, 0, "TQ000", f"unreadable: {exc}"),)
try:
tree: Final = ast.parse(source, filename=str(path))
except SyntaxError as exc:
return (Violation(path, exc.lineno or 0, "TQ000", f"syntax error: {exc.msg}"),)
skip: Final = suppressed_lines(source)
return tuple(
violation
for violation in (
*iter_assertion_violations(path, tree),
*iter_sys_path_violations(path, tree),
*iter_environ_violations(path, tree),
*iter_global_mutation_violations(path, tree),
)
if violation.line not in skip
)
def collect_paths(raw: Iterable[str]) -> Iterator[Path]:
for item in raw:
candidate: Final = Path(item)
if candidate.is_dir():
yield from sorted(candidate.rglob("*.py"))
elif candidate.suffix == ".py":
yield candidate
def main(argv: Sequence[str]) -> int:
paths: Final = tuple(a for a in argv if not a.startswith("-"))
if not paths:
print("usage: check_test_quality.py <files-or-dirs>...", file=sys.stderr)
return 2
violations: Final = sorted(v for path in collect_paths(paths) for v in check_file(path))
for violation in violations:
print(violation.render())
if violations:
print(f"\n{len(violations)} violation(s).", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))

289
scripts/test_quality_gate.py Normal file
View file

@ -0,0 +1,289 @@
#!/usr/bin/env python3
"""Total-count gate for the TQ* rules in scripts/check_test_quality.py.
Sibling of scripts/type_discipline_gate.py, pointed at the test tree instead of
the package. Each rule listed in test-quality-budget.json has a hard ``limit``.
The gate counts each rule across the whole `tests` tree and fails when a rule is
both over its limit and higher than the base it merges into, so a change is
blamed for the violations it adds, never for drift that already exists in the
base.
Every rule is seeded at exactly its count on the day the gate landed, so the
suite's existing debt is grandfathered and any net-new violation trips the gate
immediately. ``--update`` ratchets a limit down by the violations this branch
fixed relative to its branch point (the merge-base), so the ceilings only ever
fall. A rule absent from the budget at the merge-base was seeded on this branch;
``--update`` leaves its limit untouched, because the base tree predates the rule
and its whole grandfathered count would otherwise be misread as "fixed".
The deliberate difference from its sibling: this gate has no headroom anywhere.
Type discipline seeded LIT010/LIT011 at 1.5x to leave room for an in-flight
sweep; a test-quality violation has no such transition to absorb, so the line is
today's count and the only legal direction is down.
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import subprocess
import sys
import tempfile
from collections import Counter
from collections.abc import Mapping, Sequence
from pathlib import Path
from types import MappingProxyType
from typing import Final, NamedTuple
REPO_ROOT: Final = Path(__file__).resolve().parent.parent
CHECKER: Final = REPO_ROOT / "scripts" / "check_test_quality.py"
BUDGET_PATH: Final = REPO_ROOT / "test-quality-budget.json"
TARGET: Final = "tests"
DEFAULT_BASE: Final = "origin/litellm_internal_staging"
_HUNK: Final = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@", re.MULTILINE)
_FILE_HEADER: Final = re.compile(r"^\+\+\+ b/(.+)$", re.MULTILINE)
_LINE: Final = re.compile(r"^(?P<file>.+?):(?P<line>\d+): (?P<code>TQ\d+) ")
class Violation(NamedTuple):
file: str
line: int
code: str
class Breach(NamedTuple):
rule: str
total: int
cap: int
added: int
def _run(cmd: Sequence[str], cwd: Path = REPO_ROOT) -> str:
proc: Final = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
if proc.returncode not in (0, 1):
sys.stderr.write(proc.stderr)
raise SystemExit(f"{cmd[0]} exited {proc.returncode}")
return proc.stdout
def resolve_base_point(base_ref: str, cwd: Path = REPO_ROOT) -> str:
"""The snapshot commit base counts are measured at: merge-base(base_ref, HEAD),
made aware of an in-progress merge. Mid-merge, HEAD is still the pre-merge tip,
so its merge-base is the old branch point and every violation the base gained
since then would be blamed on this change."""
head_point: Final = _run(["git", "merge-base", base_ref, "HEAD"], cwd=cwd).strip()
if not head_point:
return base_ref
merge_head: Final = _run(["git", "rev-parse", "--verify", "--quiet", "MERGE_HEAD"], cwd=cwd).strip()
if not merge_head:
return head_point
merge_point: Final = _run(["git", "merge-base", base_ref, merge_head], cwd=cwd).strip()
if not merge_point:
return head_point
older: Final = _run(["git", "merge-base", head_point, merge_point], cwd=cwd).strip()
return merge_point if older == head_point else head_point
def _check(root: Path, checker: Path) -> tuple[Violation, ...]:
# macOS tempfile dirs (/var/...) resolve to /private/var/..., so relative_to needs both sides resolved.
resolved: Final = root.resolve()
out: Final = _run([sys.executable, str(checker), str(resolved / TARGET)], cwd=resolved)
return tuple(
Violation(
(resolved / match.group("file")).resolve().relative_to(resolved).as_posix(),
int(match.group("line")),
match.group("code"),
)
for line in out.splitlines()
if (match := _LINE.match(line)) is not None
)
def head_violations() -> tuple[Violation, ...]:
return _check(REPO_ROOT, CHECKER)
def count_by_rule(violations: Sequence[Violation]) -> Mapping[str, int]:
return MappingProxyType(dict(Counter(v.code for v in violations)))
def base_counts(ref: str) -> Mapping[str, int]:
"""Rule counts at `ref`, measured with the *current* rule logic rather than
whatever the checker looked like at that commit."""
parent: Final = Path(tempfile.mkdtemp(prefix="tq_base_"))
worktree: Final = parent / "wt"
try:
_run(["git", "worktree", "add", "--detach", str(worktree), ref])
(worktree / "scripts").mkdir(parents=True, exist_ok=True)
checker: Final = worktree / "scripts" / "check_test_quality.py"
shutil.copy(CHECKER, checker)
return count_by_rule(_check(worktree, checker))
finally:
# Teardown must never raise, or it masks the real error when the body failed.
subprocess.run(
["git", "worktree", "remove", "--force", str(worktree)],
cwd=REPO_ROOT, capture_output=True, text=True,
)
shutil.rmtree(parent, ignore_errors=True)
def over_ceiling(head: Mapping[str, int], budget: Mapping[str, Mapping[str, int]]) -> frozenset[str]:
"""Rules whose head count already exceeds their limit. When none are, the base
comparison cannot change the verdict and the base worktree scan is skipped."""
return frozenset(
rule for rule, spec in budget.items() if head.get(rule, 0) > spec["limit"]
)
def evaluate(
head: Mapping[str, int],
base: Mapping[str, int],
budget: Mapping[str, Mapping[str, int]],
) -> tuple[Breach, ...]:
return tuple(sorted(
Breach(rule, head.get(rule, 0), spec["limit"], head.get(rule, 0) - base.get(rule, 0))
for rule, spec in budget.items()
if head.get(rule, 0) > spec["limit"] and head.get(rule, 0) > base.get(rule, 0)
))
def _hunk_lines(body: str) -> frozenset[int]:
return frozenset(
line
for match in _HUNK.finditer(body)
for start in (int(match.group(1)),)
for line in range(start, start + (int(match.group(2)) if match.group(2) is not None else 1))
)
def parse_changed_lines(diff_text: str) -> Mapping[str, frozenset[int]]:
"""Each file in the diff mapped to the line numbers it adds. Splitting on the
`+++ b/` headers keeps this a pure expression: `split` hands back
[preamble, path, body, path, body, ...], so each file's hunks are already
grouped with it."""
parts: Final = _FILE_HEADER.split(diff_text)
return MappingProxyType({
path: _hunk_lines(body)
for path, body in zip(parts[1::2], parts[2::2])
})
def introduced(
violations: Sequence[Violation], changed: Mapping[str, frozenset[int]]
) -> tuple[Violation, ...]:
return tuple(v for v in violations if v.line in changed.get(v.file, frozenset()))
def cmd_check(base: str) -> None:
budget: Final = json.loads(BUDGET_PATH.read_text())
head: Final = head_violations()
head_counts: Final = count_by_rule(head)
if not over_ceiling(head_counts, budget):
print(f"OK: every TQ rule is within its test-suite ceiling (base {base})")
return
base_point: Final = resolve_base_point(base)
breaches: Final = evaluate(head_counts, base_counts(base_point), budget)
if not breaches:
print(f"OK: every TQ rule is within its test-suite ceiling (base {base})")
return
new: Final = introduced(
head,
parse_changed_lines(
_run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET])
),
)
print(f"FAIL: TQ-rule totals exceed their limit (base {base}):")
for breach in breaches:
print(
f" {breach.rule}: total {breach.total} over limit {breach.cap} "
f"(this change added {breach.added})"
)
for violation in sorted(v for v in new if v.code == breach.rule):
print(f" {violation.file}:{violation.line}")
print(
"Fix the new violations, or give each one a reason "
"(`# test-quality-ok: <reason>`), or remove an equal number elsewhere; "
"the ceiling is the limit in test-quality-budget.json. "
"Run `python scripts/check_test_quality.py tests/` to see every finding."
)
raise SystemExit(1)
def ratcheted_budget(
budget: Mapping[str, Mapping[str, int]],
current: Mapping[str, int],
base: Mapping[str, int],
seeded: frozenset[str] = frozenset(),
) -> Mapping[str, Mapping[str, int]]:
"""Each rule's limit lowered by the violations `current` fixed vs `base`. The drop
is clamped to what was actually cleared, so a limit only ever falls. Rules in
`seeded` were introduced on this branch and pass through untouched."""
return MappingProxyType({
rule: {
"limit": spec["limit"] if rule in seeded
else max(0, spec["limit"] - max(0, base.get(rule, 0) - current.get(rule, 0)))
}
for rule, spec in sorted(budget.items())
})
def _base_budget_rules(base_point: str) -> frozenset[str]:
proc: Final = subprocess.run(
["git", "show", f"{base_point}:{BUDGET_PATH.name}"],
cwd=REPO_ROOT, capture_output=True, text=True,
)
if proc.returncode != 0:
return frozenset()
return frozenset(json.loads(proc.stdout))
def cmd_update(base_ref: str = DEFAULT_BASE) -> None:
"""Ratchet each rule's limit down by the violations this branch fixed."""
budget: Final = json.loads(BUDGET_PATH.read_text())
base_point: Final = resolve_base_point(base_ref)
seeded: Final = frozenset(budget) - _base_budget_rules(base_point)
updated: Final = ratcheted_budget(
budget, count_by_rule(head_violations()), base_counts(base_point), seeded
)
BUDGET_PATH.write_text(json.dumps(dict(updated), indent=2, sort_keys=True) + "\n")
cleared: Final = sum(budget[rule]["limit"] - updated[rule]["limit"] for rule in updated)
print(f"Ratcheted TQ-rule limits down by {cleared} violations this branch fixed")
if seeded:
print(
"Left untouched (seeded on this branch, absent from the base budget): "
+ ", ".join(sorted(seeded))
)
def cmd_seed() -> None:
"""Write the budget from the working tree's current counts. Used once, to land
the gate; afterwards `--update` is the only thing that may move a limit."""
counts: Final = count_by_rule(head_violations())
BUDGET_PATH.write_text(
json.dumps({rule: {"limit": counts[rule]} for rule in sorted(counts)}, indent=2) + "\n"
)
print(f"Seeded {BUDGET_PATH.name} at " + ", ".join(f"{r}={counts[r]}" for r in sorted(counts)))
def main() -> None:
parser: Final = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base", default=DEFAULT_BASE)
parser.add_argument("--update", action="store_true")
parser.add_argument("--seed", action="store_true")
args: Final = parser.parse_args()
from gate_slot_lock import held_slot
with held_slot():
if args.seed:
cmd_seed()
elif args.update:
cmd_update(args.base)
else:
cmd_check(args.base)
if __name__ == "__main__":
main()

17
test-quality-budget.json Normal file
View file

@ -0,0 +1,17 @@
{
"TQ001": {
"limit": 750
},
"TQ002": {
"limit": 742
},
"TQ003": {
"limit": 1078
},
"TQ004": {
"limit": 770
},
"TQ005": {
"limit": 2835
}
}

17
tests/llm_translation/test_bedrock_completion.py
View file

@ -1195,23 +1195,6 @@ def test_not_found_error():
)
@pytest.mark.parametrize(
"model",
[
"bedrock/us.anthropic.claude-3-haiku-20240307-v1:0",
"bedrock/us.meta.llama3-2-11b-instruct-v1:0",
],
)
def test_bedrock_cross_region_inference(model):
litellm.set_verbose = True
response = completion(
model=model,
messages=messages,
max_tokens=10,
temperature=0.1,
)
@pytest.mark.parametrize(
"model, expected_base_model",
[

6
tests/llm_translation/test_openai.py
View file

@ -285,12 +285,6 @@ class TestOpenAIChatCompletion(BaseLLMChatTest):
"""Test that tool calls with no arguments is translated correctly. Relevant issue: https://github.com/BerriAI/litellm/issues/6833"""
pass
def test_prompt_caching(self):
"""
Test that prompt caching works correctly.
Skip for now, as it's working locally but not in CI
"""
pass
def test_prompt_caching(self):
"""

5
tests/local_testing/test_completion_cost.py
View file

@ -2762,11 +2762,6 @@ def model_item():
}
@pytest.mark.parametrize("base_model_arg", ["litellm_param", "model_info"])
def test_cost_calculator_with_base_model_with_router(base_model_arg, model_item):
from litellm import Router
@pytest.mark.parametrize("base_model_arg", ["litellm_param", "model_info"])
def test_cost_calculator_with_base_model_with_router(base_model_arg):
from litellm import Router

24
tests/logging_callback_tests/test_sqs_logger.py
View file

@ -150,30 +150,6 @@ async def test_async_sqs_logger_error_flush():
# =============================================================================
@pytest.mark.asyncio
async def test_async_log_success_event_adds_to_queue(monkeypatch):
monkeypatch.setattr("litellm.aws_sqs_callback_params", {})
logger = SQSLogger(sqs_queue_url="https://example.com", sqs_region_name="us-west-2")
fake_payload = {"some": "data"}
await logger.async_log_success_event(
{"standard_logging_object": fake_payload}, None, None, None
)
assert fake_payload in logger.log_queue
@pytest.mark.asyncio
async def test_async_log_failure_event_adds_to_queue(monkeypatch):
monkeypatch.setattr("litellm.aws_sqs_callback_params", {})
logger = SQSLogger(sqs_queue_url="https://example.com", sqs_region_name="us-west-2")
fake_payload = {"fail": True}
await logger.async_log_failure_event(
{"standard_logging_object": fake_payload}, None, None, None
)
assert fake_payload in logger.log_queue
# =============================================================================
# 🧾 async_send_batch Tests
# =============================================================================

50
tests/old_proxy_tests/tests/bursty_load_test_completion.py
View file

@ -1,50 +0,0 @@
import time, asyncio
from openai import AsyncOpenAI
from litellm._uuid import uuid
import traceback
litellm_client = AsyncOpenAI(api_key="test", base_url="http://0.0.0.0:8000")
async def litellm_completion():
# Your existing code for litellm_completion goes here
try:
response = await litellm_client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": f"This is a test: {uuid.uuid4()}" * 180}
], # this is about 4k tokens per request
)
return response
except Exception as e:
# If there's an exception, log the error message
with open("error_log.txt", "a") as error_log:
error_log.write(f"Error during completion: {str(e)}\n")
pass
async def main():
start = time.time()
n = 60 # Send 60 concurrent requests, each with 4k tokens = 240k Tokens
tasks = [litellm_completion() for _ in range(n)]
chat_completions = await asyncio.gather(*tasks)
successful_completions = [c for c in chat_completions if c is not None]
# Write errors to error_log.txt
with open("error_log.txt", "a") as error_log:
for completion in chat_completions:
if isinstance(completion, str):
error_log.write(completion + "\n")
print(n, time.time() - start, len(successful_completions))
if __name__ == "__main__":
# Blank out contents of error_log.txt
open("error_log.txt", "w").close()
asyncio.run(main())

0
tests/old_proxy_tests/tests/error_log.txt
View file

112
tests/old_proxy_tests/tests/large_text.py
View file

@ -1,112 +0,0 @@
text = """
Alexander the Great
This article is about the ancient king of Macedonia. For other uses, see Alexander the Great (disambiguation).
Alexander III of Macedon (Ancient Greek: Ἀλέξανδρος, romanized: Alexandros; 20/21 July 356 BC – 10/11 June 323 BC), most commonly known as Alexander the Great,[c] was a king of the ancient Greek kingdom of Macedon.[d] He succeeded his father Philip II to the throne in 336 BC at the age of 20 and spent most of his ruling years conducting a lengthy military campaign throughout Western Asia, Central Asia, parts of South Asia, and Egypt. By the age of 30, he had created one of the largest empires in history, stretching from Greece to northwestern India.[1] He was undefeated in battle and is widely considered to be one of history's greatest and most successful military commanders.[2][3]
Until the age of 16, Alexander was tutored by Aristotle. In 335 BC, shortly after his assumption of kingship over Macedon, he campaigned in the Balkans and reasserted control over Thrace and parts of Illyria before marching on the city of Thebes, which was subsequently destroyed in battle. Alexander then led the League of Corinth, and used his authority to launch the pan-Hellenic project envisaged by his father, assuming leadership over all Greeks in their conquest of Persia.[4][5]
In 334 BC, he invaded the Achaemenid Persian Empire and began a series of campaigns that lasted for 10 years. Following his conquest of Asia Minor, Alexander broke the power of Achaemenid Persia in a series of decisive battles, including those at Issus and Gaugamela; he subsequently overthrew Darius III and conquered the Achaemenid Empire in its entirety.[e] After the fall of Persia, the Macedonian Empire held a vast swath of territory between the Adriatic Sea and the Indus River. Alexander endeavored to reach the "ends of the world and the Great Outer Sea" and invaded India in 326 BC, achieving an important victory over Porus, an ancient Indian king of present-day Punjab, at the Battle of the Hydaspes. Due to the demand of his homesick troops, he eventually turned back at the Beas River and later died in 323 BC in Babylon, the city of Mesopotamia that he had planned to establish as his empire's capital. Alexander's death left unexecuted an additional series of planned military and mercantile campaigns that would have begun with a Greek invasion of Arabia. In the years following his death, a series of civil wars broke out across the Macedonian Empire, eventually leading to its disintegration at the hands of the Diadochi.
With his death marking the start of the Hellenistic period, Alexander's legacy includes the cultural diffusion and syncretism that his conquests engendered, such as Greco-Buddhism and Hellenistic Judaism. He founded more than twenty cities, with the most prominent being the city of Alexandria in Egypt. Alexander's settlement of Greek colonists and the resulting spread of Greek culture led to the overwhelming dominance of Hellenistic civilization and influence as far east as the Indian subcontinent. The Hellenistic period developed through the Roman Empire into modern Western culture; the Greek language became the lingua franca of the region and was the predominant language of the Byzantine Empire up until its collapse in the mid-15th century AD. Alexander became legendary as a classical hero in the mould of Achilles, featuring prominently in the historical and mythical traditions of both Greek and non-Greek cultures. His military achievements and unprecedented enduring successes in battle made him the measure against which many later military leaders would compare themselves,[f] and his tactics remain a significant subject of study in military academies worldwide.[6] Legends of Alexander's exploits coalesced into the third-century Alexander Romance which, in the premodern period, went through over one hundred recensions, translations, and derivations and was translated into almost every European vernacular and every language of the Islamic world.[7] After the Bible, it was the most popular form of European literature.[8]
Early life
Lineage and childhood
Alexander III was born in Pella, the capital of the Kingdom of Macedon,[9] on the sixth day of the ancient Greek month of Hekatombaion, which probably corresponds to 20 July 356 BC (although the exact date is uncertain).[10][11] He was the son of the erstwhile king of Macedon, Philip II, and his fourth wife, Olympias (daughter of Neoptolemus I, king of Epirus).[12][g] Although Philip had seven or eight wives, Olympias was his principal wife for some time, likely because she gave birth to Alexander.[13]
Several legends surround Alexander's birth and childhood.[14] According to the ancient Greek biographer Plutarch, on the eve of the consummation of her marriage to Philip, Olympias dreamed that her womb was struck by a thunderbolt that caused a flame to spread "far and wide" before dying away. Sometime after the wedding, Philip is said to have seen himself, in a dream, securing his wife's womb with a seal engraved with a lion's image.[15] Plutarch offered a variety of interpretations for these dreams: that Olympias was pregnant before her marriage, indicated by the sealing of her womb; or that Alexander's father was Zeus. Ancient commentators were divided about whether the ambitious Olympias promulgated the story of Alexander's divine parentage, variously claiming that she had told Alexander, or that she dismissed the suggestion as impious.[15]
On the day Alexander was born, Philip was preparing a siege on the city of Potidea on the peninsula of Chalcidice. That same day, Philip received news that his general Parmenion had defeated the combined Illyrian and Paeonian armies and that his horses had won at the Olympic Games. It was also said that on this day, the Temple of Artemis in Ephesus, one of the Seven Wonders of the World, burnt down. This led Hegesias of Magnesia to say that it had burnt down because Artemis was away, attending the birth of Alexander.[16] Such legends may have emerged when Alexander was king, and possibly at his instigation, to show that he was superhuman and destined for greatness from conception.[14]
In his early years, Alexander was raised by a nurse, Lanike, sister of Alexander's future general Cleitus the Black. Later in his childhood, Alexander was tutored by the strict Leonidas, a relative of his mother, and by Lysimachus of Acarnania.[17] Alexander was raised in the manner of noble Macedonian youths, learning to read, play the lyre, ride, fight, and hunt.[18] When Alexander was ten years old, a trader from Thessaly brought Philip a horse, which he offered to sell for thirteen talents. The horse refused to be mounted, and Philip ordered it away. Alexander, however, detecting the horse's fear of its own shadow, asked to tame the horse, which he eventually managed.[14] Plutarch stated that Philip, overjoyed at this display of courage and ambition, kissed his son tearfully, declaring: "My boy, you must find a kingdom big enough for your ambitions. Macedon is too small for you", and bought the horse for him.[19] Alexander named it Bucephalas, meaning "ox-head". Bucephalas carried Alexander as far as India. When the animal died (because of old age, according to Plutarch, at age 30), Alexander named a city after him, Bucephala.[20]
Education
When Alexander was 13, Philip began to search for a tutor, and considered such academics as Isocrates and Speusippus, the latter offering to resign from his stewardship of the Academy to take up the post. In the end, Philip chose Aristotle and provided the Temple of the Nymphs at Mieza as a classroom. In return for teaching Alexander, Philip agreed to rebuild Aristotle's hometown of Stageira, which Philip had razed, and to repopulate it by buying and freeing the ex-citizens who were slaves, or pardoning those who were in exile.[21]
Mieza was like a boarding school for Alexander and the children of Macedonian nobles, such as Ptolemy, Hephaistion, and Cassander. Many of these students would become his friends and future generals, and are often known as the "Companions". Aristotle taught Alexander and his companions about medicine, philosophy, morals, religion, logic, and art. Under Aristotle's tutelage, Alexander developed a passion for the works of Homer, and in particular the Iliad; Aristotle gave him an annotated copy, which Alexander later carried on his campaigns.[22] Alexander was able to quote Euripides from memory.[23]
During his youth, Alexander was also acquainted with Persian exiles at the Macedonian court, who received the protection of Philip II for several years as they opposed Artaxerxes III.[24][25][26] Among them were Artabazos II and his daughter Barsine, possible future mistress of Alexander, who resided at the Macedonian court from 352 to 342 BC, as well as Amminapes, future satrap of Alexander, or a Persian nobleman named Sisines.[24][27][28][29] This gave the Macedonian court a good knowledge of Persian issues, and may even have influenced some of the innovations in the management of the Macedonian state.[27]
Suda writes that Anaximenes of Lampsacus was one of Alexander's teachers, and that Anaximenes also accompanied Alexander on his campaigns.[30]
Heir of Philip II
Regency and ascent of Macedon
Main articles: Philip II of Macedon and Rise of Macedon
Further information: History of Macedonia (ancient kingdom)
At the age of 16, Alexander's education under Aristotle ended. Philip II had waged war against the Thracians to the north, which left Alexander in charge as regent and heir apparent.[14] During Philip's absence, the Thracian tribe of Maedi revolted against Macedonia. Alexander responded quickly and drove them from their territory. The territory was colonized, and a city, named Alexandropolis, was founded.[31]
Upon Philip's return, Alexander was dispatched with a small force to subdue the revolts in southern Thrace. Campaigning against the Greek city of Perinthus, Alexander reportedly saved his father's life. Meanwhile, the city of Amphissa began to work lands that were sacred to Apollo near Delphi, a sacrilege that gave Philip the opportunity to further intervene in Greek affairs. While Philip was occupied in Thrace, Alexander was ordered to muster an army for a campaign in southern Greece. Concerned that other Greek states might intervene, Alexander made it look as though he was preparing to attack Illyria instead. During this turmoil, the Illyrians invaded Macedonia, only to be repelled by Alexander.[32]
Philip and his army joined his son in 338 BC, and they marched south through Thermopylae, taking it after stubborn resistance from its Theban garrison. They went on to occupy the city of Elatea, only a few days' march from both Athens and Thebes. The Athenians, led by Demosthenes, voted to seek alliance with Thebes against Macedonia. Both Athens and Philip sent embassies to win Thebes's favour, but Athens won the contest.[33] Philip marched on Amphissa (ostensibly acting on the request of the Amphictyonic League), capturing the mercenaries sent there by Demosthenes and accepting the city's surrender. Philip then returned to Elatea, sending a final offer of peace to Athens and Thebes, who both rejected it.[34]
As Philip marched south, his opponents blocked him near Chaeronea, Boeotia. During the ensuing Battle of Chaeronea, Philip commanded the right wing and Alexander the left, accompanied by a group of Philip's trusted generals. According to the ancient sources, the two sides fought bitterly for some time. Philip deliberately commanded his troops to retreat, counting on the untested Athenian hoplites to follow, thus breaking their line. Alexander was the first to break the Theban lines, followed by Philip's generals. Having damaged the enemy's cohesion, Philip ordered his troops to press forward and quickly routed them. With the Athenians lost, the Thebans were surrounded. Left to fight alone, they were defeated.[35]
After the victory at Chaeronea, Philip and Alexander marched unopposed into the Peloponnese, welcomed by all cities; however, when they reached Sparta, they were refused, but did not resort to war.[36] At Corinth, Philip established a "Hellenic Alliance" (modelled on the old anti-Persian alliance of the Greco-Persian Wars), which included most Greek city-states except Sparta. Philip was then named Hegemon (often translated as "Supreme Commander") of this league (known by modern scholars as the League of Corinth), and announced his plans to attack the Persian Empire.[37][38]
Exile and return
When Philip returned to Pella, he fell in love with and married Cleopatra Eurydice in 338 BC,[39] the niece of his general Attalus.[40] The marriage made Alexander's position as heir less secure, since any son of Cleopatra Eurydice would be a fully Macedonian heir, while Alexander was only half-Macedonian.[41] During the wedding banquet, a drunken Attalus publicly prayed to the gods that the union would produce a legitimate heir.[40]
At the wedding of Cleopatra, whom Philip fell in love with and married, she being much too young for him, her uncle Attalus in his drink desired the Macedonians would implore the gods to give them a lawful successor to the kingdom by his niece. This so irritated Alexander, that throwing one of the cups at his head, "You villain," said he, "what, am I then a bastard?" Then Philip, taking Attalus's part, rose up and would have run his son through; but by good fortune for them both, either his over-hasty rage, or the wine he had drunk, made his foot slip, so that he fell down on the floor. At which Alexander reproachfully insulted over him: "See there," said he, "the man who makes preparations to pass out of Europe into Asia, overturned in passing from one seat to another."
— Plutarch, describing the feud at Philip's wedding.[42]none
In 337 BC, Alexander fled Macedon with his mother, dropping her off with her brother, King Alexander I of Epirus in Dodona, capital of the Molossians.[43] He continued to Illyria,[43] where he sought refuge with one or more Illyrian kings, perhaps with Glaucias, and was treated as a guest, despite having defeated them in battle a few years before.[44] However, it appears Philip never intended to disown his politically and militarily trained son.[43] Accordingly, Alexander returned to Macedon after six months due to the efforts of a family friend, Demaratus, who mediated between the two parties.[45]
In the following year, the Persian satrap (governor) of Caria, Pixodarus, offered his eldest daughter to Alexander's half-brother, Philip Arrhidaeus.[43] Olympias and several of Alexander's friends suggested this showed Philip intended to make Arrhidaeus his heir.[43] Alexander reacted by sending an actor, Thessalus of Corinth, to tell Pixodarus that he should not offer his daughter's hand to an illegitimate son, but instead to Alexander. When Philip heard of this, he stopped the negotiations and scolded Alexander for wishing to marry the daughter of a Carian, explaining that he wanted a better bride for him.[43] Philip exiled four of Alexander's friends, Harpalus, Nearchus, Ptolemy and Erigyius, and had the Corinthians bring Thessalus to him in chains.[46]
King of Macedon
Accession
Further information: Government of Macedonia (ancient kingdom)
In summer 336 BC, while at Aegae attending the wedding of his daughter Cleopatra to Olympias's brother, Alexander I of Epirus, Philip was assassinated by the captain of his bodyguards, Pausanias.[h] As Pausanias tried to escape, he tripped over a vine and was killed by his pursuers, including two of Alexander's companions, Perdiccas and Leonnatus. Alexander was proclaimed king on the spot by the nobles and army at the age of 20.[47][48][49]
Consolidation of power
Alexander began his reign by eliminating potential rivals to the throne. He had his cousin, the former Amyntas IV, executed.[51] He also had two Macedonian princes from the region of Lyncestis killed for having been involved in his father's assassination, but spared a third, Alexander Lyncestes. Olympias had Cleopatra Eurydice, and Europa, her daughter by Philip, burned alive. When Alexander learned about this, he was furious. Alexander also ordered the murder of Attalus,[51] who was in command of the advance guard of the army in Asia Minor and Cleopatra's uncle.[52]
Attalus was at that time corresponding with Demosthenes, regarding the possibility of defecting to Athens. Attalus also had severely insulted Alexander, and following Cleopatra's murder, Alexander may have considered him too dangerous to be left alive.[52] Alexander spared Arrhidaeus, who was by all accounts mentally disabled, possibly as a result of poisoning by Olympias.[47][49][53]
News of Philip's death roused many states into revolt, including Thebes, Athens, Thessaly, and the Thracian tribes north of Macedon. When news of the revolts reached Alexander, he responded quickly. Though advised to use diplomacy, Alexander mustered 3,000 Macedonian cavalry and rode south towards Thessaly. He found the Thessalian army occupying the pass between Mount Olympus and Mount Ossa, and ordered his men to ride over Mount Ossa. When the Thessalians awoke the next day, they found Alexander in their rear and promptly surrendered, adding their cavalry to Alexander's force. He then continued south towards the Peloponnese.[54]
Alexander stopped at Thermopylae, where he was recognized as the leader of the Amphictyonic League before heading south to Corinth. Athens sued for peace and Alexander pardoned the rebels. The famous encounter between Alexander and Diogenes the Cynic occurred during Alexander's stay in Corinth. When Alexander asked Diogenes what he could do for him, the philosopher disdainfully asked Alexander to stand a little to the side, as he was blocking the sunlight.[55] This reply apparently delighted Alexander, who is reported to have said "But verily, if I were not Alexander, I would like to be Diogenes."[56] At Corinth, Alexander took the title of Hegemon ("leader") and, like Philip, was appointed commander for the coming war against Persia. He also received news of a Thracian uprising.[57]
Balkan campaign
Main article: Alexander's Balkan campaign
Before crossing to Asia, Alexander wanted to safeguard his northern borders. In the spring of 335 BC, he advanced to suppress several revolts. Starting from Amphipolis, he travelled east into the country of the "Independent Thracians"; and at Mount Haemus, the Macedonian army attacked and defeated the Thracian forces manning the heights.[58] The Macedonians marched into the country of the Triballi, and defeated their army near the Lyginus river[59] (a tributary of the Danube). Alexander then marched for three days to the Danube, encountering the Getae tribe on the opposite shore. Crossing the river at night, he surprised them and forced their army to retreat after the first cavalry skirmish.[60]
News then reached Alexander that the Illyrian chieftain Cleitus and King Glaukias of the Taulantii were in open revolt against his authority. Marching west into Illyria, Alexander defeated each in turn, forcing the two rulers to flee with their troops. With these victories, he secured his northern frontier.[61]
Destruction of Thebes
While Alexander campaigned north, the Thebans and Athenians rebelled once again. Alexander immediately headed south.[62] While the other cities again hesitated, Thebes decided to fight. The Theban resistance was ineffective, and Alexander razed the city and divided its territory between the other Boeotian cities. The end of Thebes cowed Athens, leaving all of Greece temporarily at peace.[62] Alexander then set out on his Asian campaign, leaving Antipater as regent.[63]
Conquest of the Achaemenid Persian Empire
Main articles: Wars of Alexander the Great and Chronology of the expedition of Alexander the Great into Asia
Asia Minor
Further information: Battle of the Granicus, Siege of Halicarnassus, and Siege of Miletus
After his victory at the Battle of Chaeronea (338 BC), Philip II began the work of establishing himself as hēgemṓn (Greek: ἡγεμών) of a league which according to Diodorus was to wage a campaign against the Persians for the sundry grievances Greece suffered in 480 and free the Greek cities of the western coast and islands from Achaemenid rule. In 336 he sent Parmenion, Amyntas, Andromenes, Attalus, and an army of 10,000 men into Anatolia to make preparations for an invasion.[64][65] At first, all went well. The Greek cities on the western coast of Anatolia revolted until the news arrived that Philip had been murdered and had been succeeded by his young son Alexander. The Macedonians were demoralized by Philip's death and were subsequently defeated near Magnesia by the Achaemenids under the command of the mercenary Memnon of Rhodes.[64][65]
Taking over the invasion project of Philip II, Alexander's army crossed the Hellespont in 334 BC with approximately 48,100 soldiers, 6,100 cavalry and a fleet of 120 ships with crews numbering 38,000,[62] drawn from Macedon and various Greek city-states, mercenaries, and feudally raised soldiers from Thrace, Paionia, and Illyria.[66][i] He showed his intent to conquer the entirety of the Persian Empire by throwing a spear into Asian soil and saying he accepted Asia as a gift from the gods. This also showed Alexander's eagerness to fight, in contrast to his father's preference for diplomacy.[62]
After an initial victory against Persian forces at the Battle of the Granicus, Alexander accepted the surrender of the Persian provincial capital and treasury of Sardis; he then proceeded along the Ionian coast, granting autonomy and democracy to the cities. Miletus, held by Achaemenid forces, required a delicate siege operation, with Persian naval forces nearby. Further south, at Halicarnassus, in Caria, Alexander successfully waged his first large-scale siege, eventually forcing his opponents, the mercenary captain Memnon of Rhodes and the Persian satrap of Caria, Orontobates, to withdraw by sea.[67] Alexander left the government of Caria to a member of the Hecatomnid dynasty, Ada, who adopted Alexander.[68]
From Halicarnassus, Alexander proceeded into mountainous Lycia and the Pamphylian plain, asserting control over all coastal cities to deny the Persians naval bases. From Pamphylia onwards the coast held no major ports and Alexander moved inland. At Termessos, Alexander humbled but did not storm the Pisidian city.[69] At the ancient Phrygian capital of Gordium, Alexander "undid" the hitherto unsolvable Gordian Knot, a feat said to await the future "king of Asia".[70] According to the story, Alexander proclaimed that it did not matter how the knot was undone and hacked it apart with his sword.[71]
The Levant and Syria
Further information: Battle of Issus and Siege of Tyre (332 BC)
In spring 333 BC, Alexander crossed the Taurus into Cilicia. After a long pause due to an illness, he marched on towards Syria. Though outmanoeuvered by Darius's significantly larger army, he marched back to Cilicia, where he defeated Darius at Issus. Darius fled the battle, causing his army to collapse, and left behind his wife, his two daughters, his mother Sisygambis, and a fabulous treasure.[72] He offered a peace treaty that included the lands he had already lost, and a ransom of 10,000 talents for his family. Alexander replied that since he was now king of Asia, it was he alone who decided territorial divisions.[73] Alexander proceeded to take possession of Syria, and most of the coast of the Levant.[68] In the following year, 332 BC, he was forced to attack Tyre, which he captured after a long and difficult siege.[74][75] The men of military age were massacred and the women and children sold into slavery.[76]
Egypt
Further information: Siege of Gaza (332 BCE)
When Alexander destroyed Tyre, most of the towns on the route to Egypt quickly capitulated. However, Alexander was met with resistance at Gaza. The stronghold was heavily fortified and built on a hill, requiring a siege. When "his engineers pointed out to him that because of the height of the mound it would be impossible... this encouraged Alexander all the more to make the attempt".[77] After three unsuccessful assaults, the stronghold fell, but not before Alexander had received a serious shoulder wound. As in Tyre, men of military age were put to the sword and the women and children were sold into slavery.[78]
"""

353
tests/old_proxy_tests/tests/llama_index_data/essay.txt
View file

@ -1,353 +0,0 @@
What I Worked On
February 2021
Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.
The first programs I tried writing were on the IBM 1401 that our school district used for what was then called "data processing." This was in 9th grade, so I was 13 or 14. The school district's 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain's lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights.
The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer.
I was puzzled by the 1401. I couldn't figure out what to do with it. And in retrospect there's not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn't have any data stored on punched cards. The only other option was to do things that didn't rely on any input, like calculate approximations of pi, but I didn't know enough math to do anything interesting of that type. So I'm not surprised I can't remember any programs I wrote, because they can't have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn't. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager's expression made clear.
With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1]
The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer.
Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he'd write 2 pages at a time and then print them out, but it was a lot better than a typewriter.
Though I liked programming, I didn't plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn't much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored.
I couldn't have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI.
AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven't tried rereading The Moon is a Harsh Mistress, so I don't know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we'd have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words.
There weren't any classes in AI at Cornell then, not even graduate classes, so I started trying to teach myself. Which meant learning Lisp, since in those days Lisp was regarded as the language of AI. The commonly used programming languages then were pretty primitive, and programmers' ideas correspondingly so. The default language at Cornell was a Pascal-like language called PL/I, and the situation was similar elsewhere. Learning Lisp expanded my concept of a program so fast that it was years before I started to have a sense of where the new limits were. This was more like it; this was what I had expected college to do. It wasn't happening in a class, like it was supposed to, but that was ok. For the next couple years I was on a roll. I knew what I was going to do.
For my undergraduate thesis, I reverse-engineered SHRDLU. My God did I love working on that program. It was a pleasing bit of code, but what made it even more exciting was my belief — hard to imagine now, but not unique in 1985 — that it was already climbing the lower slopes of intelligence.
I had gotten into a program at Cornell that didn't make you choose a major. You could take whatever classes you liked, and choose whatever you liked to put on your degree. I of course chose "Artificial Intelligence." When I got the actual physical diploma, I was dismayed to find that the quotes had been included, which made them read as scare-quotes. At the time this bothered me, but now it seems amusingly accurate, for reasons I was about to discover.
I applied to 3 grad schools: MIT and Yale, which were renowned for AI at the time, and Harvard, which I'd visited because Rich Draves went there, and was also home to Bill Woods, who'd invented the type of parser I used in my SHRDLU clone. Only Harvard accepted me, so that was where I went.
I don't remember the moment it happened, or if there even was a specific moment, but during the first year of grad school I realized that AI, as practiced at the time, was a hoax. By which I mean the sort of AI in which a program that's told "the dog is sitting on the chair" translates this into some formal representation and adds it to the list of things it knows.
What these programs really showed was that there's a subset of natural language that's a formal language. But a very proper subset. It was clear that there was an unbridgeable gap between what they could do and actually understanding natural language. It was not, in fact, simply a matter of teaching SHRDLU more words. That whole way of doing AI, with explicit data structures representing concepts, was not going to work. Its brokenness did, as so often happens, generate a lot of opportunities to write papers about various band-aids that could be applied to it, but it was never going to get us Mike.
So I looked around to see what I could salvage from the wreckage of my plans, and there was Lisp. I knew from experience that Lisp was interesting for its own sake and not just for its association with AI, even though that was the main reason people cared about it at the time. So I decided to focus on Lisp. In fact, I decided to write a book about Lisp hacking. It's scary to think how little I knew about Lisp hacking when I started writing that book. But there's nothing like writing a book about something to help you learn it. The book, On Lisp, wasn't published till 1993, but I wrote much of it in grad school.
Computer Science is an uneasy alliance between two halves, theory and systems. The theory people prove things, and the systems people build things. I wanted to build things. I had plenty of respect for theory — indeed, a sneaking suspicion that it was the more admirable of the two halves — but building things seemed so much more exciting.
The problem with systems work, though, was that it didn't last. Any program you wrote today, no matter how good, would be obsolete in a couple decades at best. People might mention your software in footnotes, but no one would actually use it. And indeed, it would seem very feeble work. Only people with a sense of the history of the field would even realize that, in its time, it had been good.
There were some surplus Xerox Dandelions floating around the computer lab at one point. Anyone who wanted one to play around with could have one. I was briefly tempted, but they were so slow by present standards; what was the point? No one else wanted one either, so off they went. That was what happened to systems work.
I wanted not just to build things, but to build things that would last.
In this dissatisfied state I went in 1988 to visit Rich Draves at CMU, where he was in grad school. One day I went to visit the Carnegie Institute, where I'd spent a lot of time as a kid. While looking at a painting there I realized something that might seem obvious, but was a big surprise to me. There, right on the wall, was something you could make that would last. Paintings didn't become obsolete. Some of the best ones were hundreds of years old.
And moreover this was something you could make a living doing. Not as easily as you could by writing software, of course, but I thought if you were really industrious and lived really cheaply, it had to be possible to make enough to survive. And as an artist you could be truly independent. You wouldn't have a boss, or even need to get research funding.
I had always liked looking at paintings. Could I make them? I had no idea. I'd never imagined it was even possible. I knew intellectually that people made art — that it didn't just appear spontaneously — but it was as if the people who made it were a different species. They either lived long ago or were mysterious geniuses doing strange things in profiles in Life magazine. The idea of actually being able to make art, to put that verb before that noun, seemed almost miraculous.
That fall I started taking art classes at Harvard. Grad students could take classes in any department, and my advisor, Tom Cheatham, was very easy going. If he even knew about the strange classes I was taking, he never said anything.
So now I was in a PhD program in computer science, yet planning to be an artist, yet also genuinely in love with Lisp hacking and working away at On Lisp. In other words, like many a grad student, I was working energetically on multiple projects that were not my thesis.
I didn't see a way out of this situation. I didn't want to drop out of grad school, but how else was I going to get out? I remember when my friend Robert Morris got kicked out of Cornell for writing the internet worm of 1988, I was envious that he'd found such a spectacular way to get out of grad school.
Then one day in April 1990 a crack appeared in the wall. I ran into professor Cheatham and he asked if I was far enough along to graduate that June. I didn't have a word of my dissertation written, but in what must have been the quickest bit of thinking in my life, I decided to take a shot at writing one in the 5 weeks or so that remained before the deadline, reusing parts of On Lisp where I could, and I was able to respond, with no perceptible delay "Yes, I think so. I'll give you something to read in a few days."
I picked applications of continuations as the topic. In retrospect I should have written about macros and embedded languages. There's a whole world there that's barely been explored. But all I wanted was to get out of grad school, and my rapidly written dissertation sufficed, just barely.
Meanwhile I was applying to art schools. I applied to two: RISD in the US, and the Accademia di Belli Arti in Florence, which, because it was the oldest art school, I imagined would be good. RISD accepted me, and I never heard back from the Accademia, so off to Providence I went.
I'd applied for the BFA program at RISD, which meant in effect that I had to go to college again. This was not as strange as it sounds, because I was only 25, and art schools are full of people of different ages. RISD counted me as a transfer sophomore and said I had to do the foundation that summer. The foundation means the classes that everyone has to take in fundamental subjects like drawing, color, and design.
Toward the end of the summer I got a big surprise: a letter from the Accademia, which had been delayed because they'd sent it to Cambridge England instead of Cambridge Massachusetts, inviting me to take the entrance exam in Florence that fall. This was now only weeks away. My nice landlady let me leave my stuff in her attic. I had some money saved from consulting work I'd done in grad school; there was probably enough to last a year if I lived cheaply. Now all I had to do was learn Italian.
Only stranieri (foreigners) had to take this entrance exam. In retrospect it may well have been a way of excluding them, because there were so many stranieri attracted by the idea of studying art in Florence that the Italian students would otherwise have been outnumbered. I was in decent shape at painting and drawing from the RISD foundation that summer, but I still don't know how I managed to pass the written exam. I remember that I answered the essay question by writing about Cezanne, and that I cranked up the intellectual level as high as I could to make the most of my limited vocabulary. [2]
I'm only up to age 25 and already there are such conspicuous patterns. Here I was, yet again about to attend some august institution in the hopes of learning about some prestigious subject, and yet again about to be disappointed. The students and faculty in the painting department at the Accademia were the nicest people you could imagine, but they had long since arrived at an arrangement whereby the students wouldn't require the faculty to teach anything, and in return the faculty wouldn't require the students to learn anything. And at the same time all involved would adhere outwardly to the conventions of a 19th century atelier. We actually had one of those little stoves, fed with kindling, that you see in 19th century studio paintings, and a nude model sitting as close to it as possible without getting burned. Except hardly anyone else painted her besides me. The rest of the students spent their time chatting or occasionally trying to imitate things they'd seen in American art magazines.
Our model turned out to live just down the street from me. She made a living from a combination of modelling and making fakes for a local antique dealer. She'd copy an obscure old painting out of a book, and then he'd take the copy and maltreat it to make it look old. [3]
While I was a student at the Accademia I started painting still lives in my bedroom at night. These paintings were tiny, because the room was, and because I painted them on leftover scraps of canvas, which was all I could afford at the time. Painting still lives is different from painting people, because the subject, as its name suggests, can't move. People can't sit for more than about 15 minutes at a time, and when they do they don't sit very still. So the traditional m.o. for painting people is to know how to paint a generic person, which you then modify to match the specific person you're painting. Whereas a still life you can, if you want, copy pixel by pixel from what you're seeing. You don't want to stop there, of course, or you get merely photographic accuracy, and what makes a still life interesting is that it's been through a head. You want to emphasize the visual cues that tell you, for example, that the reason the color changes suddenly at a certain point is that it's the edge of an object. By subtly emphasizing such things you can make paintings that are more realistic than photographs not just in some metaphorical sense, but in the strict information-theoretic sense. [4]
I liked painting still lives because I was curious about what I was seeing. In everyday life, we aren't consciously aware of much we're seeing. Most visual perception is handled by low-level processes that merely tell your brain "that's a water droplet" without telling you details like where the lightest and darkest points are, or "that's a bush" without telling you the shape and position of every leaf. This is a feature of brains, not a bug. In everyday life it would be distracting to notice every leaf on every bush. But when you have to paint something, you have to look more closely, and when you do there's a lot to see. You can still be noticing new things after days of trying to paint something people usually take for granted, just as you can after days of trying to write an essay about something people usually take for granted.
This is not the only way to paint. I'm not 100% sure it's even a good way to paint. But it seemed a good enough bet to be worth trying.
Our teacher, professor Ulivi, was a nice guy. He could see I worked hard, and gave me a good grade, which he wrote down in a sort of passport each student had. But the Accademia wasn't teaching me anything except Italian, and my money was running out, so at the end of the first year I went back to the US.
I wanted to go back to RISD, but I was now broke and RISD was very expensive, so I decided to get a job for a year and then return to RISD the next fall. I got one at a company called Interleaf, which made software for creating documents. You mean like Microsoft Word? Exactly. That was how I learned that low end software tends to eat high end software. But Interleaf still had a few years to live yet. [5]
Interleaf had done something pretty bold. Inspired by Emacs, they'd added a scripting language, and even made the scripting language a dialect of Lisp. Now they wanted a Lisp hacker to write things in it. This was the closest thing I've had to a normal job, and I hereby apologize to my boss and coworkers, because I was a bad employee. Their Lisp was the thinnest icing on a giant C cake, and since I didn't know C and didn't want to learn it, I never understood most of the software. Plus I was terribly irresponsible. This was back when a programming job meant showing up every day during certain working hours. That seemed unnatural to me, and on this point the rest of the world is coming around to my way of thinking, but at the time it caused a lot of friction. Toward the end of the year I spent much of my time surreptitiously working on On Lisp, which I had by this time gotten a contract to publish.
The good part was that I got paid huge amounts of money, especially by art student standards. In Florence, after paying my part of the rent, my budget for everything else had been $7 a day. Now I was getting paid more than 4 times that every hour, even when I was just sitting in a meeting. By living cheaply I not only managed to save enough to go back to RISD, but also paid off my college loans.
I learned some useful things at Interleaf, though they were mostly about what not to do. I learned that it's better for technology companies to be run by product people than sales people (though sales is a real skill and people who are good at it are really good at it), that it leads to bugs when code is edited by too many people, that cheap office space is no bargain if it's depressing, that planned meetings are inferior to corridor conversations, that big, bureaucratic customers are a dangerous source of money, and that there's not much overlap between conventional office hours and the optimal time for hacking, or conventional offices and the optimal place for it.
But the most important thing I learned, and which I used in both Viaweb and Y Combinator, is that the low end eats the high end: that it's good to be the "entry level" option, even though that will be less prestigious, because if you're not, someone else will be, and will squash you against the ceiling. Which in turn means that prestige is a danger sign.
When I left to go back to RISD the next fall, I arranged to do freelance work for the group that did projects for customers, and this was how I survived for the next several years. When I came back to visit for a project later on, someone told me about a new thing called HTML, which was, as he described it, a derivative of SGML. Markup language enthusiasts were an occupational hazard at Interleaf and I ignored him, but this HTML thing later became a big part of my life.
In the fall of 1992 I moved back to Providence to continue at RISD. The foundation had merely been intro stuff, and the Accademia had been a (very civilized) joke. Now I was going to see what real art school was like. But alas it was more like the Accademia than not. Better organized, certainly, and a lot more expensive, but it was now becoming clear that art school did not bear the same relationship to art that medical school bore to medicine. At least not the painting department. The textile department, which my next door neighbor belonged to, seemed to be pretty rigorous. No doubt illustration and architecture were too. But painting was post-rigorous. Painting students were supposed to express themselves, which to the more worldly ones meant to try to cook up some sort of distinctive signature style.
A signature style is the visual equivalent of what in show business is known as a "schtick": something that immediately identifies the work as yours and no one else's. For example, when you see a painting that looks like a certain kind of cartoon, you know it's by Roy Lichtenstein. So if you see a big painting of this type hanging in the apartment of a hedge fund manager, you know he paid millions of dollars for it. That's not always why artists have a signature style, but it's usually why buyers pay a lot for such work. [6]
There were plenty of earnest students too: kids who "could draw" in high school, and now had come to what was supposed to be the best art school in the country, to learn to draw even better. They tended to be confused and demoralized by what they found at RISD, but they kept going, because painting was what they did. I was not one of the kids who could draw in high school, but at RISD I was definitely closer to their tribe than the tribe of signature style seekers.
I learned a lot in the color class I took at RISD, but otherwise I was basically teaching myself to paint, and I could do that for free. So in 1993 I dropped out. I hung around Providence for a bit, and then my college friend Nancy Parmet did me a big favor. A rent-controlled apartment in a building her mother owned in New York was becoming vacant. Did I want it? It wasn't much more than my current place, and New York was supposed to be where the artists were. So yes, I wanted it! [7]
Asterix comics begin by zooming in on a tiny corner of Roman Gaul that turns out not to be controlled by the Romans. You can do something similar on a map of New York City: if you zoom in on the Upper East Side, there's a tiny corner that's not rich, or at least wasn't in 1993. It's called Yorkville, and that was my new home. Now I was a New York artist — in the strictly technical sense of making paintings and living in New York.
I was nervous about money, because I could sense that Interleaf was on the way down. Freelance Lisp hacking work was very rare, and I didn't want to have to program in another language, which in those days would have meant C++ if I was lucky. So with my unerring nose for financial opportunity, I decided to write another book on Lisp. This would be a popular book, the sort of book that could be used as a textbook. I imagined myself living frugally off the royalties and spending all my time painting. (The painting on the cover of this book, ANSI Common Lisp, is one that I painted around this time.)
The best thing about New York for me was the presence of Idelle and Julian Weber. Idelle Weber was a painter, one of the early photorealists, and I'd taken her painting class at Harvard. I've never known a teacher more beloved by her students. Large numbers of former students kept in touch with her, including me. After I moved to New York I became her de facto studio assistant.
She liked to paint on big, square canvases, 4 to 5 feet on a side. One day in late 1994 as I was stretching one of these monsters there was something on the radio about a famous fund manager. He wasn't that much older than me, and was super rich. The thought suddenly occurred to me: why don't I become rich? Then I'll be able to work on whatever I want.
Meanwhile I'd been hearing more and more about this new thing called the World Wide Web. Robert Morris showed it to me when I visited him in Cambridge, where he was now in grad school at Harvard. It seemed to me that the web would be a big deal. I'd seen what graphical user interfaces had done for the popularity of microcomputers. It seemed like the web would do the same for the internet.
If I wanted to get rich, here was the next train leaving the station. I was right about that part. What I got wrong was the idea. I decided we should start a company to put art galleries online. I can't honestly say, after reading so many Y Combinator applications, that this was the worst startup idea ever, but it was up there. Art galleries didn't want to be online, and still don't, not the fancy ones. That's not how they sell. I wrote some software to generate web sites for galleries, and Robert wrote some to resize images and set up an http server to serve the pages. Then we tried to sign up galleries. To call this a difficult sale would be an understatement. It was difficult to give away. A few galleries let us make sites for them for free, but none paid us.
Then some online stores started to appear, and I realized that except for the order buttons they were identical to the sites we'd been generating for galleries. This impressive-sounding thing called an "internet storefront" was something we already knew how to build.
So in the summer of 1995, after I submitted the camera-ready copy of ANSI Common Lisp to the publishers, we started trying to write software to build online stores. At first this was going to be normal desktop software, which in those days meant Windows software. That was an alarming prospect, because neither of us knew how to write Windows software or wanted to learn. We lived in the Unix world. But we decided we'd at least try writing a prototype store builder on Unix. Robert wrote a shopping cart, and I wrote a new site generator for stores — in Lisp, of course.
We were working out of Robert's apartment in Cambridge. His roommate was away for big chunks of time, during which I got to sleep in his room. For some reason there was no bed frame or sheets, just a mattress on the floor. One morning as I was lying on this mattress I had an idea that made me sit up like a capital L. What if we ran the software on the server, and let users control it by clicking on links? Then we'd never have to write anything to run on users' computers. We could generate the sites on the same server we'd serve them from. Users wouldn't need anything more than a browser.
This kind of software, known as a web app, is common now, but at the time it wasn't clear that it was even possible. To find out, we decided to try making a version of our store builder that you could control through the browser. A couple days later, on August 12, we had one that worked. The UI was horrible, but it proved you could build a whole store through the browser, without any client software or typing anything into the command line on the server.
Now we felt like we were really onto something. I had visions of a whole new generation of software working this way. You wouldn't need versions, or ports, or any of that crap. At Interleaf there had been a whole group called Release Engineering that seemed to be at least as big as the group that actually wrote the software. Now you could just update the software right on the server.
We started a new company we called Viaweb, after the fact that our software worked via the web, and we got $10,000 in seed funding from Idelle's husband Julian. In return for that and doing the initial legal work and giving us business advice, we gave him 10% of the company. Ten years later this deal became the model for Y Combinator's. We knew founders needed something like this, because we'd needed it ourselves.
At this stage I had a negative net worth, because the thousand dollars or so I had in the bank was more than counterbalanced by what I owed the government in taxes. (Had I diligently set aside the proper proportion of the money I'd made consulting for Interleaf? No, I had not.) So although Robert had his graduate student stipend, I needed that seed funding to live on.
We originally hoped to launch in September, but we got more ambitious about the software as we worked on it. Eventually we managed to build a WYSIWYG site builder, in the sense that as you were creating pages, they looked exactly like the static ones that would be generated later, except that instead of leading to static pages, the links all referred to closures stored in a hash table on the server.
It helped to have studied art, because the main goal of an online store builder is to make users look legit, and the key to looking legit is high production values. If you get page layouts and fonts and colors right, you can make a guy running a store out of his bedroom look more legit than a big company.
(If you're curious why my site looks so old-fashioned, it's because it's still made with this software. It may look clunky today, but in 1996 it was the last word in slick.)
In September, Robert rebelled. "We've been working on this for a month," he said, "and it's still not done." This is funny in retrospect, because he would still be working on it almost 3 years later. But I decided it might be prudent to recruit more programmers, and I asked Robert who else in grad school with him was really good. He recommended Trevor Blackwell, which surprised me at first, because at that point I knew Trevor mainly for his plan to reduce everything in his life to a stack of notecards, which he carried around with him. But Rtm was right, as usual. Trevor turned out to be a frighteningly effective hacker.
It was a lot of fun working with Robert and Trevor. They're the two most independent-minded people I know, and in completely different ways. If you could see inside Rtm's brain it would look like a colonial New England church, and if you could see inside Trevor's it would look like the worst excesses of Austrian Rococo.
We opened for business, with 6 stores, in January 1996. It was just as well we waited a few months, because although we worried we were late, we were actually almost fatally early. There was a lot of talk in the press then about ecommerce, but not many people actually wanted online stores. [8]
There were three main parts to the software: the editor, which people used to build sites and which I wrote, the shopping cart, which Robert wrote, and the manager, which kept track of orders and statistics, and which Trevor wrote. In its time, the editor was one of the best general-purpose site builders. I kept the code tight and didn't have to integrate with any other software except Robert's and Trevor's, so it was quite fun to work on. If all I'd had to do was work on this software, the next 3 years would have been the easiest of my life. Unfortunately I had to do a lot more, all of it stuff I was worse at than programming, and the next 3 years were instead the most stressful.
There were a lot of startups making ecommerce software in the second half of the 90s. We were determined to be the Microsoft Word, not the Interleaf. Which meant being easy to use and inexpensive. It was lucky for us that we were poor, because that caused us to make Viaweb even more inexpensive than we realized. We charged $100 a month for a small store and $300 a month for a big one. This low price was a big attraction, and a constant thorn in the sides of competitors, but it wasn't because of some clever insight that we set the price low. We had no idea what businesses paid for things. $300 a month seemed like a lot of money to us.
We did a lot of things right by accident like that. For example, we did what's now called "doing things that don't scale," although at the time we would have described it as "being so lame that we're driven to the most desperate measures to get users." The most common of which was building stores for them. This seemed particularly humiliating, since the whole raison d'etre of our software was that people could use it to make their own stores. But anything to get users.
We learned a lot more about retail than we wanted to know. For example, that if you could only have a small image of a man's shirt (and all images were small then by present standards), it was better to have a closeup of the collar than a picture of the whole shirt. The reason I remember learning this was that it meant I had to rescan about 30 images of men's shirts. My first set of scans were so beautiful too.
Though this felt wrong, it was exactly the right thing to be doing. Building stores for users taught us about retail, and about how it felt to use our software. I was initially both mystified and repelled by "business" and thought we needed a "business person" to be in charge of it, but once we started to get users, I was converted, in much the same way I was converted to fatherhood once I had kids. Whatever users wanted, I was all theirs. Maybe one day we'd have so many users that I couldn't scan their images for them, but in the meantime there was nothing more important to do.
Another thing I didn't get at the time is that growth rate is the ultimate test of a startup. Our growth rate was fine. We had about 70 stores at the end of 1996 and about 500 at the end of 1997. I mistakenly thought the thing that mattered was the absolute number of users. And that is the thing that matters in the sense that that's how much money you're making, and if you're not making enough, you might go out of business. But in the long term the growth rate takes care of the absolute number. If we'd been a startup I was advising at Y Combinator, I would have said: Stop being so stressed out, because you're doing fine. You're growing 7x a year. Just don't hire too many more people and you'll soon be profitable, and then you'll control your own destiny.
Alas I hired lots more people, partly because our investors wanted me to, and partly because that's what startups did during the Internet Bubble. A company with just a handful of employees would have seemed amateurish. So we didn't reach breakeven until about when Yahoo bought us in the summer of 1998. Which in turn meant we were at the mercy of investors for the entire life of the company. And since both we and our investors were noobs at startups, the result was a mess even by startup standards.
It was a huge relief when Yahoo bought us. In principle our Viaweb stock was valuable. It was a share in a business that was profitable and growing rapidly. But it didn't feel very valuable to me; I had no idea how to value a business, but I was all too keenly aware of the near-death experiences we seemed to have every few months. Nor had I changed my grad student lifestyle significantly since we started. So when Yahoo bought us it felt like going from rags to riches. Since we were going to California, I bought a car, a yellow 1998 VW GTI. I remember thinking that its leather seats alone were by far the most luxurious thing I owned.
The next year, from the summer of 1998 to the summer of 1999, must have been the least productive of my life. I didn't realize it at the time, but I was worn out from the effort and stress of running Viaweb. For a while after I got to California I tried to continue my usual m.o. of programming till 3 in the morning, but fatigue combined with Yahoo's prematurely aged culture and grim cube farm in Santa Clara gradually dragged me down. After a few months it felt disconcertingly like working at Interleaf.
Yahoo had given us a lot of options when they bought us. At the time I thought Yahoo was so overvalued that they'd never be worth anything, but to my astonishment the stock went up 5x in the next year. I hung on till the first chunk of options vested, then in the summer of 1999 I left. It had been so long since I'd painted anything that I'd half forgotten why I was doing this. My brain had been entirely full of software and men's shirts for 4 years. But I had done this to get rich so I could paint, I reminded myself, and now I was rich, so I should go paint.
When I said I was leaving, my boss at Yahoo had a long conversation with me about my plans. I told him all about the kinds of pictures I wanted to paint. At the time I was touched that he took such an interest in me. Now I realize it was because he thought I was lying. My options at that point were worth about $2 million a month. If I was leaving that kind of money on the table, it could only be to go and start some new startup, and if I did, I might take people with me. This was the height of the Internet Bubble, and Yahoo was ground zero of it. My boss was at that moment a billionaire. Leaving then to start a new startup must have seemed to him an insanely, and yet also plausibly, ambitious plan.
But I really was quitting to paint, and I started immediately. There was no time to lose. I'd already burned 4 years getting rich. Now when I talk to founders who are leaving after selling their companies, my advice is always the same: take a vacation. That's what I should have done, just gone off somewhere and done nothing for a month or two, but the idea never occurred to me.
So I tried to paint, but I just didn't seem to have any energy or ambition. Part of the problem was that I didn't know many people in California. I'd compounded this problem by buying a house up in the Santa Cruz Mountains, with a beautiful view but miles from anywhere. I stuck it out for a few more months, then in desperation I went back to New York, where unless you understand about rent control you'll be surprised to hear I still had my apartment, sealed up like a tomb of my old life. Idelle was in New York at least, and there were other people trying to paint there, even though I didn't know any of them.
When I got back to New York I resumed my old life, except now I was rich. It was as weird as it sounds. I resumed all my old patterns, except now there were doors where there hadn't been. Now when I was tired of walking, all I had to do was raise my hand, and (unless it was raining) a taxi would stop to pick me up. Now when I walked past charming little restaurants I could go in and order lunch. It was exciting for a while. Painting started to go better. I experimented with a new kind of still life where I'd paint one painting in the old way, then photograph it and print it, blown up, on canvas, and then use that as the underpainting for a second still life, painted from the same objects (which hopefully hadn't rotted yet).
Meanwhile I looked for an apartment to buy. Now I could actually choose what neighborhood to live in. Where, I asked myself and various real estate agents, is the Cambridge of New York? Aided by occasional visits to actual Cambridge, I gradually realized there wasn't one. Huh.
Around this time, in the spring of 2000, I had an idea. It was clear from our experience with Viaweb that web apps were the future. Why not build a web app for making web apps? Why not let people edit code on our server through the browser, and then host the resulting applications for them? [9] You could run all sorts of services on the servers that these applications could use just by making an API call: making and receiving phone calls, manipulating images, taking credit card payments, etc.
I got so excited about this idea that I couldn't think about anything else. It seemed obvious that this was the future. I didn't particularly want to start another company, but it was clear that this idea would have to be embodied as one, so I decided to move to Cambridge and start it. I hoped to lure Robert into working on it with me, but there I ran into a hitch. Robert was now a postdoc at MIT, and though he'd made a lot of money the last time I'd lured him into working on one of my schemes, it had also been a huge time sink. So while he agreed that it sounded like a plausible idea, he firmly refused to work on it.
Hmph. Well, I'd do it myself then. I recruited Dan Giffin, who had worked for Viaweb, and two undergrads who wanted summer jobs, and we got to work trying to build what it's now clear is about twenty companies and several open source projects worth of software. The language for defining applications would of course be a dialect of Lisp. But I wasn't so naive as to assume I could spring an overt Lisp on a general audience; we'd hide the parentheses, like Dylan did.
By then there was a name for the kind of company Viaweb was, an "application service provider," or ASP. This name didn't last long before it was replaced by "software as a service," but it was current for long enough that I named this new company after it: it was going to be called Aspra.
I started working on the application builder, Dan worked on network infrastructure, and the two undergrads worked on the first two services (images and phone calls). But about halfway through the summer I realized I really didn't want to run a company — especially not a big one, which it was looking like this would have to be. I'd only started Viaweb because I needed the money. Now that I didn't need money anymore, why was I doing this? If this vision had to be realized as a company, then screw the vision. I'd build a subset that could be done as an open source project.
Much to my surprise, the time I spent working on this stuff was not wasted after all. After we started Y Combinator, I would often encounter startups working on parts of this new architecture, and it was very useful to have spent so much time thinking about it and even trying to write some of it.
The subset I would build as an open source project was the new Lisp, whose parentheses I now wouldn't even have to hide. A lot of Lisp hackers dream of building a new Lisp, partly because one of the distinctive features of the language is that it has dialects, and partly, I think, because we have in our minds a Platonic form of Lisp that all existing dialects fall short of. I certainly did. So at the end of the summer Dan and I switched to working on this new dialect of Lisp, which I called Arc, in a house I bought in Cambridge.
The following spring, lightning struck. I was invited to give a talk at a Lisp conference, so I gave one about how we'd used Lisp at Viaweb. Afterward I put a postscript file of this talk online, on paulgraham.com, which I'd created years before using Viaweb but had never used for anything. In one day it got 30,000 page views. What on earth had happened? The referring urls showed that someone had posted it on Slashdot. [10]
Wow, I thought, there's an audience. If I write something and put it on the web, anyone can read it. That may seem obvious now, but it was surprising then. In the print era there was a narrow channel to readers, guarded by fierce monsters known as editors. The only way to get an audience for anything you wrote was to get it published as a book, or in a newspaper or magazine. Now anyone could publish anything.
This had been possible in principle since 1993, but not many people had realized it yet. I had been intimately involved with building the infrastructure of the web for most of that time, and a writer as well, and it had taken me 8 years to realize it. Even then it took me several years to understand the implications. It meant there would be a whole new generation of essays. [11]
In the print era, the channel for publishing essays had been vanishingly small. Except for a few officially anointed thinkers who went to the right parties in New York, the only people allowed to publish essays were specialists writing about their specialties. There were so many essays that had never been written, because there had been no way to publish them. Now they could be, and I was going to write them. [12]
I've worked on several different things, but to the extent there was a turning point where I figured out what to work on, it was when I started publishing essays online. From then on I knew that whatever else I did, I'd always write essays too.
I knew that online essays would be a marginal medium at first. Socially they'd seem more like rants posted by nutjobs on their GeoCities sites than the genteel and beautifully typeset compositions published in The New Yorker. But by this point I knew enough to find that encouraging instead of discouraging.
One of the most conspicuous patterns I've noticed in my life is how well it has worked, for me at least, to work on things that weren't prestigious. Still life has always been the least prestigious form of painting. Viaweb and Y Combinator both seemed lame when we started them. I still get the glassy eye from strangers when they ask what I'm writing, and I explain that it's an essay I'm going to publish on my web site. Even Lisp, though prestigious intellectually in something like the way Latin is, also seems about as hip.
It's not that unprestigious types of work are good per se. But when you find yourself drawn to some kind of work despite its current lack of prestige, it's a sign both that there's something real to be discovered there, and that you have the right kind of motives. Impure motives are a big danger for the ambitious. If anything is going to lead you astray, it will be the desire to impress people. So while working on things that aren't prestigious doesn't guarantee you're on the right track, it at least guarantees you're not on the most common type of wrong one.
Over the next several years I wrote lots of essays about all kinds of different topics. O'Reilly reprinted a collection of them as a book, called Hackers & Painters after one of the essays in it. I also worked on spam filters, and did some more painting. I used to have dinners for a group of friends every thursday night, which taught me how to cook for groups. And I bought another building in Cambridge, a former candy factory (and later, twas said, porn studio), to use as an office.
One night in October 2003 there was a big party at my house. It was a clever idea of my friend Maria Daniels, who was one of the thursday diners. Three separate hosts would all invite their friends to one party. So for every guest, two thirds of the other guests would be people they didn't know but would probably like. One of the guests was someone I didn't know but would turn out to like a lot: a woman called Jessica Livingston. A couple days later I asked her out.
Jessica was in charge of marketing at a Boston investment bank. This bank thought it understood startups, but over the next year, as she met friends of mine from the startup world, she was surprised how different reality was. And how colorful their stories were. So she decided to compile a book of interviews with startup founders.
When the bank had financial problems and she had to fire half her staff, she started looking for a new job. In early 2005 she interviewed for a marketing job at a Boston VC firm. It took them weeks to make up their minds, and during this time I started telling her about all the things that needed to be fixed about venture capital. They should make a larger number of smaller investments instead of a handful of giant ones, they should be funding younger, more technical founders instead of MBAs, they should let the founders remain as CEO, and so on.
One of my tricks for writing essays had always been to give talks. The prospect of having to stand up in front of a group of people and tell them something that won't waste their time is a great spur to the imagination. When the Harvard Computer Society, the undergrad computer club, asked me to give a talk, I decided I would tell them how to start a startup. Maybe they'd be able to avoid the worst of the mistakes we'd made.
So I gave this talk, in the course of which I told them that the best sources of seed funding were successful startup founders, because then they'd be sources of advice too. Whereupon it seemed they were all looking expectantly at me. Horrified at the prospect of having my inbox flooded by business plans (if I'd only known), I blurted out "But not me!" and went on with the talk. But afterward it occurred to me that I should really stop procrastinating about angel investing. I'd been meaning to since Yahoo bought us, and now it was 7 years later and I still hadn't done one angel investment.
Meanwhile I had been scheming with Robert and Trevor about projects we could work on together. I missed working with them, and it seemed like there had to be something we could collaborate on.
As Jessica and I were walking home from dinner on March 11, at the corner of Garden and Walker streets, these three threads converged. Screw the VCs who were taking so long to make up their minds. We'd start our own investment firm and actually implement the ideas we'd been talking about. I'd fund it, and Jessica could quit her job and work for it, and we'd get Robert and Trevor as partners too. [13]
Once again, ignorance worked in our favor. We had no idea how to be angel investors, and in Boston in 2005 there were no Ron Conways to learn from. So we just made what seemed like the obvious choices, and some of the things we did turned out to be novel.
There are multiple components to Y Combinator, and we didn't figure them all out at once. The part we got first was to be an angel firm. In those days, those two words didn't go together. There were VC firms, which were organized companies with people whose job it was to make investments, but they only did big, million dollar investments. And there were angels, who did smaller investments, but these were individuals who were usually focused on other things and made investments on the side. And neither of them helped founders enough in the beginning. We knew how helpless founders were in some respects, because we remembered how helpless we'd been. For example, one thing Julian had done for us that seemed to us like magic was to get us set up as a company. We were fine writing fairly difficult software, but actually getting incorporated, with bylaws and stock and all that stuff, how on earth did you do that? Our plan was not only to make seed investments, but to do for startups everything Julian had done for us.
YC was not organized as a fund. It was cheap enough to run that we funded it with our own money. That went right by 99% of readers, but professional investors are thinking "Wow, that means they got all the returns." But once again, this was not due to any particular insight on our part. We didn't know how VC firms were organized. It never occurred to us to try to raise a fund, and if it had, we wouldn't have known where to start. [14]
The most distinctive thing about YC is the batch model: to fund a bunch of startups all at once, twice a year, and then to spend three months focusing intensively on trying to help them. That part we discovered by accident, not merely implicitly but explicitly due to our ignorance about investing. We needed to get experience as investors. What better way, we thought, than to fund a whole bunch of startups at once? We knew undergrads got temporary jobs at tech companies during the summer. Why not organize a summer program where they'd start startups instead? We wouldn't feel guilty for being in a sense fake investors, because they would in a similar sense be fake founders. So while we probably wouldn't make much money out of it, we'd at least get to practice being investors on them, and they for their part would probably have a more interesting summer than they would working at Microsoft.
We'd use the building I owned in Cambridge as our headquarters. We'd all have dinner there once a week — on tuesdays, since I was already cooking for the thursday diners on thursdays — and after dinner we'd bring in experts on startups to give talks.
We knew undergrads were deciding then about summer jobs, so in a matter of days we cooked up something we called the Summer Founders Program, and I posted an announcement on my site, inviting undergrads to apply. I had never imagined that writing essays would be a way to get "deal flow," as investors call it, but it turned out to be the perfect source. [15] We got 225 applications for the Summer Founders Program, and we were surprised to find that a lot of them were from people who'd already graduated, or were about to that spring. Already this SFP thing was starting to feel more serious than we'd intended.
We invited about 20 of the 225 groups to interview in person, and from those we picked 8 to fund. They were an impressive group. That first batch included reddit, Justin Kan and Emmett Shear, who went on to found Twitch, Aaron Swartz, who had already helped write the RSS spec and would a few years later become a martyr for open access, and Sam Altman, who would later become the second president of YC. I don't think it was entirely luck that the first batch was so good. You had to be pretty bold to sign up for a weird thing like the Summer Founders Program instead of a summer job at a legit place like Microsoft or Goldman Sachs.
The deal for startups was based on a combination of the deal we did with Julian ($10k for 10%) and what Robert said MIT grad students got for the summer ($6k). We invested $6k per founder, which in the typical two-founder case was $12k, in return for 6%. That had to be fair, because it was twice as good as the deal we ourselves had taken. Plus that first summer, which was really hot, Jessica brought the founders free air conditioners. [16]
Fairly quickly I realized that we had stumbled upon the way to scale startup funding. Funding startups in batches was more convenient for us, because it meant we could do things for a lot of startups at once, but being part of a batch was better for the startups too. It solved one of the biggest problems faced by founders: the isolation. Now you not only had colleagues, but colleagues who understood the problems you were facing and could tell you how they were solving them.
As YC grew, we started to notice other advantages of scale. The alumni became a tight community, dedicated to helping one another, and especially the current batch, whose shoes they remembered being in. We also noticed that the startups were becoming one another's customers. We used to refer jokingly to the "YC GDP," but as YC grows this becomes less and less of a joke. Now lots of startups get their initial set of customers almost entirely from among their batchmates.
I had not originally intended YC to be a full-time job. I was going to do three things: hack, write essays, and work on YC. As YC grew, and I grew more excited about it, it started to take up a lot more than a third of my attention. But for the first few years I was still able to work on other things.
In the summer of 2006, Robert and I started working on a new version of Arc. This one was reasonably fast, because it was compiled into Scheme. To test this new Arc, I wrote Hacker News in it. It was originally meant to be a news aggregator for startup founders and was called Startup News, but after a few months I got tired of reading about nothing but startups. Plus it wasn't startup founders we wanted to reach. It was future startup founders. So I changed the name to Hacker News and the topic to whatever engaged one's intellectual curiosity.
HN was no doubt good for YC, but it was also by far the biggest source of stress for me. If all I'd had to do was select and help founders, life would have been so easy. And that implies that HN was a mistake. Surely the biggest source of stress in one's work should at least be something close to the core of the work. Whereas I was like someone who was in pain while running a marathon not from the exertion of running, but because I had a blister from an ill-fitting shoe. When I was dealing with some urgent problem during YC, there was about a 60% chance it had to do with HN, and a 40% chance it had do with everything else combined. [17]
As well as HN, I wrote all of YC's internal software in Arc. But while I continued to work a good deal in Arc, I gradually stopped working on Arc, partly because I didn't have time to, and partly because it was a lot less attractive to mess around with the language now that we had all this infrastructure depending on it. So now my three projects were reduced to two: writing essays and working on YC.
YC was different from other kinds of work I've done. Instead of deciding for myself what to work on, the problems came to me. Every 6 months there was a new batch of startups, and their problems, whatever they were, became our problems. It was very engaging work, because their problems were quite varied, and the good founders were very effective. If you were trying to learn the most you could about startups in the shortest possible time, you couldn't have picked a better way to do it.
There were parts of the job I didn't like. Disputes between cofounders, figuring out when people were lying to us, fighting with people who maltreated the startups, and so on. But I worked hard even at the parts I didn't like. I was haunted by something Kevin Hale once said about companies: "No one works harder than the boss." He meant it both descriptively and prescriptively, and it was the second part that scared me. I wanted YC to be good, so if how hard I worked set the upper bound on how hard everyone else worked, I'd better work very hard.
One day in 2010, when he was visiting California for interviews, Robert Morris did something astonishing: he offered me unsolicited advice. I can only remember him doing that once before. One day at Viaweb, when I was bent over double from a kidney stone, he suggested that it would be a good idea for him to take me to the hospital. That was what it took for Rtm to offer unsolicited advice. So I remember his exact words very clearly. "You know," he said, "you should make sure Y Combinator isn't the last cool thing you do."
At the time I didn't understand what he meant, but gradually it dawned on me that he was saying I should quit. This seemed strange advice, because YC was doing great. But if there was one thing rarer than Rtm offering advice, it was Rtm being wrong. So this set me thinking. It was true that on my current trajectory, YC would be the last thing I did, because it was only taking up more of my attention. It had already eaten Arc, and was in the process of eating essays too. Either YC was my life's work or I'd have to leave eventually. And it wasn't, so I would.
In the summer of 2012 my mother had a stroke, and the cause turned out to be a blood clot caused by colon cancer. The stroke destroyed her balance, and she was put in a nursing home, but she really wanted to get out of it and back to her house, and my sister and I were determined to help her do it. I used to fly up to Oregon to visit her regularly, and I had a lot of time to think on those flights. On one of them I realized I was ready to hand YC over to someone else.
I asked Jessica if she wanted to be president, but she didn't, so we decided we'd try to recruit Sam Altman. We talked to Robert and Trevor and we agreed to make it a complete changing of the guard. Up till that point YC had been controlled by the original LLC we four had started. But we wanted YC to last for a long time, and to do that it couldn't be controlled by the founders. So if Sam said yes, we'd let him reorganize YC. Robert and I would retire, and Jessica and Trevor would become ordinary partners.
When we asked Sam if he wanted to be president of YC, initially he said no. He wanted to start a startup to make nuclear reactors. But I kept at it, and in October 2013 he finally agreed. We decided he'd take over starting with the winter 2014 batch. For the rest of 2013 I left running YC more and more to Sam, partly so he could learn the job, and partly because I was focused on my mother, whose cancer had returned.
She died on January 15, 2014. We knew this was coming, but it was still hard when it did.
I kept working on YC till March, to help get that batch of startups through Demo Day, then I checked out pretty completely. (I still talk to alumni and to new startups working on things I'm interested in, but that only takes a few hours a week.)
What should I do next? Rtm's advice hadn't included anything about that. I wanted to do something completely different, so I decided I'd paint. I wanted to see how good I could get if I really focused on it. So the day after I stopped working on YC, I started painting. I was rusty and it took a while to get back into shape, but it was at least completely engaging. [18]
I spent most of the rest of 2014 painting. I'd never been able to work so uninterruptedly before, and I got to be better than I had been. Not good enough, but better. Then in November, right in the middle of a painting, I ran out of steam. Up till that point I'd always been curious to see how the painting I was working on would turn out, but suddenly finishing this one seemed like a chore. So I stopped working on it and cleaned my brushes and haven't painted since. So far anyway.
I realize that sounds rather wimpy. But attention is a zero sum game. If you can choose what to work on, and you choose a project that's not the best one (or at least a good one) for you, then it's getting in the way of another project that is. And at 50 there was some opportunity cost to screwing around.
I started writing essays again, and wrote a bunch of new ones over the next few months. I even wrote a couple that weren't about startups. Then in March 2015 I started working on Lisp again.
The distinctive thing about Lisp is that its core is a language defined by writing an interpreter in itself. It wasn't originally intended as a programming language in the ordinary sense. It was meant to be a formal model of computation, an alternative to the Turing machine. If you want to write an interpreter for a language in itself, what's the minimum set of predefined operators you need? The Lisp that John McCarthy invented, or more accurately discovered, is an answer to that question. [19]
McCarthy didn't realize this Lisp could even be used to program computers till his grad student Steve Russell suggested it. Russell translated McCarthy's interpreter into IBM 704 machine language, and from that point Lisp started also to be a programming language in the ordinary sense. But its origins as a model of computation gave it a power and elegance that other languages couldn't match. It was this that attracted me in college, though I didn't understand why at the time.
McCarthy's 1960 Lisp did nothing more than interpret Lisp expressions. It was missing a lot of things you'd want in a programming language. So these had to be added, and when they were, they weren't defined using McCarthy's original axiomatic approach. That wouldn't have been feasible at the time. McCarthy tested his interpreter by hand-simulating the execution of programs. But it was already getting close to the limit of interpreters you could test that way — indeed, there was a bug in it that McCarthy had overlooked. To test a more complicated interpreter, you'd have had to run it, and computers then weren't powerful enough.
Now they are, though. Now you could continue using McCarthy's axiomatic approach till you'd defined a complete programming language. And as long as every change you made to McCarthy's Lisp was a discoveredness-preserving transformation, you could, in principle, end up with a complete language that had this quality. Harder to do than to talk about, of course, but if it was possible in principle, why not try? So I decided to take a shot at it. It took 4 years, from March 26, 2015 to October 12, 2019. It was fortunate that I had a precisely defined goal, or it would have been hard to keep at it for so long.
I wrote this new Lisp, called Bel, in itself in Arc. That may sound like a contradiction, but it's an indication of the sort of trickery I had to engage in to make this work. By means of an egregious collection of hacks I managed to make something close enough to an interpreter written in itself that could actually run. Not fast, but fast enough to test.
I had to ban myself from writing essays during most of this time, or I'd never have finished. In late 2015 I spent 3 months writing essays, and when I went back to working on Bel I could barely understand the code. Not so much because it was badly written as because the problem is so convoluted. When you're working on an interpreter written in itself, it's hard to keep track of what's happening at what level, and errors can be practically encrypted by the time you get them.
So I said no more essays till Bel was done. But I told few people about Bel while I was working on it. So for years it must have seemed that I was doing nothing, when in fact I was working harder than I'd ever worked on anything. Occasionally after wrestling for hours with some gruesome bug I'd check Twitter or HN and see someone asking "Does Paul Graham still code?"
Working on Bel was hard but satisfying. I worked on it so intensively that at any given time I had a decent chunk of the code in my head and could write more there. I remember taking the boys to the coast on a sunny day in 2015 and figuring out how to deal with some problem involving continuations while I watched them play in the tide pools. It felt like I was doing life right. I remember that because I was slightly dismayed at how novel it felt. The good news is that I had more moments like this over the next few years.
In the summer of 2016 we moved to England. We wanted our kids to see what it was like living in another country, and since I was a British citizen by birth, that seemed the obvious choice. We only meant to stay for a year, but we liked it so much that we still live there. So most of Bel was written in England.
In the fall of 2019, Bel was finally finished. Like McCarthy's original Lisp, it's a spec rather than an implementation, although like McCarthy's Lisp it's a spec expressed as code.
Now that I could write essays again, I wrote a bunch about topics I'd had stacked up. I kept writing essays through 2020, but I also started to think about other things I could work on. How should I choose what to do? Well, how had I chosen what to work on in the past? I wrote an essay for myself to answer that question, and I was surprised how long and messy the answer turned out to be. If this surprised me, who'd lived it, then I thought perhaps it would be interesting to other people, and encouraging to those with similarly messy lives. So I wrote a more detailed version for others to read, and this is the last sentence of it.
Notes
[1] My experience skipped a step in the evolution of computers: time-sharing machines with interactive OSes. I went straight from batch processing to microcomputers, which made microcomputers seem all the more exciting.
[2] Italian words for abstract concepts can nearly always be predicted from their English cognates (except for occasional traps like polluzione). It's the everyday words that differ. So if you string together a lot of abstract concepts with a few simple verbs, you can make a little Italian go a long way.
[3] I lived at Piazza San Felice 4, so my walk to the Accademia went straight down the spine of old Florence: past the Pitti, across the bridge, past Orsanmichele, between the Duomo and the Baptistery, and then up Via Ricasoli to Piazza San Marco. I saw Florence at street level in every possible condition, from empty dark winter evenings to sweltering summer days when the streets were packed with tourists.
[4] You can of course paint people like still lives if you want to, and they're willing. That sort of portrait is arguably the apex of still life painting, though the long sitting does tend to produce pained expressions in the sitters.
[5] Interleaf was one of many companies that had smart people and built impressive technology, and yet got crushed by Moore's Law. In the 1990s the exponential growth in the power of commodity (i.e. Intel) processors rolled up high-end, special-purpose hardware and software companies like a bulldozer.
[6] The signature style seekers at RISD weren't specifically mercenary. In the art world, money and coolness are tightly coupled. Anything expensive comes to be seen as cool, and anything seen as cool will soon become equally expensive.
[7] Technically the apartment wasn't rent-controlled but rent-stabilized, but this is a refinement only New Yorkers would know or care about. The point is that it was really cheap, less than half market price.
[8] Most software you can launch as soon as it's done. But when the software is an online store builder and you're hosting the stores, if you don't have any users yet, that fact will be painfully obvious. So before we could launch publicly we had to launch privately, in the sense of recruiting an initial set of users and making sure they had decent-looking stores.
[9] We'd had a code editor in Viaweb for users to define their own page styles. They didn't know it, but they were editing Lisp expressions underneath. But this wasn't an app editor, because the code ran when the merchants' sites were generated, not when shoppers visited them.
[10] This was the first instance of what is now a familiar experience, and so was what happened next, when I read the comments and found they were full of angry people. How could I claim that Lisp was better than other languages? Weren't they all Turing complete? People who see the responses to essays I write sometimes tell me how sorry they feel for me, but I'm not exaggerating when I reply that it has always been like this, since the very beginning. It comes with the territory. An essay must tell readers things they don't already know, and some people dislike being told such things.
[11] People put plenty of stuff on the internet in the 90s of course, but putting something online is not the same as publishing it online. Publishing online means you treat the online version as the (or at least a) primary version.
[12] There is a general lesson here that our experience with Y Combinator also teaches: Customs continue to constrain you long after the restrictions that caused them have disappeared. Customary VC practice had once, like the customs about publishing essays, been based on real constraints. Startups had once been much more expensive to start, and proportionally rare. Now they could be cheap and common, but the VCs' customs still reflected the old world, just as customs about writing essays still reflected the constraints of the print era.
Which in turn implies that people who are independent-minded (i.e. less influenced by custom) will have an advantage in fields affected by rapid change (where customs are more likely to be obsolete).
Here's an interesting point, though: you can't always predict which fields will be affected by rapid change. Obviously software and venture capital will be, but who would have predicted that essay writing would be?
[13] Y Combinator was not the original name. At first we were called Cambridge Seed. But we didn't want a regional name, in case someone copied us in Silicon Valley, so we renamed ourselves after one of the coolest tricks in the lambda calculus, the Y combinator.
I picked orange as our color partly because it's the warmest, and partly because no VC used it. In 2005 all the VCs used staid colors like maroon, navy blue, and forest green, because they were trying to appeal to LPs, not founders. The YC logo itself is an inside joke: the Viaweb logo had been a white V on a red circle, so I made the YC logo a white Y on an orange square.
[14] YC did become a fund for a couple years starting in 2009, because it was getting so big I could no longer afford to fund it personally. But after Heroku got bought we had enough money to go back to being self-funded.
[15] I've never liked the term "deal flow," because it implies that the number of new startups at any given time is fixed. This is not only false, but it's the purpose of YC to falsify it, by causing startups to be founded that would not otherwise have existed.
[16] She reports that they were all different shapes and sizes, because there was a run on air conditioners and she had to get whatever she could, but that they were all heavier than she could carry now.
[17] Another problem with HN was a bizarre edge case that occurs when you both write essays and run a forum. When you run a forum, you're assumed to see if not every conversation, at least every conversation involving you. And when you write essays, people post highly imaginative misinterpretations of them on forums. Individually these two phenomena are tedious but bearable, but the combination is disastrous. You actually have to respond to the misinterpretations, because the assumption that you're present in the conversation means that not responding to any sufficiently upvoted misinterpretation reads as a tacit admission that it's correct. But that in turn encourages more; anyone who wants to pick a fight with you senses that now is their chance.
[18] The worst thing about leaving YC was not working with Jessica anymore. We'd been working on YC almost the whole time we'd known each other, and we'd neither tried nor wanted to separate it from our personal lives, so leaving was like pulling up a deeply rooted tree.
[19] One way to get more precise about the concept of invented vs discovered is to talk about space aliens. Any sufficiently advanced alien civilization would certainly know about the Pythagorean theorem, for example. I believe, though with less certainty, that they would also know about the Lisp in McCarthy's 1960 paper.
But if so there's no reason to suppose that this is the limit of the language that might be known to them. Presumably aliens need numbers and errors and I/O too. So it seems likely there exists at least one path out of McCarthy's Lisp along which discoveredness is preserved.
Thanks to Trevor Blackwell, John Collison, Patrick Collison, Daniel Gackle, Ralph Hazell, Jessica Livingston, Robert Morris, and Harj Taggar for reading drafts of this.

68
tests/old_proxy_tests/tests/load_test_completion.py
View file

@ -1,68 +0,0 @@
import time
import asyncio
import os
from openai import AsyncOpenAI, AsyncAzureOpenAI
from litellm._uuid import uuid
import traceback
from large_text import text
from dotenv import load_dotenv
from statistics import mean, median
litellm_client = AsyncOpenAI(base_url="http://0.0.0.0:4000/", api_key="sk-1234")
async def litellm_completion():
try:
start_time = time.time()
response = await litellm_client.chat.completions.create(
model="fake-openai-endpoint",
messages=[
{
"role": "user",
"content": f"This is a test{uuid.uuid4()}",
}
],
user="my-new-end-user-1",
)
end_time = time.time()
latency = end_time - start_time
print("response time=", latency)
return response, latency
except Exception as e:
with open("error_log.txt", "a") as error_log:
error_log.write(f"Error during completion: {str(e)}\n")
return None, 0
async def main():
latencies = []
for i in range(5):
start = time.time()
n = 100 # Number of concurrent tasks
tasks = [litellm_completion() for _ in range(n)]
chat_completions = await asyncio.gather(*tasks)
successful_completions = [c for c, l in chat_completions if c is not None]
completion_latencies = [l for c, l in chat_completions if c is not None]
latencies.extend(completion_latencies)
with open("error_log.txt", "a") as error_log:
for completion, latency in chat_completions:
if isinstance(completion, str):
error_log.write(completion + "\n")
print(n, time.time() - start, len(successful_completions))
if latencies:
average_latency = mean(latencies)
median_latency = median(latencies)
print(f"Average Latency per Response: {average_latency} seconds")
print(f"Median Latency per Response: {median_latency} seconds")
if __name__ == "__main__":
open("error_log.txt", "w").close()
asyncio.run(main())

107
tests/old_proxy_tests/tests/load_test_embedding.py
View file

@ -1,107 +0,0 @@
# test time it takes to make 100 concurrent embedding requests to OpenaI
import os
import sys
import traceback
from dotenv import load_dotenv
load_dotenv()
import io
import os
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import pytest
import litellm
litellm.set_verbose = False
question = "embed this very long text" * 100
# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array.
# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere
# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions
import concurrent.futures
import random
import time
# Function to make concurrent calls to OpenAI API
def make_openai_completion(question):
try:
time.time()
import openai
client = openai.OpenAI(
api_key=os.environ["OPENAI_API_KEY"]
) # base_url="http://0.0.0.0:8000",
response = client.embeddings.create(
model="text-embedding-ada-002",
input=[question],
)
print(response)
time.time()
# Log the request details
# with open("request_log.txt", "a") as log_file:
# log_file.write(
# f"Question: {question[:100]}\nResponse ID:{response.id} Content:{response.choices[0].message.content[:10]}\nTime: {end_time - start_time:.2f} seconds\n\n"
# )
return response
except Exception:
# Log exceptions for failed calls
# with open("error_log.txt", "a") as error_log_file:
# error_log_file.write(
# f"\nException: {str(e)}\n\n"
# )
return None
start_time = time.time()
# Number of concurrent calls (you can adjust this)
concurrent_calls = 500
# List to store the futures of concurrent calls
futures = []
# Make concurrent calls
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor:
for _ in range(concurrent_calls):
futures.append(executor.submit(make_openai_completion, question))
# Wait for all futures to complete
concurrent.futures.wait(futures)
# Summarize the results
successful_calls = 0
failed_calls = 0
for future in futures:
if future.result() is not None:
successful_calls += 1
else:
failed_calls += 1
end_time = time.time()
# Calculate the duration
duration = end_time - start_time
print("Load test Summary:")
print(f"Total Requests: {concurrent_calls}")
print(f"Successful Calls: {successful_calls}")
print(f"Failed Calls: {failed_calls}")
print(f"Total Time: {duration:.2f} seconds")
# Display content of the logs
with open("request_log.txt", "r") as log_file:
print("\nRequest Log:\n", log_file.read())
with open("error_log.txt", "r") as error_log_file:
print("\nError Log:\n", error_log_file.read())

54
tests/old_proxy_tests/tests/load_test_embedding_100.py
View file

@ -1,54 +0,0 @@
import time, asyncio
from openai import AsyncOpenAI
from litellm._uuid import uuid
import traceback
litellm_client = AsyncOpenAI(api_key="test", base_url="http://0.0.0.0:8000")
async def litellm_completion():
# Your existing code for litellm_completion goes here
try:
print("starting embedding calls")
response = await litellm_client.embeddings.create(
model="text-embedding-ada-002",
input=[
"hello who are you" * 2000,
"hello who are you tomorrow 1234" * 1000,
"hello who are you tomorrow 1234" * 1000,
],
)
print(response)
return response
except Exception as e:
# If there's an exception, log the error message
with open("error_log.txt", "a") as error_log:
error_log.write(f"Error during completion: {str(e)}\n")
pass
async def main():
start = time.time()
n = 100 # Number of concurrent tasks
tasks = [litellm_completion() for _ in range(n)]
chat_completions = await asyncio.gather(*tasks)
successful_completions = [c for c in chat_completions if c is not None]
# Write errors to error_log.txt
with open("error_log.txt", "a") as error_log:
for completion in chat_completions:
if isinstance(completion, str):
error_log.write(completion + "\n")
print(n, time.time() - start, len(successful_completions))
if __name__ == "__main__":
# Blank out contents of error_log.txt
open("error_log.txt", "w").close()
asyncio.run(main())

107
tests/old_proxy_tests/tests/load_test_embedding_proxy.py
View file

@ -1,107 +0,0 @@
# test time it takes to make 100 concurrent embedding requests to OpenaI
import os
import sys
import traceback
from dotenv import load_dotenv
load_dotenv()
import io
import os
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import pytest
import litellm
litellm.set_verbose = False
question = "embed this very long text" * 100
# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array.
# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere
# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions
import concurrent.futures
import random
import time
# Function to make concurrent calls to OpenAI API
def make_openai_completion(question):
try:
time.time()
import openai
client = openai.OpenAI(
api_key=os.environ["OPENAI_API_KEY"], base_url="http://0.0.0.0:8000"
) # base_url="http://0.0.0.0:8000",
response = client.embeddings.create(
model="text-embedding-ada-002",
input=[question],
)
print(response)
time.time()
# Log the request details
# with open("request_log.txt", "a") as log_file:
# log_file.write(
# f"Question: {question[:100]}\nResponse ID:{response.id} Content:{response.choices[0].message.content[:10]}\nTime: {end_time - start_time:.2f} seconds\n\n"
# )
return response
except Exception:
# Log exceptions for failed calls
# with open("error_log.txt", "a") as error_log_file:
# error_log_file.write(
# f"\nException: {str(e)}\n\n"
# )
return None
start_time = time.time()
# Number of concurrent calls (you can adjust this)
concurrent_calls = 500
# List to store the futures of concurrent calls
futures = []
# Make concurrent calls
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor:
for _ in range(concurrent_calls):
futures.append(executor.submit(make_openai_completion, question))
# Wait for all futures to complete
concurrent.futures.wait(futures)
# Summarize the results
successful_calls = 0
failed_calls = 0
for future in futures:
if future.result() is not None:
successful_calls += 1
else:
failed_calls += 1
end_time = time.time()
# Calculate the duration
duration = end_time - start_time
print("Load test Summary:")
print(f"Total Requests: {concurrent_calls}")
print(f"Successful Calls: {successful_calls}")
print(f"Failed Calls: {failed_calls}")
print(f"Total Time: {duration:.2f} seconds")
# # Display content of the logs
# with open("request_log.txt", "r") as log_file:
# print("\nRequest Log:\n", log_file.read())
# with open("error_log.txt", "r") as error_log_file:
# print("\nError Log:\n", error_log_file.read())

121
tests/old_proxy_tests/tests/load_test_q.py
View file

@ -1,121 +0,0 @@
import os
import time
import requests
from dotenv import load_dotenv
load_dotenv()
# Set the base URL as needed
base_url = "https://api.litellm.ai"
# # Uncomment the line below if you want to switch to the local server
# base_url = "http://0.0.0.0:8000"
# Step 1 Add a config to the proxy, generate a temp key
config = {
"model_list": [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "gpt-3.5-turbo",
"api_key": os.environ["OPENAI_API_KEY"],
},
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/gpt-4.1-mini",
"api_key": os.environ["AZURE_AI_API_KEY"],
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
"api_version": "2023-07-01-preview",
},
},
]
}
print("STARTING LOAD TEST Q")
print(os.environ["AZURE_AI_API_KEY"])
response = requests.post(
url=f"{base_url}/key/generate",
json={
"config": config,
"duration": "30d", # default to 30d, set it to 30m if you want a temp key
},
headers={"Authorization": "Bearer sk-hosted-litellm"},
)
print("\nresponse from generating key", response.text)
print("\n json response from gen key", response.json())
generated_key = response.json()["key"]
print("\ngenerated key for proxy", generated_key)
# Step 2: Queue 50 requests to the proxy, using your generated_key
import concurrent.futures
def create_job_and_poll(request_num):
print(f"Creating a job on the proxy for request {request_num}")
job_response = requests.post(
url=f"{base_url}/queue/request",
json={
"model": "gpt-3.5-turbo",
"messages": [
{"role": "system", "content": "write a short poem"},
],
},
headers={"Authorization": f"Bearer {generated_key}"},
)
print(job_response.status_code)
print(job_response.text)
print("\nResponse from creating job", job_response.text)
job_response = job_response.json()
job_response["id"]
polling_url = job_response["url"]
polling_url = f"{base_url}{polling_url}"
print(f"\nCreated Job {request_num}, Polling Url {polling_url}")
# Poll each request
while True:
try:
print(f"\nPolling URL for request {request_num}", polling_url)
polling_response = requests.get(
url=polling_url, headers={"Authorization": f"Bearer {generated_key}"}
)
print(
f"\nResponse from polling url for request {request_num}",
polling_response.text,
)
polling_response = polling_response.json()
status = polling_response.get("status", None)
if status == "finished":
llm_response = polling_response["result"]
print(f"LLM Response for request {request_num}")
print(llm_response)
# Write the llm_response to load_test_log.txt
try:
with open("load_test_log.txt", "a") as response_file:
response_file.write(
f"Response for request: {request_num}\n{llm_response}\n\n"
)
except Exception as e:
print("GOT EXCEPTION", e)
break
time.sleep(0.5)
except Exception as e:
print("got exception when polling", e)
# Number of requests
num_requests = 100
# Use ThreadPoolExecutor for parallel execution
with concurrent.futures.ThreadPoolExecutor(max_workers=num_requests) as executor:
# Create and poll each request in parallel
futures = [executor.submit(create_job_and_poll, i) for i in range(num_requests)]
# Wait for all futures to complete
concurrent.futures.wait(futures)

0
tests/old_proxy_tests/tests/request_log.txt
View file

36
tests/old_proxy_tests/tests/test_anthropic_context_caching.py
View file

@ -1,36 +0,0 @@
import openai
client = openai.OpenAI(
api_key="sk-1234", # litellm proxy api key
base_url="http://0.0.0.0:4000", # litellm proxy base url
)
response = client.chat.completions.create(
model="anthropic/claude-sonnet-4-5-20250929",
messages=[
{ # type: ignore
"role": "system",
"content": [
{
"type": "text",
"text": "You are an AI assistant tasked with analyzing legal documents.",
},
{
"type": "text",
"text": "Here is the full text of a complex legal agreement" * 100,
"cache_control": {"type": "ephemeral"},
},
],
},
{
"role": "user",
"content": "what are the key terms and conditions in this agreement?",
},
],
extra_headers={
"anthropic-version": "2023-06-01",
},
)
print(response)

22
tests/old_proxy_tests/tests/test_anthropic_sdk.py
View file

@ -1,22 +0,0 @@
import os
from anthropic import Anthropic
client = Anthropic(
# This is the default and can be omitted
base_url="http://localhost:4000",
# this is a litellm proxy key :) - not a real anthropic key
api_key="sk-test-proxy-key-123",
)
message = client.messages.create(
max_tokens=1024,
messages=[
{
"role": "user",
"content": "Hello, Claude",
}
],
model="claude-3-opus-20240229",
)
print(message.content)

28
tests/old_proxy_tests/tests/test_async.py
View file

@ -1,28 +0,0 @@
# # This tests the litelm proxy
# # it makes async Completion requests with streaming
# import openai
# openai.base_url = "http://0.0.0.0:8000"
# openai.api_key = "temp-key"
# print(openai.base_url)
# async def test_async_completion():
# response = await (
# model="gpt-3.5-turbo",
# prompt='this is a test request, write a short poem',
# )
# print(response)
# print("test_streaming")
# response = await openai.chat.completions.create(
# model="gpt-3.5-turbo",
# prompt='this is a test request, write a short poem',
# stream=True
# )
# print(response)
# async for chunk in response:
# print(chunk)
# import asyncio
# asyncio.run(test_async_completion())

54
tests/old_proxy_tests/tests/test_gemini_context_caching.py
View file

@ -1,54 +0,0 @@
import datetime
import httpx
import openai
# Set Litellm proxy variables here
LITELLM_BASE_URL = "http://0.0.0.0:4000"
LITELLM_PROXY_API_KEY = "sk-1234"
client = openai.OpenAI(api_key=LITELLM_PROXY_API_KEY, base_url=LITELLM_BASE_URL)
httpx_client = httpx.Client(timeout=30)
################################
# First create a cachedContents object
print("creating cached content")
create_cache = httpx_client.post(
url=f"{LITELLM_BASE_URL}/vertex-ai/cachedContents",
headers={"Authorization": f"Bearer {LITELLM_PROXY_API_KEY}"},
json={
"model": "gemini-1.5-pro-001",
"contents": [
{
"role": "user",
"parts": [
{
"text": "This is sample text to demonstrate explicit caching."
* 4000
}
],
}
],
},
)
print("response from create_cache", create_cache)
create_cache_response = create_cache.json()
print("json from create_cache", create_cache_response)
cached_content_name = create_cache_response["name"]
#################################
# Use the `cachedContents` object in your /chat/completions
response = client.chat.completions.create( # type: ignore
model="gemini-1.5-pro-001",
max_tokens=8192,
messages=[
{
"role": "user",
"content": "what is the sample text about?",
},
],
temperature="0.7",
extra_body={"cached_content": cached_content_name}, # 👈 key change
)
print("response from proxy", response)

17
tests/old_proxy_tests/tests/test_langchain_embedding.py
View file

@ -1,17 +0,0 @@
from langchain_openai import OpenAIEmbeddings
embeddings_models = "multimodalembedding@001"
embeddings = OpenAIEmbeddings(
model="multimodalembedding@001",
base_url="http://0.0.0.0:4000",
api_key="sk-1234", # type: ignore
)
query_result = embeddings.embed_query(
"gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
)
# print(len(query_result))
# print(query_result[:5])
print(query_result)

44
tests/old_proxy_tests/tests/test_langchain_request.py
View file

@ -1,44 +0,0 @@
# # LOCAL TEST
# from langchain.chat_models import ChatOpenAI
# from langchain.prompts.chat import (
# ChatPromptTemplate,
# HumanMessagePromptTemplate,
# SystemMessagePromptTemplate,
# )
# from langchain.schema import HumanMessage, SystemMessage
# chat = ChatOpenAI(
# openai_api_base="http://0.0.0.0:8000",
# model = "azure/gpt-4.1-mini",
# temperature=0.1,
# extra_body={
# "metadata": {
# "generation_name": "ishaan-generation-langchain-client",
# "generation_id": "langchain-client-gen-id22",
# "trace_id": "langchain-client-trace-id22",
# "trace_user_id": "langchain-client-user-id2"
# }
# }
# )
# messages = [
# SystemMessage(
# content="You are a helpful assistant that im using to make a test request to."
# ),
# HumanMessage(
# content="test from litellm. tell me why it's amazing in 1 sentence"
# ),
# ]
# response = chat(messages)
# print(response)
# # claude_chat = ChatOpenAI(
# # openai_api_base="http://0.0.0.0:8000",
# # model = "claude-v1",
# # temperature=0.1
# # )
# # response = claude_chat(messages)
# # print(response)

36
tests/old_proxy_tests/tests/test_llamaindex.py
View file

@ -1,36 +0,0 @@
import os, dotenv
from dotenv import load_dotenv
load_dotenv()
from llama_index.llms import AzureOpenAI
from llama_index.embeddings import AzureOpenAIEmbedding
from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext
llm = AzureOpenAI(
engine="azure-gpt-3.5",
temperature=0.0,
azure_endpoint="http://0.0.0.0:4000",
api_key="sk-1234",
api_version="2023-07-01-preview",
)
embed_model = AzureOpenAIEmbedding(
deployment_name="azure-embedding-model",
azure_endpoint="http://0.0.0.0:4000",
api_key="sk-1234",
api_version="2023-07-01-preview",
)
# response = llm.complete("The sky is a beautiful blue and")
# print(response)
documents = SimpleDirectoryReader("llama_index_data").load_data()
service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)
index = VectorStoreIndex.from_documents(documents, service_context=service_context)
query_engine = index.as_query_engine()
response = query_engine.query("What did the author do growing up?")
print(response)

13
tests/old_proxy_tests/tests/test_mistral_sdk.py
View file

@ -1,13 +0,0 @@
import os
from mistralai.client import MistralClient
from mistralai.models.chat_completion import ChatMessage
client = MistralClient(api_key="sk-1234", endpoint="http://0.0.0.0:4000")
chat_response = client.chat(
model="mistral-small-latest",
messages=[
{"role": "user", "content": "this is a test request, write a short poem"}
],
)
print(chat_response.choices[0].message.content)

126
tests/old_proxy_tests/tests/test_openai_embedding.py
View file

@ -1,126 +0,0 @@
import openai
import asyncio
async def async_request(client, model, input_data):
response = await client.embeddings.create(model=model, input=input_data)
response = response.dict()
data_list = response["data"]
for i, embedding in enumerate(data_list):
embedding["embedding"] = []
current_index = embedding["index"]
assert i == current_index
return response
async def main():
client = openai.AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
models = [
"text-embedding-ada-002",
"text-embedding-ada-002",
"text-embedding-ada-002",
]
inputs = [
[
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
],
["1", "2", "3", "4", "5", "6"],
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
],
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
],
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
],
["1", "2", "3"],
]
tasks = []
for model, input_data in zip(models, inputs):
task = async_request(client, model, input_data)
tasks.append(task)
responses = await asyncio.gather(*tasks)
print(responses)
for response in responses:
data_list = response["data"]
for embedding in data_list:
embedding["embedding"] = []
print(response)
asyncio.run(main())

53
tests/old_proxy_tests/tests/test_openai_exception_request.py
View file

@ -1,53 +0,0 @@
import openai
import httpx
import os
from dotenv import load_dotenv
load_dotenv()
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:8000",
http_client=httpx.Client(verify=False),
)
try:
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="azure-gpt-3.5",
messages=[
{
"role": "user",
"content": "this is a test request, write a short poem" * 2000,
}
],
)
print(response)
except Exception as e:
print(e)
variables_proxy_exception = vars(e)
print("proxy exception variables", variables_proxy_exception.keys())
print(variables_proxy_exception["body"])
api_key = os.getenv("AZURE_API_KEY")
azure_endpoint = os.getenv("AZURE_API_BASE")
print(api_key, azure_endpoint)
client = openai.AzureOpenAI(
api_key=os.getenv("AZURE_API_KEY"),
azure_endpoint=os.getenv("AZURE_API_BASE", "default"),
)
try:
response = client.chat.completions.create(
model="chatgpt-v-3",
messages=[
{
"role": "user",
"content": "this is a test request, write a short poem" * 2000,
}
],
)
except Exception as e:
print(e)
variables_exception = vars(e)
print("openai client exception variables", variables_exception.keys())

41
tests/old_proxy_tests/tests/test_openai_js.js
View file

@ -1,41 +0,0 @@
const openai = require('openai');
// set DEBUG=true in env
process.env.DEBUG=false;
async function runOpenAI() {
const client = new openai.OpenAI({
apiKey: 'sk-1234',
baseURL: 'http://0.0.0.0:4000'
});
try {
const response = await client.chat.completions.create({
model: 'anthropic-claude-v2.1',
stream: true,
messages: [
{
role: 'user',
content: 'write a 20 pg essay about YC '.repeat(6000),
},
],
});
console.log(response);
let original = '';
for await (const chunk of response) {
original += chunk.choices[0].delta.content;
console.log(original);
console.log(chunk);
console.log(chunk.choices[0].delta.content);
}
} catch (error) {
console.log("got this exception from server");
console.error(error);
console.log("done with exception from proxy");
}
}
// Call the asynchronous function
runOpenAI();

60
tests/old_proxy_tests/tests/test_openai_request.py
View file

@ -1,60 +0,0 @@
import openai
client = openai.OpenAI(api_key="hi", base_url="http://0.0.0.0:8000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="azure/gpt-4.1-mini",
messages=[
{"role": "user", "content": "this is a test request, write a short poem"}
],
extra_body={
"metadata": {
"generation_name": "ishaan-generation-openai-client",
"generation_id": "openai-client-gen-id22",
"trace_id": "openai-client-trace-id22",
"trace_user_id": "openai-client-user-id2",
}
},
)
print(response)
# request sent to gpt-4-vision + enhancements
completion_extensions = client.chat.completions.create(
model="gpt-vision",
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What's in this image? Output your answer in JSON.",
},
{
"type": "image_url",
"image_url": {
"url": "https://avatars.githubusercontent.com/u/29436595?v=4",
"detail": "low",
},
},
],
}
],
max_tokens=4096,
temperature=0.0,
extra_body={
"enhancements": {"ocr": {"enabled": True}, "grounding": {"enabled": True}},
"dataSources": [
{
"type": "AzureComputerVision",
"parameters": {
"endpoint": "https://gpt-4-vision-enhancement.cognitiveservices.azure.com/",
"key": "f015cf8eeb1d4bd1b1467d21dec6063b",
},
}
],
},
)

41
tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
View file

@ -1,41 +0,0 @@
# mypy: ignore-errors
import openai
from opentelemetry import trace
from opentelemetry.context import Context
from opentelemetry.trace import SpanKind
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
trace.set_tracer_provider(TracerProvider())
memory_exporter = InMemorySpanExporter()
span_processor = SimpleSpanProcessor(memory_exporter)
trace.get_tracer_provider().add_span_processor(span_processor)
tracer = trace.get_tracer(__name__)
# create an otel traceparent header
tracer = trace.get_tracer(__name__)
with tracer.start_as_current_span("ishaan-local-dev-app") as span:
span.set_attribute("generation_name", "ishaan-generation-openai-client")
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
extra_headers = {}
context = trace.set_span_in_context(span)
traceparent = TraceContextTextMapPropagator()
traceparent.inject(carrier=extra_headers, context=context)
print("EXTRA HEADERS: ", extra_headers)
_trace_parent = extra_headers.get("traceparent")
trace_id = _trace_parent.split("-")[1]
print("Trace ID: ", trace_id)
# # request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="llama3",
messages=[
{"role": "user", "content": "this is a test request, write a short poem"}
],
extra_headers=extra_headers,
)
print(response)

10
tests/old_proxy_tests/tests/test_openai_simple_embedding.py
View file

@ -1,10 +0,0 @@
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.embeddings.create(
model="text-embedding-ada-002", input=["test"], encoding_format="base64"
)
print(response)

11
tests/old_proxy_tests/tests/test_openai_tts_request.py
View file

@ -1,11 +0,0 @@
import openai
client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# # request sent to model set on litellm proxy, `litellm --model`
response = client.audio.speech.create(
model="vertex-tts",
input="the quick brown fox jumped over the lazy dogs",
voice={"languageCode": "en-US", "name": "en-US-Studio-O"}, # type: ignore
)
print("response from proxy", response) # noqa

Some files were not shown because too many files have changed in this diff Show more