scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-20 00:11:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 75

Merge remote-tracking branch 'origin/main' into litellm_agent365_mcp_guardrail

Browse source
This commit is contained in:
yucheng 2026-09-16 00:52:27 +00:00
commit 9bd3f7b885
553 changed files with 37528 additions and 5238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
.circleci/config.yml
View file

@ -2915,6 +2915,25 @@ jobs:
exit 1
fi
provider_replay_harness:
docker:
- *python312_image
working_directory: ~/project
resource_class: medium
steps:
- setup_litellm_test_deps
- run:
name: Test provider replay harness
command: |
mkdir -p test-results/provider-replay-harness
uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
--junitxml=test-results/provider-replay-harness/junit.xml \
tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
tests/code_coverage_tests/test_provider_replay_harness.py
- store_test_results:
path: test-results/provider-replay-harness
integration_contracts:
parameters:
suite:
@ -2967,6 +2986,7 @@ workflows:
only:
- main
- /litellm_.*/
- provider_replay_harness
- base_sdk_install:
filters: *main_branches
- local_testing_part1:

15
.github/e2e-stack/assert_tests_ran.py vendored
View file

@ -3,6 +3,9 @@ import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Final
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
from coverage_registry.management_cases import MANAGEMENT_CASES
def main() -> int:
selected: Final = tuple(sys.argv[2:])
@ -16,6 +19,17 @@ def main() -> int:
case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
)
missing: Final = tuple(path for path in selected if path not in passed)
required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
passed_nodes: Final = frozenset(
prop.get("value")
for case in cases
if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
for prop in case.findall("./properties/property")
if prop.get("name") == "management_node"
)
missing_nodes: Final = required_nodes - passed_nodes
for node in sorted(missing_nodes):
_ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
for path in selected:
collected: Final = sum(case.get("file") == path for case in cases)
skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
@ -27,6 +41,7 @@ def main() -> int:
if (
selected
and not missing
and not missing_nodes
and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
):
return 0

5
.github/e2e-stack/oidc-profile.sh vendored Executable file
View file

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "${REPO_ROOT}"
exec uv run --no-sync python tests/e2e/idp.py "$@"

2
.github/e2e-stack/select_tests.py vendored
View file

@ -12,6 +12,8 @@ UNSUPPORTED: Final = re.compile(
HARNESS: Final = re.compile(
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
r"|^tests/e2e/idp_realm\.json$"
r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
r"|^tests/e2e/coverage_registry/management_cases\.py$"
r"|^tests/e2e/gateway/"
r"|^\.github/e2e-stack/"
r"|^\.github/workflows/test-e2e-changed\.yml$"

3
.github/pull_request_template.md vendored
View file

@ -101,7 +101,8 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
For bug fixes: Before shows the reproduction, After shows the same steps passing
For new features: Before shows the capability missing, After shows it working end-to-end
If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), make each endpoint its own case, not just one
For UI changes: before/after screenshots under the same headings -->
For UI changes: before/after screenshots under the same headings
If the main use case runs through a coding tool like Claude Code or Codex, drive that tool interactively the way the user does (never `claude -p`, `codex exec`, or curl on its own) and embed before/after screenshots of its pane under the same headings; curl replays and headless runs can follow as extra cases, never as the only proof -->
## Type

2
.github/workflows/_test-unit-base.yml vendored
View file

@ -57,6 +57,7 @@ permissions:
env:
UV_PYTHON: "3.12"
LITELLM_LOCAL_MODEL_COST_MAP: "True"
jobs:
run:
@ -113,6 +114,7 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'

5
.github/workflows/test-code-quality.yml vendored
View file

@ -178,7 +178,7 @@ jobs:
version: "0.10.9"
- name: Install dependencies
run: uv sync --frozen --extra proxy --python 3.10
run: uv sync --frozen --extra proxy --extra cli --python 3.10
- run: uv run --no-sync python --version
@ -187,3 +187,6 @@ jobs:
- name: Check litellm CLI
run: uv run --no-sync litellm --version
- name: Check lite CLI
run: uv run --no-sync lite version

2
.github/workflows/test-e2e-changed.yml vendored
View file

@ -183,7 +183,7 @@ jobs:
log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
echo "::group::pass ${pass} of 3"
set +e
uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
-o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
status=$?
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"

2
CLAUDE.md
View file

@ -25,6 +25,8 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
Never test structure of code only function of it
A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`

3
Makefile
View file

@ -299,6 +299,9 @@ test-rust-extension:
[ "$$#" -eq 1 ] && \
UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
"$$temporary/venv/bin/python" -I -m mypy.stubtest \
--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
litellm.rust_bridge._native && \
LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.67"
version = "0.1.68"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.67"
version = "0.1.68"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

1
gateway/routes/allowlist.py
View file

@ -96,6 +96,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/langfuse/",
"/vllm/",
"/mistral/",
"/nvidia_nim/",
"/groq/",
"/voyage/",
"/cursor/",

2
litellm-proxy-extras/litellm_proxy_extras/_logging.py
View file

@ -40,4 +40,4 @@ if not logger.handlers:
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.setLevel(os.getenv("LITELLM_LOG", "INFO").upper())

14
litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql Normal file
View file

@ -0,0 +1,14 @@
-- AlterTable
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;

18
litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- DropIndex
DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_is_act_idx";
-- DropIndex
DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_key";
-- AlterTable
-- NOT NULL DEFAULT '' (not nullable): Postgres unique constraints treat every
-- NULL as distinct, so a nullable column would let multiple unscoped mappings
-- collide on the same claim without a constraint violation. The constant
-- default is a fast, metadata-only backfill for existing rows, not a rewrite.
ALTER TABLE "LiteLLM_JWTKeyMapping" ADD COLUMN IF NOT EXISTS "jwt_issuer" TEXT NOT NULL DEFAULT '';
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_idx" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value", "is_active");
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_key" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value");

13
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_reset_at DateTime?
@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
@ -483,6 +487,10 @@ model LiteLLM_VerificationToken {
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_issuer String @default("") // Scopes the mapping to one configured issuer; "" matches any issuer.
// Not nullable: Postgres unique constraints treat every NULL as
// distinct, so a nullable column would let multiple unscoped
// mappings collide on the same claim without a constraint violation.
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
@ -495,8 +503,8 @@ model LiteLLM_JWTKeyMapping {
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
@@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
@@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
@ -534,6 +542,7 @@ model LiteLLM_DeletedVerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.97"
version = "0.4.98"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.97"
version = "0.4.98"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

3
litellm/__init__.py
View file

@ -343,6 +343,7 @@ _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
"1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
)
openai_system_messages_first: bool = False
disable_vertex_batch_output_transformation: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"
@ -538,7 +539,7 @@ context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
allow_dynamic_callback_disabling: bool = True
num_retries_per_request: Optional[int] = None # cap on Router retries of one model group; resets per fallback hop
num_retries_per_request: Optional[int] = None # for the request overall (incl. fallbacks + model retries)
####### SECRET MANAGERS #####################
secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.

6
litellm/_logging.py
View file

@ -401,10 +401,14 @@ def _parse_json_logs_env(value: str | None) -> bool:
return (value or "").lower() == "true"
def resolve_log_level(log_level: str) -> int:
return getattr(logging, log_level.upper())
json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
# Create a handler for the logger (you may need to adapt this based on your needs)
log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
numeric_level: Final[str] = getattr(logging, log_level.upper())
numeric_level: Final[int] = resolve_log_level(log_level)
handler: Final = LevelRoutingStreamHandler()
handler.setLevel(numeric_level)
handler.addFilter(_secret_filter)

3
litellm/a2a_protocol/main.py
View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -507,7 +508,7 @@ async def asend_message(
prompt_tokens,
completion_tokens,
_,
) = A2ARequestUtils.calculate_usage_from_request_response(
) = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
request=request,
response_dict=response_dict,
)

5
litellm/a2a_protocol/streaming_iterator.py
View file

@ -11,6 +11,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
if TYPE_CHECKING:
@ -99,11 +100,11 @@ class A2AStreamingIterator:
# Calculate tokens from collected text
input_message: Final = A2ARequestUtils.get_input_message_from_request(self.request)
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
# Use the last (most complete) text from chunks
output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
completion_tokens: Final = A2ARequestUtils.count_tokens(output_text)
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
total_tokens: Final = prompt_tokens + completion_tokens

6
litellm/anthropic_beta_headers_config.json
View file

@ -22,6 +22,7 @@
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@ -52,6 +53,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@ -82,6 +84,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@ -113,6 +116,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
@ -144,6 +148,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
@ -176,6 +181,7 @@
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",

2
litellm/batches/batch_utils.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import token_counter
@ -356,6 +357,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
prompt_tokens=_prompt,
completion_tokens=_completion,
total_tokens=_total,
prompt_tokens_details=vertex_prompt_tokens_details(usage_metadata),
)
try:

125
litellm/caching/affinity_cache.py Normal file
View file

@ -0,0 +1,125 @@
"""Atomic affinity claims shared by deployment and tier-model selection."""
import json
from collections.abc import Mapping
from typing import (
Final,
cast, # noqa: TID251 # Redis script results are narrowed only to object, then validated
)
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
_PIN_JSON_ADAPTER: Final = TypeAdapter[JsonValue](JsonValue)
_CLAIM_PIN_SCRIPT: Final = """
local current = redis.call('GET', KEYS[1])
if current == false then
redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
return ARGV[1]
end
if ARGV[3] then
local decoded, stored = pcall(cjson.decode, current)
if decoded and type(stored) == 'table' then
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
local matches = true
for key, value in pairs(eligible) do
if stored[key] ~= value then matches = false; break end
end
for key, _ in pairs(stored) do
if eligible[key] == nil then matches = false; break end
end
if matches then
redis.call('EXPIRE', KEYS[1], ARGV[2])
return current
end
end
end
redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
return ARGV[1]
end
if current == ARGV[1] then
redis.call('EXPIRE', KEYS[1], ARGV[2])
end
return current
"""
def set_local_affinity_pin(cache: DualCache, cache_key: str, value: object, ttl_seconds: int) -> None:
"""Replace the entry because InMemoryCache.set_cache preserves a live key's expiry."""
cache.in_memory_cache.delete_cache(cache_key)
cache.in_memory_cache.set_cache(cache_key, value, ttl=ttl_seconds)
def _legacy_pin_matches(stored: object, pin_value: Mapping[str, str]) -> bool:
if isinstance(stored, dict):
return all(stored.get(key) is not None and str(stored[key]) == value for key, value in pin_value.items())
return isinstance(stored, str) and len(pin_value) == 1 and stored in pin_value.values()
def claim_affinity_pin_in_memory(
cache: DualCache,
cache_key: str,
pin_value: Mapping[str, str],
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
) -> object:
"""No await between read and write, so same-loop claims agree during a Redis outage."""
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
if existing is not None and eligible_values is None:
if _legacy_pin_matches(existing, pin_value):
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
return existing
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
def _decode_pin(value: str) -> object:
try:
return _PIN_JSON_ADAPTER.validate_json(value)
except ValidationError:
return value
async def claim_affinity_pin(
cache: DualCache,
cache_key: str,
pin_value: Mapping[str, str],
ttl_seconds: int,
*,
eligible_values: tuple[Mapping[str, str], ...] | None = None,
) -> object:
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
Eligible claims refresh the returned winner. Legacy deployment claims only refresh
a matching candidate. Resolve Redis per call because the proxy attaches it lazily.
"""
redis_cache: Final = cache.redis_cache
if redis_cache is not None:
try:
claim_script: Final = redis_cache.async_register_script(_CLAIM_PIN_SCRIPT)
args: Final = (
json.dumps(dict(pin_value)), # mutable-ok: JSON serialization requires dict, not a generic Mapping
int(ttl_seconds),
*(
(json.dumps(tuple(dict(value) for value in eligible_values)),) # mutable-ok: JSON requires dict
if eligible_values is not None
else ()
),
)
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
object, await claim_script(keys=(cache_key,), args=args)
)
decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
if not isinstance(decoded, str):
return pin_value
winner: Final = _decode_pin(decoded)
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
return winner
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)

35
litellm/caching/caching_handler.py
View file

@ -35,6 +35,7 @@ from litellm.litellm_core_utils.logging_utils import (
_assemble_complete_response_from_streaming_chunks,
)
from litellm.types.caching import CachedEmbedding
from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
@ -107,17 +108,31 @@ def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
return "choices" in cached_result
def _should_defer_streaming_cache_hit_callbacks(*, kwargs: dict[str, object]) -> bool:
def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool:
if kwargs.get("stream", False) is True:
return True
return converted_stream_requested(kwargs) and not kwargs.get("_agentic_loop_depth")
def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> bool:
"""
When stream=True, do not run success callbacks at cache-hit time.
When the cache hit is replayed as a stream, do not run success callbacks at cache-hit time.
Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
replay uses CachedResponsesAPIStreamingIterator; cached Anthropic Messages
replay uses CachedAnthropicMessagesStreamIterator. All invoke logging success
handlers when the stream finishes; firing them here too would double-count
spend and callback records.
spend and callback records. A plain (non-stream) replay logs here, since nothing
else will.
"""
return kwargs.get("stream", False) is True
from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
CachedAnthropicMessagesStreamIterator,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
return isinstance(
cached_result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator, CachedAnthropicMessagesStreamIterator)
)
def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]:
@ -267,7 +282,7 @@ class LLMCachingHandler:
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
# LOG SUCCESS
self._async_log_cache_hit_on_callbacks(
logging_obj=logging_obj,
@ -383,7 +398,7 @@ class LLMCachingHandler:
is_async=False,
)
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
@ -823,7 +838,7 @@ class LLMCachingHandler:
if (call_type == CallTypes.acompletion.value or call_type == CallTypes.completion.value) and isinstance(
cached_result, dict
):
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
cached_result = self._convert_cached_stream_response(
cached_result=cached_result,
call_type=call_type,
@ -838,7 +853,7 @@ class LLMCachingHandler:
if (
call_type == CallTypes.atext_completion.value or call_type == CallTypes.text_completion.value
) and isinstance(cached_result, dict):
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
cached_result = self._convert_cached_stream_response(
cached_result=cached_result,
call_type=call_type,
@ -893,7 +908,7 @@ class LLMCachingHandler:
elif (call_type == "aresponses" or call_type == "responses") and isinstance(cached_result, dict):
use_chat_completion_cache: Final = _is_chat_completion_cached_dict(cached_result)
if use_chat_completion_cache:
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
bridge_call_type: Final = (
CallTypes.acompletion.value if call_type == "aresponses" else CallTypes.completion.value
)
@ -921,7 +936,7 @@ class LLMCachingHandler:
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,

3
litellm/caching/qdrant_semantic_cache.py
View file

@ -21,6 +21,7 @@ from litellm.constants import (
QDRANT_VECTOR_SIZE,
SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -255,7 +256,7 @@ class QdrantSemanticCache(BaseCache):
llm_router = None
router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
embedding_input: Final = self._embedding_input(prompt, router)
embedding_input: Final = await asyncify(self._embedding_input)(prompt, router)
embedding_call: Final = (
router.aembedding(
model=self.embedding_model,

3
litellm/caching/redis_semantic_cache.py
View file

@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -522,7 +523,7 @@ class RedisSemanticCache(BaseCache):
llm_router = None
router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
embedding_input: Final = self._embedding_input(prompt, router)
embedding_input: Final = await asyncify(self._embedding_input)(prompt, router)
embedding_call: Final = (
router.aembedding(
model=self.embedding_model,

29
litellm/compression/compress.py
View file

@ -205,21 +205,42 @@ def _extract_anthropic_tool_exchange_spans(
return spans, None
def _message_has_cache_control(message: Mapping[str, object]) -> bool:
if message.get("cache_control") is not None:
return True
content: Final = message.get("content")
if isinstance(content, list):
return any(isinstance(part, Mapping) and part.get("cache_control") is not None for part in content)
return False
def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
last_breakpoint: Final = max(
(index for index, msg in enumerate(messages) if _message_has_cache_control(msg)),
default=-1,
)
return tuple(range(last_breakpoint + 1))
def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
"""
Return indices of messages that must never be compressed:
- All system messages
- The last user message
- The last assistant message
- Every message up to and including the last one carrying an Anthropic cache_control breakpoint
The last user message is what the model is being asked to act on right now,
so compressing it replaces the live instruction with a marker. Compression
guardrails share this policy; see the Headroom guardrail.
guardrails share this policy; see the Headroom guardrail. A cache_control
breakpoint pins the provider's prompt-cache prefix to the exact bytes of every
row up to it, so rewriting any row inside that prefix turns the next request's
cache read into a cache write.
"""
system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system")
last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:]
last_assistant = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")[-1:]
return system_indices + last_user + last_assistant
assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")
return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages)))
def _combine_scores(
@ -421,7 +442,7 @@ def compress(
combined_scores = bm25_scores
# Protected messages are never compressed
protected_indices: Final = get_protected_indices(normalized_messages)
protected_indices: Final = get_protected_indices(original_messages)
kept_indices: set[int] = set(protected_indices)
tool_exchange_spans: list[set[int]] = []

8
litellm/constants.py
View file

@ -317,6 +317,8 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model"
MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved"
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
@ -1566,6 +1568,8 @@ BASE_MCP_ROUTE: Final = "/mcp"
BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour
BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24)) # for 24 hours
BATCH_TPD_WINDOW_SECONDS: Final = 86400
BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd"
HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)) # 60 seconds
_background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS")
@ -1774,6 +1778,7 @@ DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_
LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"})
LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"default_internal_user_params",
"default_team_params",
@ -1791,6 +1796,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
# test_general_settings_ui_fields_are_db_overridable enforces that pairing.
"enable_anthropic_prompt_caching",
"anthropic_prompt_caching_ttl",
"openai_system_messages_first",
"max_ui_session_budget",
"budget_rollover",
"mcp_tool_search",
@ -1976,6 +1982,8 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
STRINGIFIED_NONE: Final[str] = "None"
# A retrieved response replays the usage of the call that created it, so pricing these
# read/management routes like inference bills the same tokens twice.
NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset(

141
litellm/cost_calculator.py
View file

@ -814,6 +814,7 @@ def _select_model_name_for_cost_calc(
if (
entry.get("input_cost_per_token") is not None
or entry.get("input_cost_per_second") is not None
or entry.get("input_cost_per_query") is not None
or entry.get("tiered_pricing") is not None
):
return_model = router_model_id
@ -1202,6 +1203,24 @@ def _without_provider_stated_cost(usage: Usage | None) -> Usage | None:
return usage.model_copy(update=MappingProxyType({"cost": None}))
def _split_responses_ws_logging_object_by_service_tier(
completion_response: LiteLLMRealtimeStreamLoggingObject,
) -> tuple[LiteLLMRealtimeStreamLoggingObject, ...] | None:
partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(
cast(Sequence[Mapping[str, object]], completion_response.results)
)
if len(partition) <= 1:
return None
return tuple(
LiteLLMRealtimeStreamLoggingObject(
results=cast(OpenAIRealtimeStreamList, list(group)),
usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(group),
service_tier=tier,
)
for tier, group in partition.items()
)
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -1265,6 +1284,41 @@ def completion_cost(
try:
call_type = _infer_call_type(call_type, completion_response) or "completion"
if call_type == CallTypes.aresponses_websocket.value and isinstance(
completion_response, LiteLLMRealtimeStreamLoggingObject
):
ws_tier_parts: Final = _split_responses_ws_logging_object_by_service_tier(completion_response)
if ws_tier_parts is not None:
return sum(
completion_cost(
completion_response=part,
model=model,
prompt=prompt,
messages=messages,
completion=completion,
total_time=total_time,
call_type=call_type,
custom_llm_provider=custom_llm_provider,
region_name=region_name,
size=size,
quality=quality,
n=n,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
)
for part in ws_tier_parts
)
if (
(call_type == "aimage_generation" or call_type == "image_generation")
and model is not None
@ -1465,12 +1519,15 @@ def completion_cost(
duration_seconds = usage_obj.get("duration_seconds", None)
_vr = usage_obj.get("video_resolution", None)
provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
_vc = usage_obj.get("video_count", None)
else:
duration_seconds = getattr(usage_obj, "duration_seconds", None)
_vr = getattr(usage_obj, "video_resolution", None)
provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
_vc = getattr(usage_obj, "video_count", None)
if _vr is not None:
video_resolution = str(_vr).strip().lower()
video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1
if _video_model_info is None and provider_reported_cost is not None:
return float(provider_reported_cost)
@ -1481,12 +1538,15 @@ def completion_cost(
video_generation_cost,
)
return video_generation_cost(
model=model,
duration_seconds=duration_seconds,
custom_llm_provider=custom_llm_provider,
model_info=_video_model_info,
video_resolution=video_resolution,
return (
video_generation_cost(
model=model,
duration_seconds=duration_seconds,
custom_llm_provider=custom_llm_provider,
model_info=_video_model_info,
video_resolution=video_resolution,
)
* video_count
)
# Fallback to default video cost calculation if no duration available
return default_video_cost_calculator(
@ -2278,6 +2338,19 @@ def default_video_cost_calculator(
return 0.0
def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
fallback: float,
) -> float:
rate: Final = model_info.get(key)
return fallback if rate is None else rate
def batch_cost_calculator(
usage: Usage,
model: str,
@ -2337,7 +2410,29 @@ def batch_cost_calculator(
total_prompt_cost = 0.0
total_completion_cost = 0.0
if input_cost_per_token_batches is not None:
total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
batch_details: Final = parse_prompt_tokens_details(usage)
audio_tokens, image_tokens, video_tokens = (
batch_details["audio_tokens"],
batch_details["image_tokens"],
batch_details["video_tokens"],
)
modality_rates: Final = (
_batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches),
_batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches),
_batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches),
)
total_prompt_cost = sum(
tokens * rate
for tokens, rate in zip(
(
max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0),
audio_tokens,
image_tokens,
video_tokens,
),
(input_cost_per_token_batches, *modality_rates),
)
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = details["cache_hit_tokens"]
@ -2522,6 +2617,7 @@ _RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "re
class _ResponsesWsEventResponse(BaseModel):
usage: Mapping[str, object] | None = None
service_tier: str | None = None
class _ResponsesWsEvent(BaseModel):
@ -2529,20 +2625,39 @@ class _ResponsesWsEvent(BaseModel):
response: _ResponsesWsEventResponse | None = None
def _billable_responses_ws_events(
results: Sequence[Mapping[str, object]],
) -> tuple[tuple[Mapping[str, object], _ResponsesWsEventResponse], ...]:
return tuple(
(result, event.response)
for result in results
if (event := _ResponsesWsEvent.model_validate(result)).type in _RESPONSES_WS_BILLABLE_EVENT_TYPES
and event.response is not None
and event.response.usage is not None
)
class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
@staticmethod
def collect_usage_from_responses_ws_results(
results: Sequence[Mapping[str, object]],
) -> tuple[Usage, ...]:
events: Final = tuple(_ResponsesWsEvent.model_validate(result) for result in results)
return tuple(
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the realtime processor uses
event.response.usage
response.usage
)
for event in events
if event.type in _RESPONSES_WS_BILLABLE_EVENT_TYPES
and event.response is not None
and event.response.usage is not None
for _, response in _billable_responses_ws_events(results)
if response.usage is not None
)
@staticmethod
def partition_results_by_service_tier(
results: Sequence[Mapping[str, object]],
) -> Mapping[str | None, tuple[Mapping[str, object], ...]]:
billable: Final = _billable_responses_ws_events(results)
tiers: Final = dict.fromkeys(response.service_tier for _, response in billable)
return MappingProxyType(
{tier: tuple(result for result, response in billable if response.service_tier == tier) for tier in tiers}
)
@staticmethod

3
litellm/integrations/compression_interception/handler.py
View file

@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
CompressionSavingsMetadata,
@ -153,7 +154,7 @@ class CompressionInterceptionLogger(CustomLogger):
self._prune_expired_cache()
compressed: Final = compress(
compressed: Final = await asyncify(compress)(
messages=messages,
model=model,
call_type=CallTypes.anthropic_messages,

49
litellm/integrations/custom_logger.py
View file

@ -822,46 +822,33 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
def truncate_standard_logging_payload_content(
self,
standard_logging_object: StandardLoggingPayload,
):
) -> StandardLoggingPayload:
"""
Truncate error strings and message content in logging payload
Return a copy of the logging payload with error_str, messages, and response truncated
Some loggers like DataDog/ GCS Bucket have a limit on the size of the payload. (1MB)
This function truncates the error string and the message content if they exceed a certain length.
Every callback of a request shares one standard logging object, so the payload passed in is left
untouched and the callbacks that run later (the prompt caching router check, spend logs) still see
the original fields.
"""
MAX_STR_LENGTH: Final = 10_000
max_str_length: Final = 10_000
candidates: Final = {
field: self._truncate_field(field_value=standard_logging_object.get(field), max_length=max_str_length)
for field in ("error_str", "messages", "response")
}
truncated_fields: Final = {field: text for field, text in candidates.items() if text is not None}
return {**standard_logging_object, **truncated_fields}
# Truncate fields that might exceed max length
fields_to_truncate: Final = ["error_str", "messages", "response"]
for field in fields_to_truncate:
self._truncate_field(
standard_logging_object=standard_logging_object,
field_name=field,
max_length=MAX_STR_LENGTH,
)
def _truncate_field(
self,
standard_logging_object: StandardLoggingPayload,
field_name: str,
max_length: int,
) -> None:
def _truncate_field(self, field_value: object, max_length: int) -> str | None:
"""
Helper function to truncate a field in the logging payload
Return the truncated text of a field that exceeds max_length, or None when the field fits
This converts the field to a string and then truncates it if it exceeds the max length.
Why convert to string ?
1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
- Converting to string and then truncating the logged content catches this
2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
The field is measured as a string because users send poorly formatted lists for `messages`, so there is
no fixed place the content would be.
"""
field_value: Final[object] = standard_logging_object.get(field_name)
if field_value:
str_value: Final = str(field_value)
if len(str_value) > max_length:
standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
text: Final = str(field_value or "")
return self._truncate_text(text=text, max_length=max_length) if len(text) > max_length else None
def _truncate_text(self, text: str, max_length: int) -> str:
"""Truncate text if it exceeds max_length"""

5
litellm/integrations/datadog/datadog.py
View file

@ -563,11 +563,10 @@ class DataDogLogger(
if standard_logging_object.get("status") == "failure":
status = DataDogStatus.ERROR
# Build the initial payload
self.truncate_standard_logging_payload_content(standard_logging_object)
truncated_payload: Final = self.truncate_standard_logging_payload_content(standard_logging_object)
dd_payload: Final = self._create_datadog_logging_payload_helper(
standard_logging_object=standard_logging_object,
standard_logging_object=truncated_payload,
status=status,
)
return dd_payload

2
litellm/integrations/langsmith.py
View file

@ -39,6 +39,8 @@ def is_serializable(value):
class LangsmithLogger(CustomBatchLogger):
preserve_events_added_during_flush = True
def __init__(
self,
langsmith_api_key: str | None = None,

8
litellm/integrations/prometheus.py
View file

@ -2610,12 +2610,6 @@ class PrometheusLogger(CustomLogger):
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict,
exception=original_exception,
):
return
status_code: Final = self._extract_status_code(exception=original_exception)
try:
@ -2633,7 +2627,7 @@ class PrometheusLogger(CustomLogger):
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
user_email=user_api_key_dict.user_email,
hashed_api_key=user_api_key_dict.api_key,
hashed_api_key=None if status_code == 401 else user_api_key_dict.api_key,
api_key_alias=user_api_key_dict.key_alias,
team=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -309,8 +309,8 @@ def max_retries_per_request_hit(kwargs: Mapping[str, object], num_retries_per_re
metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs))
if not isinstance(metadata, Mapping):
return False
attempted_retries: Final = metadata.get("attempted_retries")
return type(attempted_retries) is int and 0 < attempted_retries and num_retries_per_request <= attempted_retries
retry_count: Final = metadata.get("request_retry_count")
return type(retry_count) is int and 0 < retry_count and num_retries_per_request <= retry_count
def get_or_create_metadata_bucket(

101
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -1,6 +1,9 @@
import inspect
import json
import re
import traceback
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, Protocol, cast
import httpx
@ -202,11 +205,17 @@ def _get_response_headers(original_exception: Exception) -> httpx.Headers | None
return _response_headers
def _accepted_init_kwargs(exception_class: type[Exception], candidates: Mapping[str, object]) -> Mapping[str, object]:
accepted: Final = inspect.signature(exception_class).parameters
return MappingProxyType({name: value for name, value in candidates.items() if name in accepted})
def extract_and_raise_litellm_exception(
response: Any | None,
error_str: str,
model: str,
custom_llm_provider: str,
body: object | None = None,
):
"""
Covers scenario where litellm sdk calling proxy.
@ -216,32 +225,19 @@ def extract_and_raise_litellm_exception(
Relevant Issue: https://github.com/BerriAI/litellm/issues/7259
"""
pattern: Final = r"litellm\.\w+Error"
# Search for the exception in the error string
match: Final = re.search(pattern, error_str)
# Extract the exception if found
if match:
exception_name = match.group(0)
exception_name = exception_name.strip().replace("litellm.", "")
raised_exception_obj: Final = getattr(litellm, exception_name, None)
if raised_exception_obj:
# Try with response parameter first, fall back to without it
# Some exceptions (e.g., APIConnectionError) don't accept response param
try:
raise raised_exception_obj(
message=error_str,
llm_provider=custom_llm_provider,
model=model,
response=response,
)
except TypeError:
# Exception doesn't accept response parameter
raise raised_exception_obj(
message=error_str,
llm_provider=custom_llm_provider,
model=model,
)
if match is None:
return
exception_name: Final = match.group(0).removeprefix("litellm.")
raised_exception_obj: Final = getattr(litellm, exception_name, None)
if not raised_exception_obj:
return
raise raised_exception_obj(
message=error_str,
llm_provider=custom_llm_provider,
model=model,
**_accepted_init_kwargs(raised_exception_obj, MappingProxyType({"response": response, "body": body})),
)
class _ProviderHTTPException(Protocol):
@ -254,6 +250,23 @@ class _ProviderHTTPException(Protocol):
llm_provider: str
def _litellm_proxy_response(
original_exception: _ProviderHTTPException, custom_llm_provider: str
) -> httpx.Response | None:
response: Final = getattr(original_exception, "response", None)
if custom_llm_provider != "litellm_proxy" or not isinstance(response, httpx.Response) or response.headers:
return response
headers: Final = getattr(original_exception, "headers", None)
if not isinstance(headers, Mapping) or not headers:
return response
pairs: Final = headers.multi_items() if isinstance(headers, httpx.Headers) else headers.items()
return httpx.Response(
status_code=response.status_code,
headers=[(str(k), str(v)) for k, v in pairs],
request=getattr(original_exception, "request", None),
)
def _map_openai_exception(
*,
model: str,
@ -264,6 +277,7 @@ def _map_openai_exception(
exception_provider: str,
extra_information: str,
) -> None:
response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
# custom_llm_provider is openai, make it OpenAI
message = get_error_message(error_obj=original_exception)
if message is None:
@ -292,14 +306,14 @@ def _map_openai_exception(
message=f"RateLimitError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
)
elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
raise ContextWindowExceededError(
message=f"ContextWindowExceededError: {exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif "invalid_request_error" in error_str and "model_not_found" in error_str:
@ -307,7 +321,7 @@ def _map_openai_exception(
message=f"{exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif "A timeout occurred" in error_str:
@ -326,8 +340,9 @@ def _map_openai_exception(
message=f"ContentPolicyViolationError: {exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
elif "invalid_encrypted_content" in error_str or "could not be verified" in error_str:
helpful_message: Final = (
@ -345,7 +360,7 @@ def _map_openai_exception(
message=helpful_message,
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
@ -354,7 +369,7 @@ def _map_openai_exception(
message=f"{exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
@ -372,7 +387,7 @@ def _map_openai_exception(
message=f"RateLimitError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif (
@ -383,7 +398,7 @@ def _map_openai_exception(
message=f"AuthenticationError: {exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif "Mistral API raised a streaming error" in error_str:
@ -402,15 +417,16 @@ def _map_openai_exception(
message=f"{exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
elif original_exception.status_code == 401:
raise AuthenticationError(
message=f"AuthenticationError: {exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif original_exception.status_code == 404:
@ -418,7 +434,7 @@ def _map_openai_exception(
message=f"NotFoundError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif original_exception.status_code == 408:
@ -433,7 +449,7 @@ def _map_openai_exception(
message=f"{exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
@ -442,7 +458,7 @@ def _map_openai_exception(
message=f"RateLimitError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif original_exception.status_code == 500:
@ -450,7 +466,7 @@ def _map_openai_exception(
message=f"InternalServerError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif original_exception.status_code == 502:
@ -458,7 +474,7 @@ def _map_openai_exception(
message=f"BadGatewayError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif original_exception.status_code == 503:
@ -466,7 +482,7 @@ def _map_openai_exception(
message=f"ServiceUnavailableError: {exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
response=response,
litellm_debug_info=extra_information,
)
elif original_exception.status_code == 504: # gateway timeout error
@ -2423,10 +2439,11 @@ def exception_type(
custom_llm_provider == "litellm_proxy"
): # handle special case where calling litellm proxy + exception str contains error message
extract_and_raise_litellm_exception(
response=getattr(original_exception, "response", None),
response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
error_str=error_str,
model=model,
custom_llm_provider=custom_llm_provider,
body=getattr(original_exception, "body", None),
)
if (
custom_llm_provider == "openai"

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -41,6 +41,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"azure_password",
"azure_scope",
"timeout",
"client_side_timeout",
"gcs_bucket_name",
"bucket_name",
"vertex_credentials",

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -238,6 +238,8 @@ def get_llm_provider(
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
raise Exception(f"dynamic_api_key needs to be a string. Got type={type(dynamic_api_key).__name__}")
return model, custom_llm_provider, dynamic_api_key, api_base
if "/" in model and is_registered_custom_provider(provider_prefix):
return model.split("/", 1)[1], provider_prefix, dynamic_api_key, api_base
# check if api base is a known openai compatible endpoint
if api_base:
for endpoint in litellm.openai_compatible_endpoints:
@ -536,6 +538,10 @@ def get_llm_provider(
)
def is_registered_custom_provider(custom_llm_provider: str | None) -> bool:
return any(item["provider"] == custom_llm_provider for item in litellm.custom_provider_map)
def _dashscope_family_chat_config(custom_llm_provider: str) -> "litellm.DashScopeChatConfig":
if custom_llm_provider == "qwencloud":
return litellm.QwenCloudChatConfig()

29
litellm/litellm_core_utils/litellm_logging.py
View file

@ -644,6 +644,24 @@ class Logging(LiteLLMLoggingBaseClass):
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics) # mutable-ok: kept deep-copyable
def add_dynamic_callback(self, callback: CustomLogger) -> None:
self.dynamic_input_callbacks = self._with_dynamic_callback(self.dynamic_input_callbacks, callback)
self.dynamic_success_callbacks = self._with_dynamic_callback(self.dynamic_success_callbacks, callback)
self.dynamic_async_success_callbacks = self._with_dynamic_callback(
self.dynamic_async_success_callbacks, callback
)
self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, callback)
self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
self.dynamic_async_failure_callbacks, callback
)
@staticmethod
def _with_dynamic_callback(
callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
) -> list[str | Callable | CustomLogger]:
existing: Final = tuple(callbacks or ())
return [*existing, *(() if callback in existing else (callback,))]
def process_dynamic_callbacks(self):
"""
Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks
@ -1973,6 +1991,12 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
if isinstance(usage, Usage):
self.record_partial_usage_for_failure(usage, self._response_cost_calculator(result=assembled) or 0.0)
async def dispatch_failure_handlers(
self,
exception: Exception,
@ -2077,9 +2101,14 @@ class Logging(LiteLLMLoggingBaseClass):
results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
)
)
ws_tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(
results=result # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
)
ws_service_tier: Final = next(iter(ws_tier_partition)) if len(ws_tier_partition) == 1 else None
logging_result = LiteLLMRealtimeStreamLoggingObject(
usage=combined_ws_usage,
results=result, # pyright: ignore[reportUnknownArgumentType] # raw event dicts from the WS stream
service_tier=ws_service_tier,
)
elif (

71
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -484,11 +484,12 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
def _apply_off_peak_to_base_costs(
model_info: ModelInfo,
current_time: datetime | None,
base_costs: tuple[float, float, float, float, float],
base_costs: tuple[float, float, float, float | None, float],
) -> tuple[float, float, float, float, float]:
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
produced them. The one-hour cache-creation rate passes through untouched, since
off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
present one passes through untouched and an absent one resolves to the applied
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
"""
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
rates: Final = apply_off_peak_pricing(
@ -506,7 +507,7 @@ def _apply_off_peak_to_base_costs(
rates.input_rate,
rates.output_rate,
rates.cache_creation_rate,
cache_creation_above_1hr,
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
rates.cache_read_rate,
)
@ -532,6 +533,11 @@ def _get_token_base_cost(
`missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved
input rate instead of 0.0; an explicit 0.0 rate stays a real price either way.
An absent cache-creation rate always resolves to the resolved input rate, the way the
tiered table and custom deployment pricing already do, since a provider that publishes
no write price bills cache writes as ordinary input. An absent 1h write rate resolves
to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
@ -554,10 +560,9 @@ def _get_token_base_cost(
output_image_cost: Final = _get_cost_per_unit(model_info, "output_cost_per_image_token", None)
if output_image_cost is not None:
completion_base_cost = cast(float, output_image_cost)
cache_creation_cost = cast(float, _get_cost_per_unit(model_info, cache_creation_cost_key))
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
cache_creation_cost = _get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
cache_creation_cost_above_1hr = _get_cost_per_unit(
model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
)
cache_read_cost = _get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
@ -639,22 +644,10 @@ def _get_token_base_cost(
else f"cache_read_input_token_cost_above_{threshold_str}_tokens"
)
cache_creation_cost = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_tiered_key,
cache_creation_cost,
),
)
cache_creation_cost = _get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(
model_info,
cache_creation_1hr_tiered_key,
cache_creation_cost_above_1hr,
),
cache_creation_cost_above_1hr = _get_cost_per_unit(
model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
)
cache_read_cost = _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
@ -665,16 +658,16 @@ def _get_token_base_cost(
except Exception:
continue
input_rate_for_missing_cache_rates: Final = _off_peak_rate(
_open_off_peak_block(model_info, current_time) or MappingProxyType({}),
"input_cost_per_token",
prompt_base_cost,
)
if cache_read_cost is None:
cache_read_cost = (
_off_peak_rate(
_open_off_peak_block(model_info, current_time) or MappingProxyType({}),
"input_cost_per_token",
prompt_base_cost,
)
if missing_cache_read_uses_input
else 0.0
)
cache_read_cost = input_rate_for_missing_cache_rates if missing_cache_read_uses_input else 0.0
resolved_cache_creation_cost: Final = (
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
)
return _apply_off_peak_to_base_costs(
model_info,
@ -682,7 +675,7 @@ def _get_token_base_cost(
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
resolved_cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
),
@ -956,12 +949,16 @@ def _calculate_input_cost(
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"]:
if prompt_tokens_details["audio_tokens"] and not (
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
):
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"]:
if prompt_tokens_details["image_tokens"] and not (
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
):
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
@ -970,7 +967,9 @@ def _calculate_input_cost(
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"]:
if prompt_tokens_details["video_tokens"] and not (
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
):
video_token_cost_key = "input_cost_per_video_token"
if model_info.get(video_token_cost_key) is None:
video_token_cost_key = "input_cost_per_token"

16
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -2256,6 +2256,22 @@ def drop_tool_reference_parts_from_tool_messages(
return [_drop_tool_reference_parts(message) for message in messages] # mutable-ok: pipelines mutate message lists
INSTRUCTION_MESSAGE_ROLES: Final = frozenset({"system", "developer"})
def _is_instruction_message(message: AllMessageValues) -> bool:
return message.get("role") in INSTRUCTION_MESSAGE_ROLES
def system_messages_first(
messages: list[AllMessageValues], # mutable-ok: message pipelines type messages as mutable lists
) -> list[AllMessageValues]: # mutable-ok: message pipelines type messages as mutable lists
return [ # mutable-ok: pipelines mutate message lists
*(message for message in messages if _is_instruction_message(message)),
*(message for message in messages if not _is_instruction_message(message)),
]
def _attempt_json_repair(s: str) -> object | None:
"""
Attempt to repair truncated JSON produced by LLM tool calls.

46
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1500,19 +1500,33 @@ def convert_to_gemini_tool_call_result(
return _part
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""
Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
_TOOL_USE_ID_FALLBACK: Final = "tool_use_id"
_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
_BEDROCK_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_.:-]")
_BEDROCK_TOOL_USE_ID_MAX_LEN: Final = 64
_BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8
Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
This function replaces any invalid characters with underscores.
def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str:
return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$."""
return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS)
def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
"""
# Replace any character that's not alphanumeric, underscore, or hyphen with underscore
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
# Ensure it's not empty (fallback to a default if needed)
if not sanitized:
sanitized = "tool_use_id"
return sanitized
Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars.
Ids that need rewriting get a short hash of the original appended so two ids that only
differ in a replaced char or past the cut still map to distinct values.
"""
sanitized: Final = _replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS)
if sanitized == tool_use_id and len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN:
return sanitized
digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN]
return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}"
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
@ -3661,7 +3675,9 @@ def _convert_to_bedrock_tool_call_invoke(
if parsed_objects:
# First object keeps the original tool id.
for obj_idx, obj in enumerate(parsed_objects):
block_id = tool_id if obj_idx == 0 else f"{tool_id}_{obj_idx}"
block_id = _sanitize_bedrock_tool_use_id(
tool_id if obj_idx == 0 else f"{tool_id}_{obj_idx}"
)
bedrock_tool = BedrockToolUseBlock(input=obj, name=name, toolUseId=block_id)
_parts_list.append(BedrockContentBlock(toolUse=bedrock_tool))
# cache_control applies to the whole original
@ -3678,7 +3694,9 @@ def _convert_to_bedrock_tool_call_invoke(
# Fallback: no objects extracted — use empty dict.
arguments_dict = {}
bedrock_tool = BedrockToolUseBlock(input=arguments_dict, name=name, toolUseId=tool_id)
bedrock_tool = BedrockToolUseBlock(
input=arguments_dict, name=name, toolUseId=_sanitize_bedrock_tool_use_id(tool_id)
)
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
_parts_list.append(bedrock_content_block)
@ -3849,7 +3867,7 @@ def _convert_to_bedrock_tool_call_result(
tool_result_content_blocks, used_search_results = _build_bedrock_tool_result_content_blocks(message)
message.get("name", "")
id: Final = str(message.get("tool_call_id", str(uuid.uuid4())))
id: Final = _sanitize_bedrock_tool_use_id(str(message.get("tool_call_id", str(uuid.uuid4()))))
tool_result: Final = BedrockToolResultBlock(content=tool_result_content_blocks, toolUseId=id)
if used_search_results:

3
litellm/litellm_core_utils/streaming_handler.py
View file

@ -19,6 +19,7 @@ from typing_extensions import NotRequired, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
@ -2247,7 +2248,7 @@ class CustomStreamWrapper:
if self.sent_last_chunk is True:
# log the final chunk with accurate streaming values
try:
complete_streaming_response = litellm.stream_chunk_builder(
complete_streaming_response = await asyncify(litellm.stream_chunk_builder)(
chunks=self.chunks,
messages=self.messages,
logging_obj=self.logging_obj,

240
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -20,6 +20,7 @@ from itertools import chain, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
@ -44,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
scoped_structured_message_indices,
stream_item_field,
stream_item_fingerprint,
unappliable_request_rewrite,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
AnthropicPassthroughLoggingHandler,
@ -103,9 +105,24 @@ class ToolResultBlockTextTarget:
block_idx: int
InputWriteBackTarget = (
MessageContentTarget | ContentBlockTextTarget | ToolResultStringTarget | ToolResultBlockTextTarget
)
@dataclass(frozen=True, slots=True)
class SystemStringTarget:
pass
@dataclass(frozen=True, slots=True)
class SystemBlockTextTarget:
block_idx: int
@dataclass(frozen=True, slots=True)
class ToolUseInputTarget:
msg_idx: int
content_idx: int
MessageTextTarget = MessageContentTarget | ContentBlockTextTarget | ToolResultStringTarget | ToolResultBlockTextTarget
InputWriteBackTarget = SystemStringTarget | SystemBlockTextTarget | MessageTextTarget
def _as_str_mapping(value: Mapping[str, object]) -> Mapping[str, object]:
@ -146,10 +163,17 @@ class ScannedText:
target: InputWriteBackTarget
@dataclass(frozen=True, slots=True)
class ScannedToolCall:
tool_call: ChatCompletionToolCallChunk
target: ToolUseInputTarget
@dataclass(frozen=True, slots=True)
class ExtractedInput:
scanned: tuple[ScannedText, ...]
images: tuple[str, ...]
tool_calls: tuple[ScannedToolCall, ...] = ()
EMPTY_EXTRACTED_INPUT: Final = ExtractedInput(scanned=(), images=())
@ -161,6 +185,74 @@ class _ToolCallShape:
arguments: str
def _is_client_tool_use(block: Mapping[str, object]) -> bool:
return (
block.get("type") == "tool_use"
and isinstance(block.get("id"), str)
and isinstance(block.get("name"), str)
and isinstance(block.get("input"), dict)
)
def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
if not isinstance(system, list):
return
text_blocks: Final = tuple(block for block in system if isinstance(block, dict) and block.get("type") == "text")
if block_idx < len(text_blocks):
text_blocks[block_idx]["text"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
def _write_back_message_text(message: _WritableMessage, target: MessageTextTarget, response: str) -> None:
content: Final = message.get("content", None)
if content is None:
return
match target:
case MessageContentTarget():
if isinstance(content, str):
message["content"] = response # mutable-ok: guardrails rewrite the caller's request payload in place
case ContentBlockTextTarget(content_idx=content_idx):
if isinstance(content, list):
content[content_idx]["text"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case ToolResultStringTarget(content_idx=content_idx):
if isinstance(content, list):
content[content_idx]["content"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
if isinstance(content, list):
content[content_idx]["content"][block_idx]["text"] = (
response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case _:
assert_never(target)
_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
def _rewritten_tool_use_input(arguments: str) -> Mapping[str, object] | None:
try:
return _TOOL_USE_INPUT_ADAPTER.validate_json(arguments)
except ValidationError:
return None
def _write_back_tool_use(
message: _WritableMessage, target: ToolUseInputTarget, shape: _ToolCallShape, rewritten_input: Mapping[str, object]
) -> None:
content: Final = message.get("content", None)
block: Final = content[target.content_idx] if isinstance(content, list) else None
if not isinstance(block, dict):
return
block["input"] = rewritten_input # mutable-ok: guardrails rewrite the caller's request payload in place
if shape.name is not None and shape.name != block.get("name"):
block["name"] = shape.name # mutable-ok: guardrails rewrite the caller's request payload in place
@dataclass(frozen=True, slots=True)
class _SSEFieldRewrite:
"""One field of one nested section of a buffered SSE event, rewritten."""
@ -452,9 +544,8 @@ class AnthropicMessagesHandler(BaseTranslation):
skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
scan_only_tool_results: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
# Exclude only the trusted top-level prompt. In-sequence system entries are untrusted
# and must stay aligned with texts_to_check for positional masking. When the top-level
# prompt is included, the pre-existing count mismatch disables positional masking.
# The top-level prompt is translated on its own below so it can be hoisted in front of
# any mid-turn system entries and scanned first, aligned with that structured position.
translation_source: Final = { # mutable-ok: API message payload
key: value for key, value in data.items() if key != "system"
}
@ -490,7 +581,12 @@ class AnthropicMessagesHandler(BaseTranslation):
]
)
# Step 1: Extract all text content and images
# Step 1: Extract all text content, images, and tool calls
top_level_system_scanned: Final = (
()
if hoisted_system_message is None or scan_only_tool_results
else self._extract_top_level_system_text(hoisted_system_message)
)
extracted: Final = tuple(
self._extract_input_text_and_images(
message=message,
@ -501,17 +597,27 @@ class AnthropicMessagesHandler(BaseTranslation):
)
for msg_idx, message in enumerate(messages)
)
scanned: Final = tuple(item for one_message in extracted for item in one_message.scanned)
scanned: Final = (
*top_level_system_scanned,
*(item for one_message in extracted for item in one_message.scanned),
)
texts_to_check: Final = [item.text for item in scanned] # mutable-ok: GenericGuardrailAPIInputs takes list[str]
images_to_check: Final = [
image for one_message in extracted for image in one_message.images
] # mutable-ok: GenericGuardrailAPIInputs takes list[str]
scanned_tool_calls: Final = tuple(item for one_message in extracted for item in one_message.tool_calls)
tool_calls_to_check: Final = [
item.tool_call for item in scanned_tool_calls
] # mutable-ok: GenericGuardrailAPIInputs takes list[ChatCompletionToolCallChunk]
pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check)
# Step 2: Apply guardrail to all texts in batch
if texts_to_check:
# Step 2: Apply guardrail to all texts and tool calls in batch
if texts_to_check or tool_calls_to_check:
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if images_to_check:
inputs["images"] = images_to_check
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check
if tools_to_check:
inputs["tools"] = tools_to_check
original_structured_messages: Final = structured_messages
@ -570,9 +676,18 @@ class AnthropicMessagesHandler(BaseTranslation):
preserve_system_messages=has_midturn_system_message,
)
else:
if guardrailed_texts and len(guardrailed_texts) != len(scanned):
raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
self._apply_guardrail_tool_calls_to_input(
messages=messages,
scanned_tool_calls=scanned_tool_calls,
pre_guardrail_tool_calls=pre_guardrail_tool_calls,
returned_tool_calls=guardrailed_inputs.get("tool_calls"),
guardrail_name=guardrail_to_apply.guardrail_name,
)
# Step 3: Map guardrail responses back to original message structure
await self._apply_guardrail_responses_to_input(
messages=messages,
data=data,
responses=guardrailed_texts,
scanned=scanned,
)
@ -598,6 +713,19 @@ class AnthropicMessagesHandler(BaseTranslation):
hoisted: Final = probe.get("messages") or [] # mutable-ok: API message payload
return hoisted[0] if hoisted else None
@staticmethod
def _extract_top_level_system_text(hoisted_system_message: AllMessageValues) -> tuple[ScannedText, ...]:
content: Final = hoisted_system_message.get("content")
if isinstance(content, str):
return (ScannedText(content, SystemStringTarget()),)
if not isinstance(content, list):
return ()
return tuple(
ScannedText(text_str, SystemBlockTextTarget(block_idx))
for block_idx, block in enumerate(content)
if isinstance(block, dict) and isinstance(text_str := block.get("text"), str)
)
@staticmethod
def _openai_system_message_to_anthropic(
message: Mapping[str, object],
@ -852,9 +980,25 @@ class AnthropicMessagesHandler(BaseTranslation):
for content_idx, content_item in enumerate(content)
if isinstance(content_item, dict)
)
tool_use_blocks: Final = (
()
if scan_only_tool_results
else tuple(
(content_idx, content_item)
for content_idx, content_item in enumerate(content)
if isinstance(content_item, dict) and _is_client_tool_use(content_item)
)
)
return ExtractedInput(
scanned=tuple(item for block in blocks for item in block.scanned),
images=tuple(image for block in blocks for image in block.images),
tool_calls=tuple(
ScannedToolCall(
tool_call=AnthropicConfig.convert_tool_use_to_openai_format(content_item, tool_call_idx),
target=ToolUseInputTarget(msg_idx, content_idx),
)
for tool_call_idx, (content_idx, content_item) in enumerate(tool_use_blocks)
),
)
@classmethod
@ -940,43 +1084,59 @@ class AnthropicMessagesHandler(BaseTranslation):
async def _apply_guardrail_responses_to_input(
self,
messages: Sequence[_WritableMessage],
responses: list[str],
data: dict[str, object], # mutable-ok: API message payload
responses: Sequence[str],
scanned: tuple[ScannedText, ...],
) -> None:
"""
Apply guardrail responses back to input messages.
Apply guardrail responses back to the top-level system prompt and the input messages.
"""
raw_messages: Final = data.get("messages")
messages: Final[Sequence[_WritableMessage]] = raw_messages if isinstance(raw_messages, list) else ()
for item, guardrail_response in zip(scanned, responses):
target = item.target
message = messages[target.msg_idx]
content = message.get("content", None)
if content is None:
continue
match target:
case MessageContentTarget():
if isinstance(content, str):
message["content"] = (
guardrail_response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case ContentBlockTextTarget(content_idx=content_idx):
if isinstance(content, list):
content[content_idx]["text"] = (
guardrail_response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case ToolResultStringTarget(content_idx=content_idx):
if isinstance(content, list):
content[content_idx]["content"] = (
guardrail_response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
if isinstance(content, list):
content[content_idx]["content"][block_idx]["text"] = (
match item.target:
case SystemStringTarget():
if isinstance(data.get("system"), str):
data["system"] = (
guardrail_response # mutable-ok: guardrails rewrite the caller's request payload in place
)
case SystemBlockTextTarget(block_idx=block_idx):
_write_back_system_block(data.get("system"), block_idx, guardrail_response)
case (
MessageContentTarget()
| ContentBlockTextTarget()
| ToolResultStringTarget()
| ToolResultBlockTextTarget() as message_target
):
_write_back_message_text(messages[message_target.msg_idx], message_target, guardrail_response)
case _:
assert_never(target)
assert_never(item.target)
@staticmethod
def _apply_guardrail_tool_calls_to_input(
messages: Sequence[_WritableMessage],
scanned_tool_calls: tuple[ScannedToolCall, ...],
pre_guardrail_tool_calls: tuple[_ToolCallShape, ...],
returned_tool_calls: Sequence[object] | None,
guardrail_name: str | None,
) -> None:
post_guardrail_tool_calls: Final = _tool_call_shapes(
returned_tool_calls
if returned_tool_calls is not None and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
else tuple(item.tool_call for item in scanned_tool_calls)
)
rewritten: Final = tuple(
(item, after, _rewritten_tool_use_input(after.arguments))
for item, before, after in zip(scanned_tool_calls, pre_guardrail_tool_calls, post_guardrail_tool_calls)
if before != after
)
applicable: Final = tuple(
(item, after, rewritten_input) for item, after, rewritten_input in rewritten if rewritten_input is not None
)
if len(applicable) != len(rewritten):
raise unappliable_request_rewrite(guardrail_name)
for item, after, rewritten_input in applicable:
_write_back_tool_use(messages[item.target.msg_idx], item.target, after, rewritten_input)
async def process_output_response(
self,

7
litellm/llms/anthropic/common_utils.py
View file

@ -539,6 +539,13 @@ class AnthropicModelInfo(BaseLLMModelInfo):
value: Final = litellm.model_cost.get(model, {}).get(key)
return value if isinstance(value, bool) else None
@staticmethod
def supports_fast_mode(model: str, custom_llm_provider: str) -> bool:
return (
custom_llm_provider == "anthropic"
and AnthropicModelInfo._get_exact_model_capability(model, "supports_fast_mode") is True
)
@staticmethod
def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> bool | None:
"""Resolve boolean capability ``key`` for ``model`` under the caller's provider.

9
litellm/llms/anthropic/experimental_pass_through/context_management/dispatcher.py
View file

@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, TypeAlias
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import AppliedEdit
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
@ -82,9 +83,9 @@ async def apply_context_management(
"""Run edits in order; return a single ``PolyfillResult``.
The dispatcher is async so async editors (``compact_20260112``) can
``await`` the configured summarization model. Sync editors are called
inline — ``inspect.iscoroutinefunction`` decides how each editor is
invoked.
``await`` the configured summarization model. Sync editors run in a
worker thread so their token counts stay off the event loop;
``inspect.iscoroutinefunction`` decides how each editor is invoked.
"""
edits: Final = _normalize_spec(context_management_spec)
if not edits:
@ -121,7 +122,7 @@ async def apply_context_management(
user_api_key_auth=user_api_key_auth,
)
if editor_is_async
else editor(
else await asyncify(editor)(
model=model,
messages=current_messages,
tools=tools,

3
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -20,6 +20,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import (
AppliedEdit,
CompactionBlock,
@ -1157,7 +1158,7 @@ async def apply_compact_20260112(
# Phase B: threshold check.
try:
current_tokens = _count_effective_tokens(
current_tokens = await asyncify(_count_effective_tokens)(
model=model,
effective_messages=effective_messages,
# ``augmented_system`` already carries the prior compaction summary

2
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -678,7 +678,7 @@ class BaseAnthropicMessagesStreamingIterator:
"""
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
PassThroughStreamingHandler.schedule_stream_failure_logging(
await PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=self.litellm_logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
request_body=self.request_body,

52
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -42,6 +42,10 @@ DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
)
def _messages_carry_output_config(messages: Sequence[object]) -> bool:
return any(isinstance(message, Mapping) and "output_config" in message for message in messages)
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
@property
def custom_llm_provider(self) -> str | None:
@ -331,6 +335,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=messages,
)
return headers, api_base
@ -664,6 +669,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
headers: dict,
optional_params: dict,
custom_llm_provider: str = "anthropic",
messages: Sequence[object] = (),
) -> dict:
"""
Auto-inject anthropic-beta headers based on features used.
@ -673,24 +679,30 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
- tool_search: adds provider-specific tool search header
- output_format: adds 'structured-outputs-2025-11-13'
- speed: adds 'fast-mode-2026-02-01'
- a message carrying output_config: adds 'per-turn-control-2026-07-01'
Args:
headers: Request headers dict
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
messages: Request messages, scanned for per-message output_config
"""
beta_values: Final[set] = set()
# Get existing beta headers if any
existing_beta: Final = headers.get("anthropic-beta")
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
existing_beta: Final = tuple(
piece.strip()
for key, value in headers.items()
if key.lower() == "anthropic-beta"
for piece in value.split(",")
if piece.strip()
)
beta_values.update(existing_beta)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
@ -722,24 +734,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
# Check for advisor tool
tools = optional_params.get("tools")
if tools:
for tool in tools:
if isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
break
if _messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
# Check for tool search tools
tools = optional_params.get("tools")
if tools:
anthropic_model_info: Final = AnthropicModelInfo()
if anthropic_model_info.is_tool_search_used(tools):
# Use provider-specific tool search header
tool_search_header: Final = get_tool_search_beta_header(custom_llm_provider)
beta_values.add(tool_search_header)
tools: Final = optional_params.get("tools")
if any(isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for tool in tools or ()):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
if beta_values:
headers["anthropic-beta"] = ",".join(sorted(beta_values))
if AnthropicModelInfo().is_tool_search_used(tools):
beta_values.add(get_tool_search_beta_header(custom_llm_provider))
return headers
if not beta_values:
return headers
merged: Final = {key: value for key, value in headers.items() if key.lower() != "anthropic-beta"}
merged["anthropic-beta"] = ",".join(sorted(beta_values))
return merged

4
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
flatten_combinators_and_drop_non_python_regex_patterns,
hoist_images_from_tool_messages,
system_messages_first,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -276,7 +277,8 @@ class AzureOpenAIConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
return {
"model": model,

1
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -68,6 +68,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=messages,
)
return headers, api_base

25
litellm/llms/azure_ai/passthrough/transformation.py
View file

@ -5,7 +5,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.llms.azure_ai.common_utils import (
@ -18,6 +18,8 @@ from litellm.llms.base_llm.passthrough.transformation import (
BasePassthroughConfig,
RelayShape,
logged_relay_shape,
model_group_from,
relayed_body,
strip_leading_model_segment,
)
from litellm.types.llms.openai import AllMessageValues
@ -35,19 +37,6 @@ if TYPE_CHECKING:
EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({})
class PassthroughMetadata(BaseModel):
model_config = ConfigDict(extra="ignore")
model_group: str = ""
def model_group_from(litellm_params: Mapping[str, object]) -> str:
try:
return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group
except ValidationError:
return ""
def api_version_from(litellm_params: Mapping[str, object]) -> str | None:
try:
return TypeAdapter(str | None).validate_python(litellm_params.get("api_version"))
@ -96,14 +85,6 @@ def relay_query_params(
return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version})
def relayed_body(httpx_response: Response) -> str | dict:
try:
body: Final[object] = httpx_response.json()
except ValueError:
return httpx_response.text
return body if isinstance(body, dict) else httpx_response.text
FOUNDRY_RELAY_SHAPES: Final = (
RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate),
RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate),

68
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -1,8 +1,8 @@
from __future__ import annotations
import json
from collections.abc import Callable, Iterator, Sequence
from typing import Final, TypeVar
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel
@ -364,3 +364,67 @@ def merge_guardrailed_scoped_messages(
yield from appended
return list(_merged())
def _content_part_text(part: object) -> str | None:
if not isinstance(part, Mapping):
return None
text: Final = part.get("text")
return text if isinstance(text, str) else None
def message_slot_texts(message: Mapping[str, object]) -> tuple[str, ...]:
content: Final = message.get("content")
if isinstance(content, str):
return (content,)
if isinstance(content, list):
return tuple(text for part in content if (text := _content_part_text(part)) is not None)
return ()
def message_text_slot_count(message: AllMessageValues) -> int:
return len(message_slot_texts(message))
def _part_with_text(part: object, text: str) -> object:
if not isinstance(part, Mapping):
return part
return {**part, "text": text} # mutable-ok: content parts stay JSON-plain dicts
def _content_with_slot_texts(content: Sequence[object], texts: Sequence[str]) -> Sequence[object]:
remaining_texts: Final = iter(texts)
return [ # mutable-ok: message content stays a JSON list
_part_with_text(part, next(remaining_texts)) if _content_part_text(part) is not None else part
for part in content
]
def message_with_slot_texts(message: AllMessageValues, texts: Sequence[str]) -> AllMessageValues | None:
"""Swap one rewritten text into each text slot of a chat row, in order.
A slot is a string ``content`` or one list part carrying a string ``text``;
images and other parts ride along untouched. Returns None unless the counts
line up exactly, so a rewrite never lands on the wrong slot.
"""
if message_text_slot_count(message) != len(texts):
return None
content: Final = message.get("content")
if not isinstance(content, (str, list)):
return message
rewritten_content: Final = texts[0] if isinstance(content, str) else _content_with_slot_texts(content, texts)
rewritten: Final = {**message, "content": rewritten_content} # mutable-ok: chat rows stay JSON-plain dicts
return cast("AllMessageValues", rewritten) # cast-ok: the same row with only its text slots swapped
class UnappliableRequestRewrite(Exception):
def __init__(self, guardrail_name: str) -> None:
super().__init__(
f"Guardrail '{guardrail_name}' rewrote the request in a way this endpoint cannot apply, "
"so the request was rejected rather than sent unrewritten"
)
self.guardrail_name: Final = guardrail_name
def unappliable_request_rewrite(guardrail_name: str | None) -> UnappliableRequestRewrite:
return UnappliableRequestRewrite(guardrail_name or "unknown")

23
litellm/llms/base_llm/passthrough/transformation.py
View file

@ -6,7 +6,7 @@ from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
from pydantic import TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.types.utils import CallTypes
@ -29,6 +29,19 @@ if TYPE_CHECKING:
RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
class PassthroughMetadata(BaseModel):
model_config = ConfigDict(extra="ignore")
model_group: str = ""
def model_group_from(litellm_params: Mapping[str, object]) -> str:
try:
return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group
except ValidationError:
return ""
def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str:
path: Final = endpoint.lstrip("/")
for model_name in model_names:
@ -55,6 +68,14 @@ def relayed_json_object(httpx_response: Response) -> Mapping[str, object] | None
return None
def relayed_body(httpx_response: Response) -> str | dict:
try:
body: Final[object] = httpx_response.json()
except ValueError:
return httpx_response.text
return body if isinstance(body, dict) else httpx_response.text
@dataclass(frozen=True, slots=True)
class RelayShape:
path_suffix: str

122
litellm/llms/bedrock/base_aws_llm.py
View file

@ -11,6 +11,7 @@ from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from functools import partial
from threading import Lock
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, ParamSpec, TypeVar, cast, get_args, overload
import httpx
@ -96,6 +97,77 @@ def _assume_role_params(
)
_SecureTransportBool = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
class _SecureTransportCondition(TypedDict):
Bool: ReadOnly[_SecureTransportBool]
class _SessionPolicyStatement(TypedDict):
Sid: ReadOnly[str]
Effect: ReadOnly[Literal["Allow"]]
Action: ReadOnly[tuple[str, ...]]
Resource: ReadOnly[Literal["*"]]
Condition: ReadOnly[_SecureTransportCondition]
class WebIdentitySessionPolicy(TypedDict):
Version: ReadOnly[Literal["2012-10-17"]]
Statement: ReadOnly[tuple[_SessionPolicyStatement, ...]]
_WEB_IDENTITY_SESSION_POLICY_ACTIONS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
{
"BedrockLiteLLM": (
"bedrock:InvokeModel",
"bedrock:InvokeModelWithResponseStream",
"bedrock:CountTokens",
"bedrock:Rerank",
"bedrock:Retrieve",
"bedrock:ListKnowledgeBases",
"bedrock:InvokeAgent",
"bedrock:ApplyGuardrail",
"bedrock:GetGuardrail",
"bedrock:ListGuardrails",
),
"BedrockAgentCoreLiteLLM": (
"bedrock-agentcore:InvokeAgentRuntime",
"bedrock-agentcore:InvokeAgentRuntimeForUser",
"bedrock-agentcore:InvokeGateway",
),
"ClaudePlatformLiteLLM": (
"aws-external-anthropic:CreateInference",
"aws-external-anthropic:CreateBatchInference",
"aws-external-anthropic:CancelBatchInference",
"aws-external-anthropic:DeleteBatchInference",
"aws-external-anthropic:CountTokens",
"aws-external-anthropic:Get*",
"aws-external-anthropic:List*",
),
"BedrockMantleLiteLLM": ("bedrock-mantle:CreateInference",),
}
)
_SECURE_TRANSPORT_ONLY: Final = _SecureTransportCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
return WebIdentitySessionPolicy(
Version="2012-10-17",
Statement=tuple(
_SessionPolicyStatement(
Sid=sid,
Effect="Allow",
Action=actions,
Resource="*",
Condition=_SECURE_TRANSPORT_ONLY,
)
for sid, actions in _WEB_IDENTITY_SESSION_POLICY_ACTIONS.items()
),
)
class BedrockRequestTarget(BaseModel):
aws_region_name: str
aws_bedrock_runtime_endpoint: str | None
@ -940,60 +1012,12 @@ class BaseAWSLLM(SignsRequestsWithAWS):
# auth only (static creds + IRSA take other code paths).
# https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
bedrock_session_policy: Final = {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "BedrockLiteLLM",
"Effect": "Allow",
"Action": [
"bedrock:InvokeModel",
"bedrock:InvokeModelWithResponseStream",
"bedrock:CountTokens",
"bedrock:ApplyGuardrail",
"bedrock:GetGuardrail",
"bedrock:ListGuardrails",
],
"Resource": "*",
"Condition": {"Bool": {"aws:SecureTransport": "true"}},
},
# Claude Platform on AWS (added by #27678 for the
# ``bedrock/claude_platform/<model>`` route) lives under
# a separate IAM action namespace; without these entries
# the OIDC path 403s on every claude_platform request
# even with a fully permissive identity policy (#30200).
{
"Sid": "ClaudePlatformLiteLLM",
"Effect": "Allow",
"Action": [
"aws-external-anthropic:CreateInference",
"aws-external-anthropic:CreateBatchInference",
"aws-external-anthropic:CancelBatchInference",
"aws-external-anthropic:DeleteBatchInference",
"aws-external-anthropic:CountTokens",
"aws-external-anthropic:Get*",
"aws-external-anthropic:List*",
],
"Resource": "*",
"Condition": {"Bool": {"aws:SecureTransport": "true"}},
},
{
"Sid": "BedrockMantleLiteLLM",
"Effect": "Allow",
"Action": [
"bedrock-mantle:CreateInference",
],
"Resource": "*",
"Condition": {"Bool": {"aws:SecureTransport": "true"}},
},
],
}
assume_role_params: Final = {
"RoleArn": aws_role_name,
"RoleSessionName": aws_session_name,
"WebIdentityToken": oidc_token,
"DurationSeconds": 3600,
"Policy": json.dumps(bedrock_session_policy, separators=(",", ":")),
"Policy": json.dumps(build_web_identity_session_policy(), separators=(",", ":")),
}
# Add ExternalId parameter if provided

1
litellm/llms/bedrock/claude_platform/messages_transformation.py
View file

@ -46,6 +46,7 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=messages,
)
return headers, api_base

63
litellm/llms/bedrock_mantle/responses/transformation.py
View file

@ -17,7 +17,7 @@ BaseAWSLLM._sign_request after the request body is finalized.
import json
from collections.abc import Mapping
from typing import Any, Final
from typing import Any, Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict
import httpx
from typing_extensions import ReadOnly, TypedDict
@ -32,6 +32,7 @@ from litellm.llms.bedrock_mantle.common_utils import (
BedrockMantleAuthMixin,
)
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
ResponseInputParam,
@ -58,8 +59,6 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset(
_BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
_BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
@ -233,17 +232,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
normalized_input: Final = self._normalize_codex_input_items(remaining_input)
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
)
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
normalized_input: Final = self._normalize_codex_input_items(hoisted.input)
request_params: Final = (
{
**response_api_optional_request_params,
"tools": [
*(response_api_optional_request_params.get("tools") or []),
*hoisted_tools,
],
}
if hoisted_tools
self._params_with_hoisted_tools(params, hoisted)
if hoisted.hoisted
else response_api_optional_request_params
)
return super().transform_responses_api_request(
@ -254,41 +250,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
headers=headers,
)
@staticmethod
def _is_codex_additional_tools_item(item: Any) -> bool:
return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
@staticmethod
def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]":
tools: Final = item.get("tools")
return tools if isinstance(tools, list) else []
@classmethod
def _hoist_codex_additional_tools(
cls,
input: "str | ResponseInputParam",
) -> "tuple[str | ResponseInputParam, list[Any]]":
"""Codex's "responses lite" wire mode ships tool definitions inside
`input` as {"type": "additional_tools", "role": "developer",
"tools": [...]} items. api.openai.com accepts that item type; Mantle
rejects the whole request with 400 "Invalid 'input': value did not
match any expected variant" but accepts the same tools at the top
level, so move them there and strip the items from `input`.
"""
if not isinstance(input, list):
return input, []
additional_tools_items: Final = [item for item in input if cls._is_codex_additional_tools_item(item)]
if not additional_tools_items:
return input, []
remaining_input: Final = [item for item in input if not cls._is_codex_additional_tools_item(item)]
hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)]
verbose_logger.debug(
"Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) "
"into the top-level tools param (Mantle rejects that input item type).",
len(hoisted_tools),
len(additional_tools_items),
)
return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
def _params_with_hoisted_tools(
cls, params: Mapping[str, object], hoisted: HoistedAdditionalTools
) -> dict[str, object]:
supported_tools: Final = cls._filter_unsupported_tools(list(hoisted.tools))
if supported_tools:
return {**params, "tools": supported_tools}
return {key: value for key, value in params.items() if key != "tools"}
@staticmethod
def _agent_message_text(item: "Mapping[str, object]") -> str:

1
litellm/llms/deepseek/messages/transformation.py
View file

@ -66,6 +66,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
headers=headers,
optional_params=optional_params,
custom_llm_provider=self.custom_llm_provider or "deepseek",
messages=messages,
)
return headers, api_base

3
litellm/llms/fireworks_ai/rerank/transformation.py
View file

@ -250,8 +250,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig):
rerank_results.append(rerank_result)
# Use model name as id if no id is provided
response_id: Final = raw_response_json.get("id") or raw_response_json.get("model") or str(uuid.uuid4())
response_id: Final = raw_response_json.get("id") or str(uuid.uuid4())
return RerankResponse(
id=response_id,

20
litellm/llms/gemini/realtime/transformation.py
View file

@ -115,6 +115,19 @@ def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup
return envelope.get("setup", empty_setup)
def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
"""Read ``serverContent.groundingMetadata`` off the frame that carries the turn's usage.
Live reports grounding in the server frames rather than in ``usageMetadata``, and it emits both
on the same frame, so the per-query charge is countable at the point usage is built.
"""
server_content: Final = frame.get("serverContent")
if not isinstance(server_content, Mapping):
return ()
metadata: Final = server_content.get("groundingMetadata")
return (metadata,) if isinstance(metadata, Mapping) else ()
# Google bills Live transcription at an estimated 25 audio tokens/sec of input and
# 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
@ -323,7 +336,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
elif key == "turn_detection":
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (
isinstance(value_typed, dict)
@ -1049,6 +1062,11 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
{**cast(dict, message), "usageMetadata": resolved_usage_metadata},
),
)
grounding_metadata: Final = _grounding_metadata_from_frame(message)
if grounding_metadata:
VertexGeminiConfig._set_grounding_usage_counters( # pyright: ignore[reportPrivateUsage] # shared with the chat path; no public alias exists yet
_chat_completion_usage, grounding_metadata
)
else:
_chat_completion_usage = get_empty_usage()

4
litellm/llms/gemini/videos/transformation.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
from litellm.images.utils import ImageEditRequestUtils
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.vertex_ai.videos.transformation import veo_video_count_from_parameters
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.gemini import (
GeminiLongRunningOperationResponse,
@ -354,6 +355,9 @@ class GeminiVideoConfig(BaseVideoConfig):
video_resolution: Final = _usage_video_resolution_from_parameters(parameters)
if video_resolution is not None:
usage_data["video_resolution"] = video_resolution
video_count: Final = veo_video_count_from_parameters(parameters)
if video_count is not None:
usage_data["video_count"] = video_count
video_obj.usage = usage_data
return video_obj

2
litellm/llms/github_copilot/messages/transformation.py
View file

@ -92,7 +92,7 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig):
headers["anthropic-version"] = "2023-06-01"
headers = self._update_headers_with_anthropic_beta(
headers, optional_params, custom_llm_provider="github_copilot"
headers, optional_params, custom_llm_provider="github_copilot", messages=messages
)
return headers, dynamic_api_base

0
litellm/llms/nvidia_nim/passthrough/__init__.py Normal file
View file

139
litellm/llms/nvidia_nim/passthrough/transformation.py Normal file
View file

@ -0,0 +1,139 @@
from __future__ import annotations
import re
from collections.abc import Collection, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
import httpx
from litellm.llms.base_llm.passthrough.transformation import (
BasePassthroughConfig,
model_group_from,
relayed_body,
strip_leading_model_segment,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import DeploymentTypedDict
from litellm.types.utils import LlmProviders, StandardPassThroughResponseObject
if TYPE_CHECKING:
from httpx import URL, Response
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse
API_VERSION_SEGMENT: Final = re.compile(r"^v\d+$")
NVIDIA_NIM_MODEL_PREFIX: Final = f"{LlmProviders.NVIDIA_NIM.value}/"
NVIDIA_NIM_ROUTE_PREFIX: Final = re.compile(rf"^/{LlmProviders.NVIDIA_NIM.value}/", re.IGNORECASE)
def is_nvidia_nim_deployment(deployment: DeploymentTypedDict) -> bool:
litellm_params: Final = deployment["litellm_params"]
return litellm_params.get("custom_llm_provider") == LlmProviders.NVIDIA_NIM.value or litellm_params.get(
"model", ""
).startswith(NVIDIA_NIM_MODEL_PREFIX)
def nvidia_nim_model_groups(deployments: Iterable[DeploymentTypedDict] | None) -> frozenset[str]:
listed: Final = tuple(deployments or ())
nim_groups: Final = frozenset(d["model_name"] for d in listed if is_nvidia_nim_deployment(d))
other_groups: Final = frozenset(d["model_name"] for d in listed if not is_nvidia_nim_deployment(d))
return nim_groups - other_groups
def nvidia_nim_model_group_in_path(path: str, deployments: Iterable[DeploymentTypedDict] | None) -> str | None:
return nvidia_nim_router_model_in_endpoint(
NVIDIA_NIM_ROUTE_PREFIX.sub("", path), nvidia_nim_model_groups(deployments)
)
def nvidia_nim_router_model_in_endpoint(endpoint: str, router_models: Collection[str]) -> str | None:
segments: Final = tuple(segment for segment in endpoint.split("/") if segment)
return next(
(
"/".join(segments[:length])
for length in range(len(segments), 0, -1)
if "/".join(segments[:length]) in router_models
),
None,
)
def without_repeated_version_prefix(api_base: str, native_endpoint: str) -> str:
url: Final = httpx.URL(api_base)
base_segments: Final = tuple(segment for segment in url.path.split("/") if segment)
first_native_segment: Final = native_endpoint.lstrip("/").split("/", 1)[0]
repeated: Final = (
bool(base_segments)
and API_VERSION_SEGMENT.match(first_native_segment) is not None
and base_segments[-1] == first_native_segment
)
kept_segments: Final = base_segments[:-1] if repeated else base_segments
return str(url.copy_with(path="/" + "/".join(kept_segments), query=None)).rstrip("/")
class NvidiaNimPassthroughConfig(BasePassthroughConfig):
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return bool(request_data.get("stream", False))
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
endpoint: str,
request_query_params: dict | None,
litellm_params: dict,
) -> tuple[URL, str]:
base_target_url: Final = self.get_api_base(api_base)
if base_target_url is None:
raise ValueError("NVIDIA NIM api base not found: set `api_base` on the deployment or NVIDIA_NIM_API_BASE")
native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params)))
root: Final = without_repeated_version_prefix(base_target_url, native_endpoint)
return (self.format_url(native_endpoint, root, request_query_params), root)
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: base class contract returns dict for httpx
if api_key is None:
return dict(headers) # mutable-ok: base class contract returns dict for httpx
return {
**headers,
"Authorization": f"Bearer {api_key}",
} # mutable-ok: base class contract returns dict for httpx
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
return api_base or get_secret_str("NVIDIA_NIM_API_BASE")
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return api_key or get_secret_str("NVIDIA_NIM_API_KEY")
@staticmethod
def get_base_model(model: str) -> str | None:
return model
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
return []
def logging_non_streaming_response(
self,
model: str,
custom_llm_provider: str,
httpx_response: Response,
request_data: Mapping[str, object],
logging_obj: Logging,
endpoint: str,
) -> LoggedRelayResponse | OCRResponse | StandardPassThroughResponseObject | None:
return StandardPassThroughResponseObject(response=relayed_body(httpx_response))

19
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -12,6 +12,7 @@ from urllib.parse import urlparse
import httpx
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_extract_reasoning_content,
@ -24,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
flatten_combinators_and_drop_non_python_regex_patterns,
get_tool_call_names,
hoist_images_from_tool_messages,
system_messages_first,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -463,6 +465,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
]
return MappingProxyType({"tools": sanitized})
def _prompt_cache_ordered_messages(
self, messages: list[AllMessageValues], litellm_params: Mapping[str, object]
) -> list[AllMessageValues]:
if not litellm.openai_system_messages_first:
return messages
if litellm_params.get("custom_llm_provider") not in OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS:
return messages
return system_messages_first(messages)
def transform_request(
self,
model: str,
@ -477,7 +488,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
Returns:
dict: The transformed request. Sent as the body of the API call.
"""
messages = self._transform_messages(messages=messages, model=model)
messages = self._transform_messages(
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model
)
if not self._should_preserve_cache_control_for_endpoint(
litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
):
@ -506,7 +519,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
transformed_messages = await self._transform_messages(messages=messages, model=model, is_async=True)
transformed_messages = await self._transform_messages(
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
)
if not self._should_preserve_cache_control_for_endpoint(
litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
):

36
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -42,6 +42,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_field,
stream_item_fingerprint,
stream_item_items,
unappliable_request_rewrite,
)
from litellm.main import stream_chunk_builder
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@ -196,6 +197,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
else:
# Step 3: Map guardrail responses back to original message structure
if guardrailed_texts and texts_to_check:
if len(guardrailed_texts) != len(text_task_mappings):
raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
await self._apply_guardrail_responses_to_input_texts(
messages=messages,
responses=guardrailed_texts,
@ -210,6 +213,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
task_mappings=tool_call_task_mappings,
)
elif (
not images_to_check
and not guardrail_to_apply.records_own_guardrail_information
and (not_run_reason := self._not_run_reason(messages)) is not None
):
guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data(
guardrail_json_response=not_run_reason,
request_data=data,
guardrail_status="not_run",
)
verbose_proxy_logger.debug(
"OpenAI Chat Completions: Processed input messages: %s",
data.get("messages"),
@ -217,6 +231,28 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return data
def _not_run_reason(
self,
messages: Sequence[dict[str, Any]], # mutable-ok: raw request messages consumed by _extract_inputs
) -> str | None:
"""Why nothing was scanned, or None when the only unscoped content is images, which this handler never scans."""
texts: Final[list[str]] = [] # mutable-ok: filled by _extract_inputs
images: Final[list[str]] = [] # mutable-ok: filled by _extract_inputs
tool_calls: Final[list[ChatCompletionToolParam]] = [] # mutable-ok: filled by _extract_inputs
for msg_idx, message in enumerate(messages):
self._extract_inputs(
message=message,
msg_idx=msg_idx,
texts_to_check=texts,
images_to_check=images,
tool_calls_to_check=tool_calls,
text_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here
tool_call_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here
)
if texts or tool_calls:
return "no scannable content after message scoping"
return None if images else "no scannable content"
def extract_request_tool_names(self, data: dict) -> list[str]:
"""Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name)."""
names: Final[list[str]] = []

7
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -56,6 +56,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_field,
stream_item_fingerprint,
stream_item_items,
unappliable_request_rewrite,
)
from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
from litellm.responses.litellm_completion_transformation.transformation import (
@ -495,13 +496,13 @@ class OpenAIResponsesHandler(BaseTranslation):
data["instructions"] = written_back.instructions # rebind-ok: data is an out-param
elif isinstance(input_data, str):
guardrailed_texts: Final = guardrailed_inputs.get("texts") or ()
if len(guardrailed_texts) > 1:
raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data # rebind-ok: data is an out-param
else:
rewritten_texts: Final = guardrailed_inputs.get("texts") or ()
if len(rewritten_texts) != len(extracted.task_mappings):
from litellm.proxy.policy_engine.pipeline_executor import UnappliableRequestRewrite
raise UnappliableRequestRewrite(guardrail_to_apply.guardrail_name or "unknown")
raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
await self._apply_guardrail_responses_to_input(
messages=input_data,
responses=rewritten_texts,

31
litellm/llms/openai/responses/guardrail_translation/tool_merge.py
View file

@ -6,8 +6,10 @@ from typing import Final, TypeAlias
from pydantic import BaseModel, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
from litellm.responses.litellm_completion_transformation.transformation import (
NAMESPACE_DESCRIPTION_SEPARATOR,
NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS,
LiteLLMCompletionResponsesConfig,
)
@ -34,8 +36,8 @@ def _validated_tools(values: Iterable[object]) -> tuple[Tool, ...]:
return tuple(tool for tool in validated if tool is not None)
def _is_function(tool: Tool) -> bool:
return tool.get("type") == "function"
def _has_chat_tool(member: Tool) -> bool:
return member.get("type") in NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS
def _chat_tool_key(tool: Tool) -> str:
@ -67,18 +69,19 @@ def _function_fields(tool: Tool) -> Tool:
return function if function is not None else MappingProxyType({})
def _without_namespace_prefix(key: str, value: object, prefix: str) -> object:
if key != "description" or not isinstance(value, str) or not value.startswith(prefix):
def _member_description(key: str, value: object, prefix: str, suffix: str) -> object:
if key != "description" or not isinstance(value, str):
return value
return value[len(prefix) :]
return value.replace(prefix, "", 1).replace(suffix, "", 1)
def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_description: str) -> Tool:
flattened_function: Final = _function_fields(flattened)
prefix: Final = f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}" if namespace_description else ""
suffix: Final = custom_tool_grammar_suffix(member.get("format")) if member.get("type") == "custom" else ""
changed_function: Final = MappingProxyType(
{
key: _without_namespace_prefix(key, value, prefix)
key: _member_description(key, value, prefix, suffix)
for key, value in _function_fields(guardrailed).items()
if flattened_function.get(key) != value
}
@ -93,8 +96,8 @@ def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_
return {**member, **changed_extras, **changed_function} # mutable-ok: json.dumps rejects MappingProxyType
def _rebuilt_function_members(
function_members: Sequence[Tool],
def _rebuilt_flattened_members(
flattened_members: Sequence[Tool],
flattened_group: Sequence[Tool],
group_keys: Sequence[IndexedKey],
guardrailed_by_key: Mapping[IndexedKey, Tool],
@ -106,7 +109,7 @@ def _rebuilt_function_members(
else member
if guardrailed_by_key[key] == flattened
else _rebuilt_member(member, flattened, guardrailed_by_key[key], namespace_description)
for member, flattened, key in zip(function_members, flattened_group, group_keys)
for member, flattened, key in zip(flattened_members, flattened_group, group_keys)
)
@ -118,9 +121,9 @@ def _rebuilt_namespace(
guardrailed_by_key: Mapping[IndexedKey, Tool],
) -> tuple[Tool, ...]:
namespace_description: Final = str(original.get("description") or "")
rebuilt_functions: Final = iter(
_rebuilt_function_members(
tuple(member for member in members if _is_function(member)),
rebuilt_flattened: Final = iter(
_rebuilt_flattened_members(
tuple(member for member in members if _has_chat_tool(member)),
flattened_group,
group_keys,
guardrailed_by_key,
@ -129,7 +132,7 @@ def _rebuilt_namespace(
)
rebuilt_members: Final = tuple(
rebuilt
for rebuilt in (next(rebuilt_functions) if _is_function(member) else member for member in members)
for rebuilt in (next(rebuilt_flattened) if _has_chat_tool(member) else member for member in members)
if rebuilt is not None
)
if not rebuilt_members:
@ -149,7 +152,7 @@ def _merged_original(
if guardrailed_group == tuple(flattened_group):
return (original,)
members: Final = _namespace_members(original) if original.get("type") == "namespace" else ()
if members and sum(map(_is_function, members)) == len(flattened_group):
if members and sum(map(_has_chat_tool, members)) == len(flattened_group):
return _rebuilt_namespace(original, members, flattened_group, group_keys, guardrailed_by_key)
if not guardrailed_group:
return ()

1
litellm/llms/openai_like/messages/transformation.py
View file

@ -56,6 +56,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
merged: Final = self._update_headers_with_anthropic_beta(
headers=normalized,
optional_params=optional_params,
messages=messages,
)
return merged, api_base

32
litellm/llms/vertex_ai/batches/transformation.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any, Final
from urllib.parse import unquote
@ -8,7 +9,36 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
from litellm.types.llms.vertex_ai import *
from litellm.types.utils import LiteLLMBatch
from litellm.types.utils import LiteLLMBatch, PromptTokensDetailsWrapper
def vertex_prompt_tokens_details(
usage_metadata: Mapping[str, object],
) -> PromptTokensDetailsWrapper | None:
raw_details: Final = usage_metadata.get("promptTokensDetails")
if not isinstance(raw_details, list):
return None
def _normalize(detail: object) -> tuple[str, int] | None:
if not isinstance(detail, Mapping):
return None
modality: Final = detail.get("modality")
token_count: Final = detail.get("tokenCount")
if not isinstance(modality, str) or not isinstance(token_count, int):
return None
return modality.upper(), token_count
parsed_details: Final = tuple(_normalize(detail) for detail in raw_details)
normalized: Final = tuple(detail for detail in parsed_details if detail is not None)
if len(normalized) != len(parsed_details):
return None
return PromptTokensDetailsWrapper(
text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")),
audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"),
image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"),
video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"),
)
class VertexAIBatchTransformation:

41
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py
View file

@ -298,8 +298,6 @@ def transform_openai_input_gemini_embed_content(
_IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"})
_VIDEO_TOKENS_PER_SECOND: Final = 258.0
_AUDIO_TOKENS_PER_SECOND: Final = 32.0
_usage_metadata_adapter: Final = TypeAdapter(UsageMetadata)
@ -339,11 +337,12 @@ def _is_image_element(
return False
def _count_input_images(
def _is_image_only_input(
input: GeminiEmbeddingInput,
resolved_files: Mapping[str, Mapping[str, str]],
) -> int:
return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files))
) -> bool:
elements: Final = _flatten_input(input)
return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements)
def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int:
@ -372,30 +371,29 @@ def _usage_from_embed_content_response(
total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens
details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
if not details:
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0,
image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0,
),
)
text_tokens: Final = _tokens_for_modality(details, "TEXT")
audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
image_tokens: Final = _tokens_for_modality(details, "IMAGE")
video_tokens: Final = _tokens_for_modality(details, "VIDEO")
image_count: Final = _count_input_images(input, resolved_files)
video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
# generic_cost_per_token rewrites text_tokens to the full prompt minus
# other modalities when both text_tokens and image_count are zero. For
# video, that misallocates video tokens to text; a 1-token floor sidesteps
# the rewrite and keeps billing on input_cost_per_video_per_second.
needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=resolved_text_tokens,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
image_count=image_count,
video_length_seconds=video_length_seconds,
audio_length_seconds=audio_length_seconds,
image_tokens=image_tokens,
video_tokens=video_tokens,
),
)
@ -415,8 +413,7 @@ def process_embed_content_response(
model_response: EmbeddingResponse to populate
model: Model name
response_json: Raw JSON response from embedContent endpoint
resolved_files: Mapping of file references (files/abc) to {mime_type, uri},
used to bill resolved image references at the per-image rate
resolved_files: Mapping of file references to resolved metadata
Returns:
EmbeddingResponse with single embedding

11
litellm/llms/vertex_ai/rerank/transformation.py
View file

@ -4,6 +4,8 @@ Translates from Cohere's `/v1/rerank` input format to Vertex AI Discovery Engine
Why separate file? Make it easy to see how transformation works
"""
import math
import uuid
from collections.abc import Mapping
from typing import Any, Final
@ -32,6 +34,8 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
Reference: https://cloud.google.com/generative-ai-app-builder/docs/ranking#rank_or_rerank_a_set_of_records_according_to_a_query
"""
MAX_RECORDS_PER_SEARCH_UNIT = 100
def __init__(self) -> None:
super().__init__()
@ -208,10 +212,11 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
RerankResponseResult(index=result["index"], relevance_score=result["relevance_score"])
)
# Create meta object
meta: Final = RerankResponseMeta(billed_units=RerankBilledUnits(search_units=len(records)))
input_record_count: Final = len(request_data.get("records", ()))
search_units: Final = math.ceil(input_record_count / self.MAX_RECORDS_PER_SEARCH_UNIT)
meta: Final = RerankResponseMeta(billed_units=RerankBilledUnits(search_units=search_units))
return RerankResponse(id=f"vertex_ai_rerank_{model}", results=rerank_results, meta=meta)
return RerankResponse(id=f"vertex_ai_rerank_{uuid.uuid4()}", results=rerank_results, meta=meta)
def get_supported_cohere_rerank_params(self, model: str) -> list:
return [

12
litellm/llms/vertex_ai/videos/transformation.py
View file

@ -70,10 +70,17 @@ def _parse_veo_operation(raw_response: httpx.Response) -> _VeoOperation:
return operation
def veo_video_count_from_parameters(parameters: Mapping[str, object]) -> int | None:
sample_count: Final = parameters.get("sampleCount")
if isinstance(sample_count, bool) or not isinstance(sample_count, int) or sample_count < 1:
return None
return sample_count
def _build_vertex_video_usage_from_request_data(
request_data: dict[str, Any] | None,
) -> dict[str, float | str]:
"""Build usage metadata (duration, resolution) for video cost calculation."""
"""Build usage metadata (duration, resolution, video count) for video cost calculation."""
usage_data: Final[dict[str, float | str]] = {}
if not request_data:
return usage_data
@ -88,6 +95,9 @@ def _build_vertex_video_usage_from_request_data(
res: Final = parameters.get("resolution")
if res is not None and str(res).strip() != "":
usage_data["video_resolution"] = str(res).strip().lower()
video_count: Final = veo_video_count_from_parameters(parameters)
if video_count is not None:
usage_data["video_count"] = video_count
return usage_data

3
litellm/llms/voyage/rerank/transformation.py
View file

@ -9,6 +9,7 @@ from typing import Any, Final
import httpx
from litellm._uuid import uuid
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.secret_managers.main import get_secret_str
@ -127,7 +128,7 @@ class VoyageRerankConfig(BaseRerankConfig):
rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)
return RerankResponse(
id=_json_response.get("id", f"voyage-rerank-{model}"),
id=_json_response.get("id") or str(uuid.uuid4()),
results=transformed_results,
meta=rerank_meta,
)

2
litellm/llms/watsonx/rerank/transformation.py
View file

@ -191,7 +191,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig):
transformed_results.append(transformed_result)
response_id: Final = raw_response_json.get("id") or raw_response_json.get("model_id") or str(uuid.uuid4())
response_id: Final = raw_response_json.get("id") or str(uuid.uuid4())
# Extract usage information
_tokens: Final = RerankTokens(

18
litellm/llms/xai/chat/transformation.py
View file

@ -219,13 +219,19 @@ class XAIChatConfig(OpenAIGPTConfig):
litellm_params: dict,
headers: dict,
) -> dict:
"""
Handle https://github.com/BerriAI/litellm/issues/9720
"""Handle https://github.com/BerriAI/litellm/issues/9720"""
if "web_search_options" in optional_params:
verbose_logger.warning(
"XAI no longer supports web search on /chat/completions (Live Search is deprecated). "
"Dropping 'web_search_options'. Use the Responses API for XAI web search."
)
Filter out 'name' from messages
"""
messages = strip_name_from_messages(messages)
return super().transform_request(model, messages, optional_params, litellm_params, headers)
chat_params: Final = { # mutable-ok: base transform_request takes a plain dict of optional params
key: value for key, value in optional_params.items() if key != "web_search_options"
}
return super().transform_request(
model, strip_name_from_messages(messages), chat_params, litellm_params, headers
)
@staticmethod
def _fix_choice_finish_reason_for_tool_calls(choice: Choices) -> None:

22
litellm/llms/xai/responses/transformation.py
View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -32,6 +33,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_STR_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _usage_restated_from_xai_ticks(usage: ResponseAPIUsage | None) -> ResponseAPIUsage | None:
reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None))
@ -81,30 +84,25 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- enable_image_understanding
XAI does NOT support search_context_size (OpenAI-specific).
Domains may come nested under 'filters' (the OpenAI/XAI documented shape) or flat on the tool.
"""
xai_tool: Final[dict[str, object]] = {"type": "web_search"}
# Remove search_context_size if present (not supported by XAI)
if "search_context_size" in tool:
verbose_logger.info(
"XAI does not support 'search_context_size' parameter. Removing it from web_search tool."
)
# Handle filters (XAI-specific structure)
filters: Final = {}
if "allowed_domains" in tool:
allowed_domains: Final = tool["allowed_domains"]
filters["allowed_domains"] = allowed_domains
nested_filters: Final = tool.get("filters")
domains: Final = (
_STR_MAPPING_ADAPTER.validate_python(nested_filters) if isinstance(nested_filters, Mapping) else tool
)
filters: Final = {key: domains[key] for key in ("allowed_domains", "excluded_domains") if key in domains}
if "excluded_domains" in tool:
excluded_domains: Final = tool["excluded_domains"]
filters["excluded_domains"] = excluded_domains
# Add filters if any were specified
if filters:
xai_tool["filters"] = filters
# Handle enable_image_understanding (top-level in XAI format)
if "enable_image_understanding" in tool:
xai_tool["enable_image_understanding"] = tool["enable_image_understanding"]

12
litellm/main.py
View file

@ -67,7 +67,7 @@ from litellm.constants import (
)
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
from litellm.litellm_core_utils.audio_utils.utils import (
calculate_request_duration,
get_audio_file_for_health_check,
@ -1072,10 +1072,6 @@ def responses_api_bridge_check(
mode = "responses"
model_info["mode"] = mode
if web_search_options is not None and custom_llm_provider == "xai":
model_info["mode"] = "responses"
model = model.replace("responses/", "")
except Exception as e:
verbose_logger.debug("Error getting model info: %s", e)
@ -1084,6 +1080,10 @@ def responses_api_bridge_check(
mode = "responses"
model_info["mode"] = mode
if web_search_options is not None and custom_llm_provider == "xai":
model_info["mode"] = "responses"
model = model.replace("responses/", "")
# OpenAI/Azure GPT-5 chat-completions that need Responses-only fields (e.g.
# ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects
# those keys.
@ -9127,7 +9127,7 @@ async def acount_tokens(
fallback_messages = messages or []
if system and fallback_messages:
fallback_messages = [{"role": "system", "content": system}] + fallback_messages
local_count: Final = litellm.token_counter(
local_count: Final = await asyncify(litellm.token_counter)(
model=model,
messages=fallback_messages,
tools=tools,

1198
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

1
litellm/models/budget.py
View file

@ -26,6 +26,7 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
max_parallel_requests: int | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
model_max_budget: dict | None = None
budget_duration: str | None = None
allowed_models: list[str] | None = None # per-member model scope; empty = inherit team models

9
litellm/models/credentials.py
View file

@ -5,6 +5,8 @@ These are the canonical credential types for the proxy. They live in the model
layer; ``litellm.types.utils`` re-exports them for backwards compatibility.
"""
from collections.abc import Mapping
from pydantic import BaseModel, model_validator
@ -27,3 +29,10 @@ class CreateCredentialItem(CredentialBase):
if not values.get("credential_values") and not values.get("model_id"):
raise ValueError("Either credential_values or model_id must be set")
return values
class UpdateCredentialItem(BaseModel):
credential_name: str
credential_info: Mapping[str, object]
credential_values: Mapping[str, object] | None = None
model_id: str | None = None

1
litellm/models/team.py
View file

@ -71,6 +71,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
budget_duration: str | None = None

1
litellm/models/verification_token.py
View file

@ -31,6 +31,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
metadata: dict = {}
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
allowed_cache_controls: list | None = []

1
litellm/proxy/_lazy_features.py
View file

@ -205,6 +205,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/gigachat/",
"/milvus/",
"/mistral/",
"/nvidia_nim/",
"/openai/",
"/openai_passthrough/",
"/vertex-ai/",

300
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -11539,6 +11539,12 @@
"description": "Enable content moderation to check for harmful content (harassment, hate speech, etc.).",
"title": "Content Moderation Check"
},
"contextual_grounding_from_messages": {
"default": false,
"description": "ApplyGuardrail: when True, post-call scans of a request with no grounding_source / query content parts send the system and developer messages as the grounding source and the latest user message as the query, so the guardrail's contextual grounding policy can score the response. Bedrock bills contextual grounding units for these scans and rejects queries, sources and responses over its contextual grounding length limits, so leave this off for guardrails without a contextual grounding policy. Default False: plain messages are never sent as grounding context.",
"title": "Contextual Grounding From Messages",
"type": "boolean"
},
"credentials": {
"anyOf": [
{
@ -13022,18 +13028,24 @@
"PHONE_NUMBER",
"MEDICAL_LICENSE",
"URL",
"MAC_ADDRESS",
"UUID",
"US_BANK_NUMBER",
"US_DRIVER_LICENSE",
"US_ITIN",
"US_PASSPORT",
"US_SSN",
"US_MBI",
"US_NPI",
"UK_NHS",
"UK_NINO",
"UK_PASSPORT",
"UK_POSTCODE",
"UK_VEHICLE_REGISTRATION",
"UK_DRIVING_LICENCE",
"ES_NIF",
"ES_NIE",
"ES_PASSPORT",
"IT_FISCAL_CODE",
"IT_DRIVER_LICENSE",
"IT_VAT_CODE",
@ -13051,7 +13063,38 @@
"IN_VEHICLE_REGISTRATION",
"IN_VOTER",
"IN_PASSPORT",
"FI_PERSONAL_IDENTITY_CODE"
"IN_GSTIN",
"FI_PERSONAL_IDENTITY_CODE",
"DE_TAX_ID",
"DE_TAX_NUMBER",
"DE_VAT_ID",
"DE_PASSPORT",
"DE_ID_CARD",
"DE_FUEHRERSCHEIN",
"DE_SOCIAL_SECURITY",
"DE_HEALTH_INSURANCE",
"DE_LANR",
"DE_BSNR",
"DE_KFZ",
"DE_HANDELSREGISTER",
"DE_PLZ",
"KR_RRN",
"KR_FRN",
"KR_PASSPORT",
"KR_DRIVER_LICENSE",
"KR_BRN",
"CA_SIN",
"SE_PERSONNUMMER",
"SE_ORGANISATIONSNUMMER",
"TH_TNIN",
"TR_NATIONAL_ID",
"TR_LICENSE_PLATE",
"NG_NIN",
"NG_VEHICLE_REGISTRATION",
"PH_TIN",
"PH_UMID",
"PH_PASSPORT",
"ZA_ID_NUMBER"
],
"title": "PiiEntityType",
"type": "string"
@ -15243,6 +15286,17 @@
"title": "Jwt Claim Value",
"type": "string"
},
"jwt_issuer": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Jwt Issuer"
},
"key": {
"title": "Key",
"type": "string"
@ -15327,6 +15381,17 @@
"title": "Jwt Claim Value",
"type": "string"
},
"jwt_issuer": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Jwt Issuer"
},
"updated_at": {
"format": "date-time",
"title": "Updated At",
@ -15383,6 +15448,17 @@
],
"title": "Is Active"
},
"jwt_issuer": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Jwt Issuer"
},
"key": {
"anyOf": [
{
@ -18929,6 +19005,228 @@
]
}
},
"/nvidia_nim/{endpoint}": {
"delete": {
"description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
"operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__delete",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Nvidia Nim Proxy Route",
"tags": [
"llm_passthrough"
]
},
"get": {
"description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
"operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__get",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Nvidia Nim Proxy Route",
"tags": [
"llm_passthrough"
]
},
"patch": {
"description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
"operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__patch",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Nvidia Nim Proxy Route",
"tags": [
"llm_passthrough"
]
},
"post": {
"description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
"operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__post",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Nvidia Nim Proxy Route",
"tags": [
"llm_passthrough"
]
},
"put": {
"description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
"operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__put",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Nvidia Nim Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/openai/deployments/{model}/chat/completions": {
"post": {
"description": "Follows the exact same API spec as `OpenAI's Chat API https://platform.openai.com/docs/api-reference/chat`\n\n```bash\ncurl -X POST http://localhost:4000/v1/chat/completions \n-H \"Content-Type: application/json\" \n-H \"Authorization: Bearer sk-1234\" \n-d '{\n \"model\": \"gpt-4o\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Hello!\"\n }\n ]\n}'\n```",

41
litellm/proxy/_types.py
View file

@ -4,11 +4,12 @@ import os
from collections.abc import Callable, Mapping
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple
import httpx
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
Json,
@ -47,6 +48,7 @@ from litellm.types.proxy.carried_budget_state import (
)
from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry
from litellm.types.router import RouterErrors, UpdateRouterConfig
from litellm.types.router_weights import validate_router_settings_dict
from litellm.types.secret_managers.main import KeyManagementSystem
from litellm.types.utils import (
CallTypes,
@ -244,6 +246,7 @@ class Litellm_EntityType(enum.Enum):
TEAM = "team"
TEAM_MEMBER = "team_member"
ORGANIZATION = "organization"
ORGANIZATION_MEMBER = "organization_member"
PROJECT = "project"
TAG = "tag"
AGENT = "agent"
@ -284,6 +287,7 @@ class KeyManagementRoutes(str, enum.Enum):
# team's `team_member_permissions`, non-admin members of that team may set
# `access_group_ids` on keys they create/update. Default-deny.
KEY_ACCESS_GROUP_ASSIGNMENT = "/key/access_group_assignment"
AUTO_ROUTER_MANAGE = "/auto_router/manage"
# info and health routes
KEY_INFO = "/key/info"
@ -482,6 +486,7 @@ class LiteLLMRoutes(enum.Enum):
"/milvus",
"/gigachat",
"/watsonx",
"/nvidia_nim",
]
#########################################################
@ -650,15 +655,18 @@ class LiteLLMRoutes(enum.Enum):
KeyManagementRoutes.KEY_RESET_SPEND.value,
KeyManagementRoutes.KEY_ALIASES.value,
KeyManagementRoutes.KEY_ACCESS_GROUP_ASSIGNMENT.value,
KeyManagementRoutes.AUTO_ROUTER_MANAGE.value,
]
management_routes = (
[
# user
"/user/new",
"/management/v1/users/bulk",
"/user/update",
"/user/bulk_update",
"/user/delete",
"/management/v1/users/bulk_delete",
"/user/info",
"/user/list",
"/user/daily/activity",
@ -838,6 +846,7 @@ class LiteLLMRoutes(enum.Enum):
self_managed_routes = [
"/team/member_add",
"/team/member_delete",
"/management/v1/teams/{team_id}/members/bulk_delete",
"/team/member_update",
"/team/{team_id}/member/{user_id}/reset_spend",
"/team/permissions_list",
@ -864,6 +873,7 @@ class LiteLLMRoutes(enum.Enum):
"/organization/daily/activity",
"/user/available_roles", # read-only role metadata; any authenticated user may read
"/user/list", # org admins checked in endpoint; non-admins get 403
"/management/v1/users/bulk_delete", # proxy admins delete anyone, org admins only their orgs' users; others 403
"/model/{model_id}/update",
"/prompt/list",
"/prompt/info",
@ -1198,6 +1208,7 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
class KeyRequestBase(GenerateRequestBase):
key: str | None = None
tpd_limit: int | None = None
default_estimated_output_tokens: PositiveInt | None = None
default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None
budget_id: str | None = None
@ -1883,6 +1894,9 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
)
tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.")
rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.")
tpd_limit: int | None = Field(
default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id."
)
budget_duration: str | None = Field(
default=None,
description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')",
@ -1981,8 +1995,14 @@ class OrgMember(MemberBase):
from litellm.models.team import TeamBase as TeamBase # noqa: E402
RouterSettingsDict = Annotated[
dict[str, object],
BeforeValidator(validate_router_settings_dict, json_schema_input_type=UpdateRouterConfig),
]
class NewTeamRequest(TeamBase):
router_settings: RouterSettingsDict | None = None
model_aliases: dict | None = None
tags: list | None = None
guardrails: list[str] | None = None
@ -2053,6 +2073,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
models: list | None = None
@ -2080,7 +2101,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None
enforced_batch_output_expires_after: dict | None = None
enforced_file_expires_after: dict | None = None
router_settings: dict | None = None
router_settings: RouterSettingsDict | None = None
access_group_ids: list[str] | None = None
budget_limits: list[BudgetLimitEntry] | None = None # multiple concurrent budget windows
default_team_member_models: list[str] | None = None # default allowed_models seeded onto new team members
@ -3008,6 +3029,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
team_alias: str | None = None
team_tpm_limit: int | None = None
team_rpm_limit: int | None = None
team_tpd_limit: int | None = None
team_max_budget: float | None = None
team_soft_budget: float | None = None
team_models: list = []
@ -3027,6 +3049,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
end_user_id: str | None = None
end_user_tpm_limit: int | None = None
end_user_rpm_limit: int | None = None
end_user_tpd_limit: int | None = None
end_user_max_budget: float | None = None
end_user_model_max_budget: dict | None = None
@ -3825,6 +3848,7 @@ class SpendLogsMetadata(TypedDict):
user_api_key_team_alias: str | None
spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call
requester_ip_address: str | None
user_agent: ReadOnly[str | None]
litellm_call_id: str | None
applied_guardrails: list[str] | None
mcp_tool_call_metadata: StandardLoggingMCPToolCall | None
@ -4464,12 +4488,14 @@ class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
jwt_claim_name: str
jwt_claim_value: str
key: str
jwt_issuer: str | None = None
description: str | None = None
class UpdateJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
id: str
key: str | None = None
jwt_issuer: str | None = None
description: str | None = None
is_active: bool | None = None
@ -4480,6 +4506,7 @@ class DeleteJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
class JWTKeyMappingResponse(LiteLLMPydanticObjectBase):
id: str
jwt_issuer: str | None = None
jwt_claim_name: str
jwt_claim_value: str
description: str | None = None
@ -4702,6 +4729,7 @@ class JWTAuthBuilderResult(TypedDict):
org_id: str | None
team_membership: LiteLLM_TeamMembership | None
jwt_claims: dict # Decoded JWT token claims (avoids re-decoding)
agent_id: ReadOnly[str | None]
class ClientSideFallbackModel(TypedDict, total=False):
@ -4940,6 +4968,14 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
user_allowed_roles: list[str] | None = None
user_id_upsert: bool = Field(default=False, description="If user doesn't exist, upsert them into the db.")
end_user_id_jwt_field: str | None = None
agent_id_jwt_field: str | None = Field(
default=None,
description=(
"The field in the JWT token that identifies the calling agent (e.g. 'azp' for a Microsoft Entra ID "
"app token). Supports dot notation. The value is matched against a registered agent's agent_id, "
"then agent_name, and the request is rejected when it matches neither."
),
)
public_key_ttl: float = 600
public_key_stale_ttl: float = Field(
default=DEFAULT_JWKS_STALE_TTL,
@ -5222,6 +5258,7 @@ class DBSpendUpdateTransactions(TypedDict):
team_list_transactions: dict[str, float] | None
team_member_list_transactions: dict[str, float] | None
org_list_transactions: dict[str, float] | None
org_member_list_transactions: ReadOnly[dict[str, float] | None]
tag_list_transactions: dict[str, float] | None
agent_list_transactions: dict[str, float] | None
model_access_group_list_transactions: ReadOnly[dict[str, float] | None]

47
litellm/proxy/auth/auth_checks.py
View file

@ -39,6 +39,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.models.project import LiteLLM_ProjectTable
from litellm.proxy._types import (
RBAC_ROLES,
CallInfo,
@ -109,7 +110,7 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import RowT_co
from litellm.repositories.prisma_protocols import DatabaseClient, RowT_co
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import (
AccessGroupRepository,
@ -147,6 +148,7 @@ class _PrismaDictableRow(Protocol):
class _PrismaJWTKeyMappingRow(Protocol):
token: str
jwt_issuer: str
jwt_claim_name: str
jwt_claim_value: str
@ -847,6 +849,7 @@ BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset(
"/health",
"/health/services",
"/health/test_connection",
"/auto_router/test_routing",
}
)
@ -3172,7 +3175,7 @@ async def _delete_cache_access_object(
@log_db_metrics
async def get_access_object(
access_group_id: str,
prisma_client: PrismaClient | None,
prisma_client: DatabaseClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_AccessGroupTable:
@ -3599,9 +3602,18 @@ async def _fetch_key_object_from_db_with_reconnect(
raise
def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str) -> str:
"""Cache key under which ``_resolve_jwt_to_virtual_key`` stores a JWT-claim-to-key mapping."""
return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str, jwt_issuer: str | None = None) -> str:
"""Cache key under which a JWT-claim-to-key mapping is stored, scoped to one
issuer (or the issuer-agnostic/global scope when ``jwt_issuer`` is falsy).
Scoped by issuer (when one is configured) so a cached hit or ``__NO_MAPPING__`` miss
for one issuer's claim value can never be served to a different issuer whose claim
value happens to collide. Unchanged for the global scope, keeping the single-issuer
(no ``litellm_jwtauth.issuers`` configured) cache key format stable across this fix.
"""
if not jwt_issuer:
return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
return f"jwt_key_mapping:{jwt_issuer}:{jwt_claim_name}:{jwt_claim_value}"
@log_db_metrics
@ -3613,7 +3625,7 @@ async def get_jwt_key_mapping_cache_keys_for_token(
mappings: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_many(
where={"token": hashed_token}
)
return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value) for m in mappings)
return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value, m.jwt_issuer) for m in mappings)
@log_db_metrics
@ -3621,9 +3633,14 @@ async def get_jwt_key_mapping_object(
jwt_claim_name: str,
jwt_claim_value: str,
prisma_client: PrismaClient,
jwt_issuer: str | None = None,
) -> str | None:
"""
Lookup a JWT-to-virtual-key mapping from the database.
Lookup a JWT-to-virtual-key mapping from the database for one exact scope:
``jwt_issuer`` (or the global/issuer-agnostic scope when falsy). Does not fall
back to the global scope itself -- a caller that wants "issuer-scoped mapping,
else the global one" queries both scopes itself, so each result can be cached
under its own scope's key (see ``_resolve_jwt_to_virtual_key``).
Returns the hashed token (str) if a matching active mapping is found, else None.
"""
@ -3631,6 +3648,7 @@ async def get_jwt_key_mapping_object(
where={
"jwt_claim_name": jwt_claim_name,
"jwt_claim_value": jwt_claim_value,
"jwt_issuer": jwt_issuer or "",
"is_active": True,
}
)
@ -3918,7 +3936,7 @@ async def get_org_object(
async def _get_resources_from_access_groups(
access_group_ids: Sequence[str],
resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
prisma_client: PrismaClient | None = None,
prisma_client: DatabaseClient | None = None,
user_api_key_cache: UserApiKeyCache | None = None,
proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
@ -3976,7 +3994,7 @@ async def _get_resources_from_access_groups(
async def _get_models_from_access_groups(
access_group_ids: Sequence[str],
prisma_client: PrismaClient | None = None,
prisma_client: DatabaseClient | None = None,
user_api_key_cache: UserApiKeyCache | None = None,
proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
@ -4475,6 +4493,7 @@ async def can_key_call_model(
llm_model_list: Sequence[object] | None,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
prisma_client: DatabaseClient | None = None,
) -> Literal[True]:
"""
Checks if token can call a given model
@ -4504,6 +4523,7 @@ async def can_key_call_model(
if key_access_group_ids:
models_from_groups: Final = await _get_models_from_access_groups(
access_group_ids=key_access_group_ids,
prisma_client=prisma_client,
)
if models_from_groups:
return _can_object_call_model(
@ -4632,6 +4652,7 @@ async def can_team_access_model(
team_object: LiteLLM_TeamTable | None,
llm_router: Router | None,
team_model_aliases: dict[str, str] | None = None,
prisma_client: DatabaseClient | None = None,
) -> Literal[True]:
"""
Returns True if the team can access a specific model.
@ -4654,12 +4675,13 @@ async def can_team_access_model(
if team_access_group_ids:
models_from_groups: Final = await _get_models_from_access_groups(
access_group_ids=team_access_group_ids,
prisma_client=prisma_client,
)
if models_from_groups:
return _can_object_call_model(
model=model,
llm_router=llm_router,
models=models_from_groups,
models=list(dict.fromkeys([*(team_object.models if team_object else []), *models_from_groups])),
team_model_aliases=team_model_aliases,
team_id=team_object.team_id if team_object else None,
object_type="team",
@ -4749,7 +4771,7 @@ async def _key_access_group_grants_model(
def can_project_access_model(
model: str | list[str],
project_object: LiteLLM_ProjectTableCachedObj,
project_object: LiteLLM_ProjectTable,
llm_router: Router | None,
) -> Literal[True]:
"""
@ -5767,8 +5789,7 @@ async def _organization_max_budget_check(
if org_table.litellm_budget_table is not None:
org_max_budget = org_table.litellm_budget_table.max_budget
# Only check if organization has a valid max_budget set
if org_max_budget is None or org_max_budget <= 0:
if org_max_budget is None:
return
# Read spend from cross-pod counter (Redis-first) or cached object (fallback)

29
litellm/proxy/auth/auth_exception_handler.py
View file

@ -23,6 +23,7 @@ from litellm.proxy.auth.auth_utils import (
_get_request_ip_address,
is_invalid_virtual_key_error,
mark_invalid_virtual_key_error,
normalize_request_route,
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes
@ -74,17 +75,28 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
)
def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]:
def _get_user_agent(request: Request) -> str | None:
if "headers" not in request.scope:
return None
return request.headers.get("user-agent")
def _with_client_context(
request_data: dict[str, object], requester_ip: str | None, user_agent: str | None
) -> dict[str, object]:
"""Auth gate rejections are raised before `add_litellm_data_to_request` records the
caller IP, so their failure logs would otherwise carry no IP nor key/user identity."""
if not requester_ip:
return request_data
caller IP and User-Agent, so their failure logs would otherwise carry neither."""
key: Final = "litellm_metadata" if "litellm_metadata" in request_data else "metadata"
metadata: Final = request_data.get(key)
base: Final[Mapping[str, object]] = metadata if isinstance(metadata, Mapping) else EMPTY_MAPPING
if base.get("requester_ip_address"):
stamped: Final = {
name: value
for name, value in (("requester_ip_address", requester_ip), ("user_agent", user_agent))
if value and not base.get(name)
}
if not stamped:
return request_data
return {**request_data, key: {**base, "requester_ip_address": requester_ip}} # mutable-ok: logging needs dicts
return {**request_data, key: {**base, **stamped}} # mutable-ok: logging needs dicts
class UserAPIKeyAuthExceptionHandler:
@ -148,6 +160,7 @@ class UserAPIKeyAuthExceptionHandler:
request=request,
use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True,
)
user_agent: Final = _get_user_agent(request)
# Log authentication failures before identity seeding and callbacks, so the log
# survives a raising callback pipeline. Classify and route malformed virtual-key
@ -172,7 +185,7 @@ class UserAPIKeyAuthExceptionHandler:
# so the handler is side-effect-free for the caller's identity object.
user_api_key_dict = resolved_identity.model_copy() if resolved_identity is not None else UserAPIKeyAuth()
user_api_key_dict.parent_otel_span = parent_otel_span
user_api_key_dict.request_route = route
user_api_key_dict.request_route = normalize_request_route(route)
user_api_key_dict.api_key = user_api_key_dict.api_key or UserAPIKeyAuth(api_key=api_key).api_key
# Stamp identity onto the request's server span now, before the request
@ -200,7 +213,7 @@ class UserAPIKeyAuthExceptionHandler:
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
request_data=_with_requester_ip_address(request_data, requester_ip),
request_data=_with_client_context(request_data, requester_ip, user_agent),
original_exception=e,
user_api_key_dict=user_api_key_dict,
error_type=ProxyErrorTypes.auth_error,

66
litellm/proxy/auth/auth_utils.py
View file

@ -28,6 +28,7 @@ from litellm.litellm_core_utils.url_utils import (
validate_url,
)
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
from litellm.proxy._types import *
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@ -976,6 +977,26 @@ def _get_deployment_default_tpm_limit(model_name: str) -> int | None:
return _get_deployment_default_limit(model_name, "default_api_key_tpm_limit")
def get_key_own_model_rate_limit(
user_api_key_dict: UserAPIKeyAuth,
rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
) -> dict[str, int] | None:
if user_api_key_dict.metadata:
result: Final = user_api_key_dict.metadata.get(rate_limit_key)
if result:
return result
if not user_api_key_dict.model_max_budget:
return None
budget_key: Final = "rpm_limit" if rate_limit_key == "model_rpm_limit" else "tpm_limit"
model_limit: Final = {
model: budget[budget_key]
for model, budget in user_api_key_dict.model_max_budget.items()
if isinstance(budget, dict) and budget.get(budget_key) is not None
}
return model_limit or None
def get_key_model_rpm_limit(
user_api_key_dict: UserAPIKeyAuth,
model_name: str | None = None,
@ -989,20 +1010,9 @@ def get_key_model_rpm_limit(
3. Team metadata (model_rpm_limit)
4. Deployment default_api_key_rpm_limit (when model_name is provided)
"""
# 1. Check key metadata first (takes priority)
if user_api_key_dict.metadata:
result: Final = user_api_key_dict.metadata.get("model_rpm_limit")
if result:
return result
# 2. Check model_max_budget
if user_api_key_dict.model_max_budget:
model_rpm_limit: Final[dict[str, int]] = {}
for model, budget in user_api_key_dict.model_max_budget.items():
if isinstance(budget, dict) and budget.get("rpm_limit") is not None:
model_rpm_limit[model] = budget["rpm_limit"]
if model_rpm_limit:
return model_rpm_limit
key_own_limit: Final = get_key_own_model_rate_limit(user_api_key_dict, "model_rpm_limit")
if key_own_limit is not None:
return key_own_limit
# 3. Fallback to team metadata
if user_api_key_dict.team_metadata:
@ -1032,20 +1042,9 @@ def get_key_model_tpm_limit(
3. Team metadata (model_tpm_limit)
4. Deployment default_api_key_tpm_limit (when model_name is provided)
"""
# 1. Check key metadata first (takes priority)
if user_api_key_dict.metadata:
result: Final = user_api_key_dict.metadata.get("model_tpm_limit")
if result:
return result
# 2. Check model_max_budget (iterate per-model like RPM does)
if user_api_key_dict.model_max_budget:
model_tpm_limit: Final[dict[str, int]] = {}
for model, budget in user_api_key_dict.model_max_budget.items():
if isinstance(budget, dict) and budget.get("tpm_limit") is not None:
model_tpm_limit[model] = budget["tpm_limit"]
if model_tpm_limit:
return model_tpm_limit
key_own_limit: Final = get_key_own_model_rate_limit(user_api_key_dict, "model_tpm_limit")
if key_own_limit is not None:
return key_own_limit
# 3. Fallback to team metadata
if user_api_key_dict.team_metadata:
@ -1967,6 +1966,11 @@ def request_dispatched_to_pass_through_endpoint(request: Request | None) -> bool
return getattr(endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, False) is True
def request_dispatched_to_provider_pass_through(request: Request) -> bool:
"""Built-in provider pass-through handlers (``/anthropic/{endpoint:path}``, ...) bind ``endpoint``."""
return "endpoint" in request.path_params
def get_model_from_request(
request_data: dict,
route: str,
@ -2040,6 +2044,12 @@ def get_model_from_request(
azure_model: Final = _router_model_from_azure_route(route, llm_router)
return model if azure_model is None else azure_model
if route.lower().startswith("/nvidia_nim/"):
nvidia_nim_model: Final = (
nvidia_nim_model_group_in_path(route, llm_router.get_model_list()) if llm_router else None
)
return model if nvidia_nim_model is None else nvidia_nim_model
return model

136
litellm/proxy/auth/auto_router_checks.py Normal file
View file

@ -0,0 +1,136 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
if TYPE_CHECKING:
from litellm.router import Router
_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _mapping(value: object) -> Mapping[str, object] | None:
try:
return _MAPPING_ADAPTER.validate_python(value)
except ValidationError:
return None
async def authorize_member_auto_router_inference(
*,
deployment: Mapping[str, object] | None,
request_kwargs: Mapping[str, object],
llm_router: Router,
) -> None:
if deployment is None:
return
model_info: Final = _mapping(deployment.get("model_info"))
if model_info is None or model_info.get("member_auto_router") is not True:
return
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import (
OrganizationNotFoundError,
TeamNotFoundError,
get_org_object,
get_project_object,
get_team_membership,
get_team_object,
)
from litellm.proxy.management_helpers.auto_router_permissions import (
MemberAutoRouterDependencyObjects,
authorize_member_auto_router_dependencies,
validate_member_auto_router_config,
)
metadata: Final = _mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)))
actor: Final = metadata.get("user_api_key_auth") if metadata is not None else None
team_id: Final = model_info.get("team_id")
if not isinstance(actor, UserAPIKeyAuth) or not isinstance(team_id, str) or not team_id:
raise HTTPException(status_code=403, detail="Member auto-routers require authenticated team access")
if actor.team_id != team_id and actor.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="This auto-router belongs to a different team")
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
if prisma_client is None:
raise HTTPException(status_code=503, detail="Cannot verify auto-router model access without a database")
try:
team: Final = await get_team_object(
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=actor.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
except TeamNotFoundError as error:
raise HTTPException(status_code=403, detail="The auto-router team no longer exists") from error
if (
actor.user_role != LitellmUserRoles.PROXY_ADMIN
and actor.user_id is not None
and (not actor.user_id or not any(member.user_id == actor.user_id for member in team.members_with_roles))
):
raise HTTPException(status_code=403, detail="You are no longer a member of this auto-router's team")
if team.blocked:
raise HTTPException(status_code=403, detail="This auto router's team is blocked.")
params: Final = _mapping(deployment.get("litellm_params"))
if params is None:
raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
raw_config: Final = _mapping(params.get("complexity_router_config"))
if raw_config is None:
raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
default_model: Final = params.get("complexity_router_default_model")
config: Final = validate_member_auto_router_config(raw_config)
membership: Final = (
await get_team_membership(
user_id=actor.user_id,
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=actor.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if actor.user_id
else None
)
try:
organization: Final = (
await get_org_object(
org_id=team.organization_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=actor.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if team.organization_id
else None
)
except OrganizationNotFoundError as error:
raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.") from error
project: Final = (
await get_project_object(
project_id=actor.project_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
if actor.project_id
else None
)
await authorize_member_auto_router_dependencies(
config=config,
default_model=default_model if isinstance(default_model, str) else None,
user_api_key_dict=actor,
team=team,
prisma_client=None,
llm_router=llm_router,
dependency_objects=MemberAutoRouterDependencyObjects(
membership=membership, organization=organization, project=project
),
)

71
litellm/proxy/auth/handle_jwt.py
View file

@ -14,7 +14,7 @@ import hashlib
import os
import re
import time
from collections.abc import Awaitable, Callable, Sequence
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Any, Final, Literal, NoReturn, Protocol, TypeVar, cast
import httpx
@ -61,6 +61,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
)
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.user_repository import UserRepository
from litellm.types.agents import AgentResponse
from .auth_checks import (
_allowed_routes_check,
@ -127,6 +128,26 @@ class _UserInfoResponse(Protocol):
def json(self) -> dict[str, object]: ...
class AgentLookup(Protocol):
"""The registered-agent lookups a JWT agent claim is matched against."""
def get_agent_by_id(self, agent_id: str) -> AgentResponse | None:
"""The agent registered under ``agent_id``, if any."""
def get_agent_by_name(self, agent_name: str) -> AgentResponse | None:
"""The agent registered under ``agent_name``, if any."""
class _NoRegisteredAgents:
"""The lookup in force until the proxy binds its agent registry: no agent is registered, so no claim matches."""
def get_agent_by_id(self, agent_id: str) -> None:
return None
def get_agent_by_name(self, agent_name: str) -> None:
return None
def _discovery_document(response: _OIDCDiscoveryResponse) -> _OIDCDiscoveryBody:
"""Decode an OIDC discovery response body."""
return response.json()
@ -198,6 +219,10 @@ class JWTHandler:
self.leeway = 0
# Per-cache-key locks so a TTL lapse triggers one refresh instead of one per in-flight request.
self._refresh_locks: dict[str, asyncio.Lock] = {} # mutable-ok: lock registry, keyed by JWKS url
self.agent_lookup: AgentLookup = _NoRegisteredAgents()
def bind_agent_lookup(self, agent_lookup: AgentLookup) -> None:
self.agent_lookup = agent_lookup
def update_environment(
self,
@ -623,6 +648,12 @@ class JWTHandler:
object_id = default_value
return object_id
def get_agent_claim(self, token: Mapping[str, object]) -> str | None:
if self.litellm_jwtauth.agent_id_jwt_field is None:
return None
claim: Final[object] = get_nested_value(data=token, key_path=self.litellm_jwtauth.agent_id_jwt_field)
return claim if isinstance(claim, str) and claim else None
def get_org_id(self, token: dict, default_value: str | None) -> str | None:
if self._has_trusted_issuer_normalized_claim(token=token, claim=self.LITELLM_ORG_ID_CLAIM):
return token.get(self.LITELLM_ORG_ID_CLAIM)
@ -1380,6 +1411,7 @@ class JWTAuthManager:
api_key: str,
jwt_valid_token: dict | None = None,
user_email: str | None = None,
agent_id: str | None = None,
) -> JWTAuthBuilderResult | None:
"""Check admin status and route access permissions"""
if not jwt_handler.is_admin(scopes=scopes):
@ -1409,8 +1441,28 @@ class JWTAuthManager:
org_id=org_id,
team_membership=None,
jwt_claims=jwt_valid_token or {},
agent_id=agent_id,
)
@staticmethod
def resolve_agent_id(
jwt_handler: JWTHandler,
jwt_valid_token: Mapping[str, object],
agent_registry: AgentLookup,
) -> str | None:
agent_claim: Final = jwt_handler.get_agent_claim(token=jwt_valid_token)
if agent_claim is None:
return None
agent: Final = agent_registry.get_agent_by_id(agent_id=agent_claim) or agent_registry.get_agent_by_name(
agent_name=agent_claim
)
if agent is None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"No registered agent matches JWT claim {jwt_handler.litellm_jwtauth.agent_id_jwt_field}={agent_claim}",
)
return agent.agent_id
@staticmethod
async def find_and_validate_specific_team_id(
jwt_handler: JWTHandler,
@ -2268,9 +2320,23 @@ class JWTAuthManager:
elif rbac_role == LitellmUserRoles.INTERNAL_USER:
user_id = object_id
agent_id: Final = JWTAuthManager.resolve_agent_id(
jwt_handler=jwt_handler,
jwt_valid_token=jwt_valid_token,
agent_registry=jwt_handler.agent_lookup,
)
# Check admin access
admin_result: Final = await JWTAuthManager.check_admin_access(
jwt_handler, scopes, route, user_id, org_id, api_key, jwt_valid_token, user_email=user_email
jwt_handler,
scopes,
route,
user_id,
org_id,
api_key,
jwt_valid_token,
user_email=user_email,
agent_id=agent_id,
)
if admin_result:
await JWTAuthManager._attach_team_from_header_for_admin(
@ -2514,4 +2580,5 @@ class JWTAuthManager:
token=api_key,
team_membership=team_membership_object,
jwt_claims=jwt_valid_token,
agent_id=agent_id,
)

4
litellm/proxy/auth/litellm_license.py
View file

@ -155,8 +155,8 @@ class LicenseCheck:
def auto_router_capability_limit(self) -> int | None:
"""
How many auto-routers may claim each licensed capability (heuristic_v2, operator-defined
tier_definitions): unlimited (None) only when the signed license lists the auto_router
How many auto-routers may claim each gated classifier or customization capability:
unlimited (None) only when the signed license lists the auto_router
feature, otherwise one per capability. A license verified through the API carries no
feature list, so it does not lift the limit either.
"""

2
litellm/proxy/auth/login_utils.py
View file

@ -249,6 +249,7 @@ async def authenticate_user(
if os.getenv("DATABASE_URL") is not None:
response = await generate_key_helper_fn(
llm_router=None,
request_type="key",
**{
"user_role": LitellmUserRoles.PROXY_ADMIN,
@ -324,6 +325,7 @@ async def authenticate_user(
await _rehash_password_if_needed(_user_row.user_id, password, _password)
if os.getenv("DATABASE_URL") is not None:
response = await generate_key_helper_fn(
llm_router=None,
request_type="key",
**{
"user_role": user_role,

18
litellm/proxy/auth/route_checks.py
View file

@ -1,5 +1,5 @@
import re
from collections.abc import Sequence
from collections.abc import Collection
from typing import Final
from fastapi import HTTPException, Request, status
@ -24,10 +24,13 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
[
# user
"/user/new",
"/management/v1/users/bulk",
"/user/delete",
"/management/v1/users/bulk_delete",
"/user/bulk_update",
# team
"/team/new",
"/management/v1/teams/{team_id}/members/bulk_delete",
"/team/update",
"/team/delete",
"/team/block",
@ -587,7 +590,7 @@ class RouteChecks:
return False
@staticmethod
def check_route_access(route: str, allowed_routes: Sequence[str]) -> bool:
def check_route_access(route: str, allowed_routes: Collection[str]) -> bool:
"""
Check if a route has access by checking both exact matches and patterns
@ -758,9 +761,12 @@ class RouteChecks:
_ADMIN_VIEWER_BLOCKED_WRITE_ROUTES = frozenset(
[
"/user/new",
"/management/v1/users/bulk",
"/user/delete",
"/management/v1/users/bulk_delete",
"/user/bulk_update",
"/team/new",
"/management/v1/teams/{team_id}/members/bulk_delete",
"/team/update",
"/team/delete",
"/model/new",
@ -824,7 +830,7 @@ class RouteChecks:
status_code=status.HTTP_403_FORBIDDEN,
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated",
)
elif route in _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES or (
elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or (
route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES)
):
# Block write operations for PROXY_ADMIN_VIEW_ONLY
@ -859,9 +865,9 @@ class RouteChecks:
# Hard-block known write routes regardless of HTTP method (defensive
# — these are POSTs in practice, but pinning them here protects
# against future GET-shaped writes).
if route in RouteChecks._ADMIN_VIEWER_BLOCKED_WRITE_ROUTES or (
route.startswith("/key/") and route.endswith("/regenerate")
):
if RouteChecks.check_route_access(
route=route, allowed_routes=RouteChecks._ADMIN_VIEWER_BLOCKED_WRITE_ROUTES
) or (route.startswith("/key/") and route.endswith("/regenerate")):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",

2
litellm/proxy/auth/team_grants.py
View file

@ -56,6 +56,7 @@ class TeamGrants(TypedDict, total=False):
team_alias: ReadOnly[str | None]
team_tpm_limit: ReadOnly[int | None]
team_rpm_limit: ReadOnly[int | None]
team_tpd_limit: ReadOnly[int | None]
team_max_budget: ReadOnly[float | None]
team_soft_budget: ReadOnly[float | None]
team_spend: ReadOnly[float | None]
@ -97,6 +98,7 @@ def team_grants(
team_alias=team_object.team_alias,
team_tpm_limit=team_object.tpm_limit,
team_rpm_limit=team_object.rpm_limit,
team_tpd_limit=team_object.tpd_limit,
team_max_budget=team_object.max_budget,
team_soft_budget=team_object.soft_budget,
team_spend=team_object.spend,

159
litellm/proxy/auth/user_api_key_auth.py
View file

@ -26,11 +26,13 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.caching.redis_cache import RedisCache
from litellm.constants import (
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
GLOBAL_PROXY_SPEND_CACHE_KEY,
INVALID_VIRTUAL_KEY_ERROR_MARKER,
INVALID_VIRTUAL_KEY_ERROR_MESSAGE,
LITELLM_PROXY_BUDGET_NAME,
LITELLM_PROXY_MASTER_KEY_ALIAS,
MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY,
)
from litellm.integrations.otel.model.config import is_otel_v2_enabled
from litellm.integrations.otel.runtime import phase_span, seed_request_identity
@ -76,6 +78,8 @@ from litellm.proxy.auth.auth_utils import (
iter_request_fallback_targets,
normalize_request_route,
pre_db_read_auth_checks,
request_dispatched_to_pass_through_endpoint,
request_dispatched_to_provider_pass_through,
route_in_additonal_public_routes,
)
from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
@ -103,6 +107,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
_safe_set_request_parsed_body,
populate_request_with_path_params,
read_raw_json_body,
rewrite_request_model,
)
from litellm.proxy.common_utils.model_listing_utils import claude_code_requested_group
from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
@ -124,6 +129,7 @@ from litellm.proxy.utils import (
normalize_route_for_root_path,
)
from litellm.repositories.table_repositories import TeamMembershipRepository
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.secret_managers.main import get_secret_bool
from litellm.types.services import ServiceTypes
@ -235,11 +241,45 @@ async def _normalize_claude_model(
request.scope[_CLAUDE_MODEL_NORMALIZED] = True
if source is None:
return
request_data["model"] = source
_safe_set_request_parsed_body(request=request, parsed_body=request_data)
if request is not None:
request._json = request_data
request._body = orjson.dumps(request_data)
rewrite_request_model(request_data, request, source)
async def _resolve_router_settings_model_group_alias(
request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader
valid_token: UserAPIKeyAuth,
request: Request | None,
route: str,
) -> None:
"""Rewrite the requested model through the key's or team's ``router_settings.model_group_alias``
before the allowlist checks, so they authorize the model group the request is routed to.
"""
from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj
if request is None or llm_router is None or not RouteChecks.is_llm_api_route(route=route):
return
if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is True:
return
request.scope[MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY] = True
if request_dispatched_to_pass_through_endpoint(request) or request_dispatched_to_provider_pass_through(request):
return
requested: Final = request_data.get("model")
if not isinstance(requested, str) or await read_raw_json_body(request=request) is None:
return
settings: Final = await proxy_config.get_hierarchical_router_settings(
user_api_key_dict=valid_token, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
)
if not isinstance(settings, Mapping):
return
target: Final = resolve_model_group_alias(settings.get("model_group_alias"), requested)
if target is None or target == requested:
return
verbose_proxy_logger.debug(
"router_settings.model_group_alias resolved %s -> %s before auth",
requested.replace("\r", "").replace("\n", ""),
target.replace("\r", "").replace("\n", ""),
)
request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested)
rewrite_request_model(request_data, request, target)
def _get_model_names_for_budget_checks(
@ -269,6 +309,17 @@ class _TokenTeamModels(Protocol):
def team_models(self) -> list[str]: ...
class _RawCacheRead(Protocol):
async def async_get_cache(self, *, key: str) -> object: ...
def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead:
"""View an untyped cache object's ``async_get_cache`` as returning ``object``
instead of ``Any``, so a caller can ``isinstance``-narrow it without paying
the ``reportAny`` cost of the underlying (unannotated) cache implementation."""
return cache
def _token_team_models(valid_token: _TokenTeamModels) -> list[str]:
return valid_token.team_models
@ -537,6 +588,9 @@ def _apply_budget_limits_to_end_user_params(
if budget_info.rpm_limit is not None:
end_user_params["end_user_rpm_limit"] = budget_info.rpm_limit
if budget_info.tpd_limit is not None:
end_user_params["end_user_tpd_limit"] = budget_info.tpd_limit
if budget_info.max_budget is not None:
end_user_params["end_user_max_budget"] = budget_info.max_budget
@ -621,6 +675,8 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use
valid_token.end_user_tpm_limit = end_user_params["end_user_tpm_limit"]
if end_user_params.get("end_user_rpm_limit") is not None:
valid_token.end_user_rpm_limit = end_user_params["end_user_rpm_limit"]
if end_user_params.get("end_user_tpd_limit") is not None:
valid_token.end_user_tpd_limit = end_user_params["end_user_tpd_limit"]
if end_user_params.get("allowed_model_region") is not None:
valid_token.allowed_model_region = end_user_params["allowed_model_region"]
if end_user_params.get("end_user_model_max_budget") is not None:
@ -837,6 +893,7 @@ class _PendingAutoRegister(NamedTuple):
claim_field: str
claim_value: str
cache_key: str
jwt_issuer: str | None = None
async def _auto_register_jwt_mapping(
@ -848,10 +905,12 @@ async def _auto_register_jwt_mapping(
parent_otel_span: Span | None,
proxy_logging_obj: ProxyLogging,
cache_key: str,
jwt_issuer: str | None = None,
team_id: str | None = None,
user_id: str | None = None,
org_id: str | None = None,
end_user_id: str | None = None,
agent_id: str | None = None,
) -> UserAPIKeyAuth | None:
"""
Auto-register: create a new virtual key + mapping for an unrecognised JWT
@ -878,11 +937,13 @@ async def _auto_register_jwt_mapping(
# the NOT NULL @id constraint. Every successful key-creation caller (e.g.
# /key/generate) passes table_name="key" explicitly.
key_data: Final = await generate_key_helper_fn(
llm_router=None,
request_type="key",
table_name="key",
team_id=team_id,
user_id=user_id,
organization_id=org_id,
agent_id=agent_id,
metadata={
"auto_registered": True,
"jwt_claim_field": virtual_key_claim_field,
@ -897,6 +958,7 @@ async def _auto_register_jwt_mapping(
try:
await prisma_client.db.litellm_jwtkeymapping.create(
data={
"jwt_issuer": jwt_issuer or "",
"jwt_claim_name": virtual_key_claim_field,
"jwt_claim_value": claim_value,
"token": token_hash,
@ -931,6 +993,7 @@ async def _auto_register_jwt_mapping(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=claim_value,
prisma_client=prisma_client,
jwt_issuer=jwt_issuer,
)
if token_hash is None:
# The winner's mapping vanished between the unique-constraint
@ -975,6 +1038,43 @@ async def _auto_register_jwt_mapping(
return auto_registered_key
async def _lookup_jwt_mapping_token_hash(
prisma_client: PrismaClient,
user_api_key_cache: UserApiKeyCache,
virtual_key_claim_field: str,
claim_value: str,
normalized_issuer: str | None,
cache_key: str,
ttl: float,
) -> str | None:
issuer_scoped: Final = await get_jwt_key_mapping_object(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=claim_value,
prisma_client=prisma_client,
jwt_issuer=normalized_issuer,
)
if issuer_scoped is not None:
await user_api_key_cache.async_set_cache(key=cache_key, value=issuer_scoped, ttl=ttl)
return issuer_scoped
if normalized_issuer is None:
return None
# Another issuer may have already resolved (and cached) this same
# global mapping -- check its cache entry before re-querying the DB.
global_cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, claim_value)
cached_global: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=global_cache_key)
if isinstance(cached_global, str) and cached_global != "__NO_MAPPING__":
return cached_global
global_row: Final = await get_jwt_key_mapping_object(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=claim_value,
prisma_client=prisma_client,
jwt_issuer=None,
)
if global_row is not None:
await user_api_key_cache.async_set_cache(key=global_cache_key, value=global_row, ttl=ttl)
return global_row
async def _resolve_jwt_to_virtual_key(
jwt_claims: dict,
jwt_handler: JWTHandler,
@ -1033,7 +1133,7 @@ async def _resolve_jwt_to_virtual_key(
)
return None
cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value), normalized_issuer)
raw_cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key)
sentinel_written_by_this_policy: Final = behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER
cached_mapping: Final = (
@ -1073,6 +1173,7 @@ async def _resolve_jwt_to_virtual_key(
claim_field=virtual_key_claim_field,
claim_value=str(claim_value),
cache_key=cache_key,
jwt_issuer=normalized_issuer,
)
return None
elif cached_mapping is not None:
@ -1086,21 +1187,30 @@ async def _resolve_jwt_to_virtual_key(
)
# Resolve the mapping from DB, or treat prisma_client=None as a definitive
# miss (no DB → no mapping can exist → apply no-match policy below).
token_hash: str | None = None
if prisma_client is not None:
token_hash = await get_jwt_key_mapping_object(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=str(claim_value),
# miss (no DB → no mapping can exist → apply no-match policy below). An
# issuer-scoped row wins; falling back to the global (no-issuer) row keeps
# mappings created before issuer scoping existed working for every issuer.
# Each tier is cached under ITS OWN key (the global tier under the
# issuer-less cache key, not under `cache_key`/this issuer's key) so that
# updating or deleting either row invalidates exactly the cache entries it
# can affect. Caching a global-row hit under the requesting issuer's key
# would leave every OTHER issuer that had fallen back to that same global
# mapping serving its stale token until TTL after the row changes.
token_hash: Final = (
await _lookup_jwt_mapping_token_hash(
prisma_client=prisma_client,
)
if token_hash is not None:
await user_api_key_cache.async_set_cache(
key=cache_key,
value=token_hash,
user_api_key_cache=user_api_key_cache,
virtual_key_claim_field=virtual_key_claim_field,
claim_value=str(claim_value),
normalized_issuer=normalized_issuer,
cache_key=cache_key,
ttl=jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl,
)
if prisma_client is not None
else None
)
if token_hash is not None:
return IdentityStore.key_from_principal(
await IdentityStore(
prisma_client,
@ -1141,6 +1251,7 @@ async def _resolve_jwt_to_virtual_key(
claim_field=virtual_key_claim_field,
claim_value=str(claim_value),
cache_key=cache_key,
jwt_issuer=normalized_issuer,
)
# FALLBACK_TEAM_MAPPING (default): cache the miss and return None so the
@ -1566,6 +1677,7 @@ async def _user_api_key_auth_builder(
org_id: Final = result["org_id"]
team_membership: Final[LiteLLM_TeamMembership | None] = result.get("team_membership", None)
jwt_claims = result.get("jwt_claims", None)
agent_id: Final[str | None] = result.get("agent_id")
if is_proxy_admin:
# Proxy admins authenticate via auth_builder (full
@ -1591,6 +1703,7 @@ async def _user_api_key_auth_builder(
end_user_id=end_user_id,
parent_otel_span=parent_otel_span,
jwt_claims=jwt_claims,
agent_id=agent_id,
**team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id),
)
@ -1611,6 +1724,7 @@ async def _user_api_key_auth_builder(
user_rpm_limit=(user_object.rpm_limit if user_object is not None else None),
user_model_max_budget=(user_object.model_max_budget if user_object is not None else None),
jwt_claims=jwt_claims,
agent_id=agent_id,
**team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id),
)
@ -1630,10 +1744,12 @@ async def _user_api_key_auth_builder(
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
cache_key=pending_auto_register.cache_key,
jwt_issuer=pending_auto_register.jwt_issuer,
team_id=team_id,
user_id=user_id,
org_id=org_id,
end_user_id=end_user_id,
agent_id=agent_id,
)
if auto_registered is not None:
auto_registered.jwt_claims = jwt_claims
@ -2019,6 +2135,7 @@ async def _user_api_key_auth_builder(
valid_token.end_user_id = end_user_params.get("end_user_id")
valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit")
valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit")
valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit")
valid_token.allowed_model_region = end_user_params.get("allowed_model_region")
if valid_token is not None:
@ -2295,6 +2412,7 @@ async def _user_api_key_auth_builder(
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
tpd_limit=valid_token.team_tpd_limit,
blocked=valid_token.team_blocked,
models=token_team_models,
metadata=valid_token.team_metadata,
@ -2448,6 +2566,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
tpd_limit=valid_token.team_tpd_limit,
blocked=valid_token.team_blocked,
models=token_team_models,
metadata=valid_token.team_metadata,
@ -2489,7 +2608,7 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc
async def _run_centralized_common_checks(
user_api_key_auth_obj: UserAPIKeyAuth,
request: Request,
request_data: dict,
request_data: dict[str, object],
route: str,
) -> None:
"""Run ``common_checks`` once at the ``user_api_key_auth`` wrapper
@ -2911,6 +3030,7 @@ async def _authorize_authenticated_request(
## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ##
RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request)
await _normalize_claude_model(request_data, user_api_key_auth_obj, request, route)
await _resolve_router_settings_model_group_alias(request_data, user_api_key_auth_obj, request, route)
# Single authorization point. Builder paths MUST NOT call common_checks.
# Route through the same exception handler the builder uses so
@ -3297,6 +3417,7 @@ async def _enforce_key_and_fallback_model_access(
Not included in common_checks — common_checks enforces team/user/project model access only.
"""
await _normalize_claude_model(request_data, valid_token, request, route)
await _resolve_router_settings_model_group_alias(request_data, valid_token, request, route)
config: Final = valid_token.config
if config != {}:

6
litellm/proxy/client/cli/README.md
View file

@ -490,7 +490,7 @@ lite codex exec "summarize the repo"
Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.
@ -580,8 +580,8 @@ What the command changed is recorded in `~/.litellm/claude_configure_state.json`
`lite configure claude`, `lite login --config-claude`, `lite up` and `lite autoroute up` also install a status line (`~/.litellm/statusline.py`, registered as `statusLine` in `~/.claude/settings.json` unless you already run one) that shows which model the auto-router actually served the last turn and, once the proxy has recorded the session, what the session cost against the router's savings baseline:
```
claude-auto · Routed to: claude-haiku-4-5 -63% vs Claude Opus 5
LiteLLM ████████░░░░░░░░░░░░░░░░ $0.14
Routed to: claude-haiku-4-5 -63% vs Claude Opus 5
claude-auto ████████░░░░░░░░░░░░░░░░ $0.14
Claude Opus 5 ████████████████████████ $0.38
```

325
litellm/proxy/client/cli/commands/agents.py
View file

@ -4,15 +4,16 @@ import re
import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, TypeAlias
from typing import Final, Literal, TypeAlias
import click
import requests
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login
from .claude_settings import ClaudeSettingsError, install_statusline_script
@ -65,6 +66,10 @@ _INSTALL_DOCS: Final[dict[str, str]] = {
_HIDDEN_AGENTS: Final = frozenset({"pi"})
CODEX_PROXY_PROVIDER: Final = "litellm"
CODEX_HOME_ENV: Final = "CODEX_HOME"
CODEX_MODEL_CATALOG_FILENAME: Final = "litellm-models.json"
_CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
_CODEX_PREFLIGHT_TIMEOUT_SECONDS: Final = 10.0
class AgentRunError(Exception):
@ -252,7 +257,7 @@ def agent_launch_args(command: str, base_url: str) -> list[str]:
class ListedModel(BaseModel):
"""The fields of a /v1/models entry that an OpenCode model entry is built from."""
"""The fields of a /v1/models entry that an OpenCode or Codex model entry is built from."""
id: str
mode: str | None = None
@ -265,7 +270,7 @@ class _ModelListing(BaseModel):
_MODEL_LISTING: Final = TypeAdapter(_ModelListing)
_OPENCODE_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
_NO_EXTRA_ENV: Final[Mapping[str, str]] = MappingProxyType({})
@ -274,6 +279,40 @@ class ModelSyncSkipped:
reason: str
@dataclass(frozen=True, slots=True)
class ModelSyncArgs:
"""CLI args, placed before the user's own, that hand an agent the synced model list."""
args: tuple[str, ...]
ModelSyncResult: TypeAlias = Mapping[str, str] | ModelSyncArgs | ModelSyncSkipped
def _chat_models(models: Sequence[ListedModel]) -> tuple[ListedModel, ...]:
return tuple(m for m in models if m.mode is None or m.mode in _CHAT_MODES)
def _fetch_model_listing(
base_url: str,
api_key: str,
*,
get: Callable[..., requests.Response],
) -> tuple[ListedModel, ...] | ModelSyncSkipped:
url: Final = base_url.rstrip("/") + "/v1/models"
try:
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
except requests.RequestException as e:
return ModelSyncSkipped(f"could not reach {url}: {e}")
if resp.status_code != 200:
return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
try:
listing: Final = _MODEL_LISTING.validate_json(resp.content)
except ValidationError:
return ModelSyncSkipped(f"{url} returned an unexpected body")
return listing.data
class _OpenCodeLimit(BaseModel):
context: int
output: int
@ -317,7 +356,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
it never lands in the config text. OpenCode merges this inline config over
the user's own files, leaving unrelated keys and providers untouched.
"""
chat_models: Final = tuple(m for m in models if m.mode is None or m.mode in _OPENCODE_CHAT_MODES)
chat_models: Final = _chat_models(models)
provider: Final = _OpenCodeProvider(
npm=OPENCODE_PROVIDER_NPM,
name=OPENCODE_PROVIDER_NAME,
@ -347,40 +386,269 @@ def opencode_model_sync_env(
"""
if OPENCODE_CONFIG_CONTENT_ENV in base_env:
return ModelSyncSkipped(f"{OPENCODE_CONFIG_CONTENT_ENV} is already set")
url: Final = base_url.rstrip("/") + "/v1/models"
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
if isinstance(listing, ModelSyncSkipped):
return listing
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
class _CodexTruncationPolicy(BaseModel):
mode: Literal["bytes"] = "bytes"
limit: int = 10_000
class _CodexModel(BaseModel):
"""One `ModelInfo` entry of a Codex model catalog for a model the installed Codex does not know.
Every field that some Codex release since `model_catalog_json` appeared
(0.105.0) deserializes without a default is spelled out here, so one catalog
parses on all of them; the values match the fallback metadata Codex uses for
a model slug it does not know, so picking such a proxy model behaves the
same as `codex -m` did.
"""
slug: str
display_name: str
description: None = None
supported_reasoning_levels: tuple[()] = ()
shell_type: Literal["unified_exec"] = "unified_exec"
visibility: Literal["list"] = "list"
supported_in_api: Literal[True] = True
priority: int
availability_nux: None = None
upgrade: None = None
support_verbosity: Literal[False] = False
supports_reasoning_summaries: Literal[False] = False
supports_parallel_tool_calls: Literal[False] = False
default_verbosity: None = None
apply_patch_tool_type: None = None
truncation_policy: _CodexTruncationPolicy = _CodexTruncationPolicy()
experimental_supported_tools: tuple[()] = ()
context_window: int | None
base_instructions: str
class _StockCodexUpgrade(BaseModel):
model_config = ConfigDict(extra="allow")
model: str
class _StockCodexModel(BaseModel):
"""One `ModelInfo` entry as the installed Codex prints it from `codex debug models`.
Only the fields the sync rewrites are named; everything else that release
knows about the model (its reasoning levels, prompt, tool support) rides
along untouched, whatever the release's schema.
"""
model_config = ConfigDict(extra="allow")
slug: str
priority: int
visibility: str
supported_in_api: bool = True
upgrade: _StockCodexUpgrade | None = None
class _StockCodexCatalog(BaseModel):
models: tuple[_StockCodexModel, ...]
class _CodexCatalog(BaseModel):
models: tuple[_CodexModel | _StockCodexModel, ...]
def _codex_catalog_entry(
priority: int,
listed: ListedModel,
stock: _StockCodexModel | None,
served: frozenset[str],
instructions: str,
) -> _CodexModel | _StockCodexModel:
if stock is None:
return _CodexModel(
slug=listed.id,
display_name=listed.id,
priority=priority,
context_window=listed.max_input_tokens,
base_instructions=instructions,
)
upgrade: Final = stock.upgrade if stock.upgrade is not None and stock.upgrade.model in served else None
return stock.model_copy(
update={"priority": priority, "visibility": "list", "supported_in_api": True, "upgrade": upgrade}
)
def codex_model_catalog(
models: Sequence[ListedModel], stock: Sequence[_StockCodexModel], instructions: str
) -> str | None:
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
Codex refuses an empty catalog, hence None instead of `{"models": []}`.
Passing a catalog replaces Codex's built-in one, so a proxy model the
installed Codex knows keeps that Codex's own entry and the proxy only
decides its place in the picker: the listing orders it, lists it even when
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
when the model it points at is served too. A model Codex does not know gets the fallback
entry, with the same base instructions Codex itself uses so the agent never
runs without a system prompt.
"""
chat_models: Final = _chat_models(models)
if not chat_models:
return None
served: Final = frozenset(m.id for m in chat_models)
known: Final = MappingProxyType({m.slug: m for m in stock})
catalog: Final = _CodexCatalog(
models=tuple(
_codex_catalog_entry(index, m, known.get(m.id), served, instructions) for index, m in enumerate(chat_models)
)
)
return catalog.model_dump_json()
def codex_model_catalog_path(env: Mapping[str, str], *, home: Callable[[], Path] = Path.home) -> Path:
override: Final = env.get(CODEX_HOME_ENV)
root: Final = Path(override) if override else home() / ".codex"
return root / CODEX_MODEL_CATALOG_FILENAME
def _replace_file(path: Path, text: str) -> None:
with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
_ = tmp.write(text)
try:
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
except requests.RequestException as e:
return ModelSyncSkipped(f"could not reach {url}: {e}")
if resp.status_code != 200:
return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
os.replace(tmp.name, path)
except OSError:
Path(tmp.name).unlink(missing_ok=True)
raise
def _codex_debug_models(
binary: str,
args: Sequence[str],
env: Mapping[str, str],
*,
run: Callable[..., subprocess.CompletedProcess[str]],
) -> str | ModelSyncSkipped:
"""What `codex debug models` prints with `args` in front, or why the installed Codex could not run it.
The command prints the catalog Codex would launch with, without touching
the network, so it lists the installed Codex's own models and parses a
catalog override the way a launch does. Releases before 0.130.0 have no
such command and are reported the same way. A batch shim goes through
cmd.exe exactly as the launch will.
"""
name: Final = os.path.basename(binary)
command: Final = _windows_command(binary, (binary, *args, "debug", "models"))
try:
listing: Final = _MODEL_LISTING.validate_json(resp.content)
except ValidationError:
return ModelSyncSkipped(f"{url} returned an unexpected body")
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing.data)})
completed: Final = run(
command,
env=dict(env),
stdin=subprocess.DEVNULL,
capture_output=True,
encoding="utf-8",
timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
)
except (OSError, subprocess.TimeoutExpired) as e:
return ModelSyncSkipped(f"`{name} debug models` failed: {e}")
if completed.returncode == 0:
return completed.stdout
lines: Final = completed.stderr.strip().splitlines()
detail: Final = lines[0] if lines else "no output"
return ModelSyncSkipped(f"`{name} debug models` exited {completed.returncode}: {detail}")
def _stock_codex_models(
binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
) -> tuple[_StockCodexModel, ...] | ModelSyncSkipped:
printed: Final = _codex_debug_models(binary, (), env, run=run)
if isinstance(printed, ModelSyncSkipped):
return printed
try:
return _StockCodexCatalog.model_validate_json(printed).models
except ValidationError as e:
name: Final = os.path.basename(binary)
return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
def codex_model_sync_args(
base_env: Mapping[str, str],
base_url: str,
api_key: str,
*,
binary: str = "codex",
get: Callable[..., requests.Response] = requests.get,
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
home: Callable[[], Path] = Path.home,
instructions_path: Path = _CODEX_BASE_INSTRUCTIONS_PATH,
) -> ModelSyncArgs | ModelSyncSkipped:
"""`-c model_catalog_json=...` pointing Codex at the proxy's model list, or why it was skipped.
Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
atomically replaced on every launch. The Codex at `binary` first lists its
own models, so the ones the proxy serves keep that Codex's entries, and then
reads the file back once before it is handed over. The key never lands in
the file. A failed fetch, read, listing, write or read-back is reported
rather than raised: Codex still launches with its built-in catalog and takes
a proxy model by name via -m, and a rejected file stays on disk to be looked
at.
"""
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
if isinstance(listing, ModelSyncSkipped):
return listing
try:
instructions: Final = instructions_path.read_text(encoding="utf-8")
except OSError as e:
return ModelSyncSkipped(f"could not read {instructions_path}: {e}")
path: Final = codex_model_catalog_path(base_env, home=home)
try:
path.parent.mkdir(parents=True, exist_ok=True)
except OSError as e:
return ModelSyncSkipped(f"could not write {path}: {e}")
stock: Final = _stock_codex_models(binary, base_env, run=run)
if isinstance(stock, ModelSyncSkipped):
return stock
catalog: Final = codex_model_catalog(listing, stock, instructions)
if catalog is None:
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
try:
_replace_file(path, catalog)
except OSError as e:
return ModelSyncSkipped(f"could not write {path}: {e}")
override: Final = f"model_catalog_json={json.dumps(str(path))}"
read_back: Final = _codex_debug_models(binary, ("-c", override), base_env, run=run)
if isinstance(read_back, ModelSyncSkipped):
return read_back
return ModelSyncArgs(("-c", override))
def agent_model_sync_env(
command: str,
binary: str,
base_env: Mapping[str, str],
base_url: str,
api_key: str,
skip_verify: bool,
*,
get: Callable[..., requests.Response] = requests.get,
) -> Mapping[str, str] | ModelSyncSkipped:
"""Extra env an agent needs to see the proxy's model list.
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
) -> ModelSyncResult:
"""Extra env or args an agent needs to see the proxy's model list.
Only OpenCode needs one: Claude Code discovers models through
CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY and Codex takes the model by name.
skip_verify means the caller wants no pre-launch proxy call at all, so the
listing is skipped too rather than hanging on an offline proxy.
binary is the resolved path the launch will run (`codex.cmd` on a Windows
npm install). OpenCode takes the list as env, Codex as a `-c` override that
binary has read back first; Claude Code discovers models itself through
CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY. skip_verify means the caller
wants no pre-launch proxy call at all, so the listing is skipped too rather
than hanging on an offline proxy.
"""
if os.path.basename(command) != "opencode":
agent: Final = os.path.splitext(os.path.basename(binary))[0]
if agent not in ("opencode", "codex"):
return _NO_EXTRA_ENV
if skip_verify:
return ModelSyncSkipped(f"{_SKIP_VERIFY_FLAG} was passed")
if agent == "codex":
return codex_model_sync_args(base_env, base_url, api_key, binary=binary, get=get, run=run)
return opencode_model_sync_env(base_env, base_url, api_key, get=get)
@ -508,9 +776,7 @@ def run_agent(
base_env: Mapping[str, str] | None = None,
which: Callable[[str], str | None] = shutil.which,
verify: Callable[[str, str], None] = verify_proxy_key,
sync_models: Callable[[str, Mapping[str, str], str, str, bool], Mapping[str, str] | ModelSyncSkipped] = (
agent_model_sync_env
),
sync_models: Callable[[str, Mapping[str, str], str, str, bool], ModelSyncResult] = agent_model_sync_env,
warn: Callable[[str], None] = _warn,
launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off,
reattach_terminal: Callable[[], None] | None = None,
@ -537,7 +803,7 @@ def run_agent(
verify(base_url, api_key)
env_before_sync: Final = base_env if base_env is not None else os.environ
synced: Final = sync_models(command[0], env_before_sync, base_url, api_key, skip_verify)
synced: Final = sync_models(binary, env_before_sync, base_url, api_key, skip_verify)
if isinstance(synced, ModelSyncSkipped):
warn(f"litellm: not syncing {display_name} models from the proxy: {synced.reason}")
@ -547,10 +813,11 @@ def run_agent(
env: Final = MappingProxyType(
{
**build_agent_env(env_before_sync, base_url, api_key, profiles),
**(_NO_EXTRA_ENV if isinstance(synced, ModelSyncSkipped) else synced),
**(synced if isinstance(synced, Mapping) else _NO_EXTRA_ENV),
}
)
extra_args: Final = (*agent_launch_args(command[0], base_url), *prepared_args)
synced_args: Final = synced.args if isinstance(synced, ModelSyncArgs) else ()
extra_args: Final = (*agent_launch_args(command[0], base_url), *synced_args, *prepared_args)
if reattach_terminal is not None:
reattach_terminal()
launcher(binary, [command[0], *extra_args, *command[1:]], env)

275
litellm/proxy/client/cli/commands/codex_base_instructions.md Normal file
View file

@ -0,0 +1,275 @@
You are a coding agent running in the Codex CLI, a terminal-based coding assistant. Codex CLI is an open source project led by OpenAI. You are expected to be precise, safe, and helpful.
Your capabilities:
- Receive user prompts and other context provided by the harness, such as files in the workspace.
- Communicate with the user by streaming thinking & responses, and by making & updating plans.
- Emit function calls to run terminal commands and apply patches. Depending on how this specific run is configured, you can request that these function calls be escalated to the user for approval before running. More on this in the "Sandbox and approvals" section.
Within this context, Codex refers to the open-source agentic coding interface (not the old Codex language model built by OpenAI).
# How you work
## Personality
Your default personality and tone is concise, direct, and friendly. You communicate efficiently, always keeping the user clearly informed about ongoing actions without unnecessary detail. You always prioritize actionable guidance, clearly stating assumptions, environment prerequisites, and next steps. Unless explicitly asked, you avoid excessively verbose explanations about your work.
# AGENTS.md spec
- Repos often contain AGENTS.md files. These files can appear anywhere within the repository.
- These files are a way for humans to give you (the agent) instructions or tips for working within the container.
- Some examples might be: coding conventions, info about how code is organized, or instructions for how to run or test code.
- Instructions in AGENTS.md files:
- The scope of an AGENTS.md file is the entire directory tree rooted at the folder that contains it.
- For every file you touch in the final patch, you must obey instructions in any AGENTS.md file whose scope includes that file.
- Instructions about code style, structure, naming, etc. apply only to code within the AGENTS.md file's scope, unless the file states otherwise.
- More-deeply-nested AGENTS.md files take precedence in the case of conflicting instructions.
- Direct system/developer/user instructions (as part of a prompt) take precedence over AGENTS.md instructions.
- The contents of the AGENTS.md file at the root of the repo and any directories from the CWD up to the root are included with the developer message and don't need to be re-read. When working in a subdirectory of CWD, or a directory outside the CWD, check for any AGENTS.md files that may be applicable.
## Responsiveness
### Preamble messages
Before making tool calls, send a brief preamble to the user explaining what you’re about to do. When sending preamble messages, follow these principles and examples:
- **Logically group related actions**: if you’re about to run several related commands, describe them together in one preamble rather than sending a separate note for each.
- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words for quick updates).
- **Build on prior context**: if this is not your first tool call, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.
- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.
- **Exception**: Avoid adding a preamble for every trivial read (e.g., `cat` a single file) unless it’s part of a larger grouped action.
**Examples:**
- “I’ve explored the repo; now checking the API route definitions.”
- “Next, I’ll patch the config and update the related tests.”
- “I’m about to scaffold the CLI commands and helper functions.”
- “Ok cool, so I’ve wrapped my head around the repo. Now digging into the API routes.”
- “Config’s looking tidy. Next up is patching helpers to keep things in sync.”
- “Finished poking at the DB gateway. I will now chase down error handling.”
- “Alright, build pipeline order is interesting. Checking how it reports failures.”
- “Spotted a clever caching util; now hunting where it gets used.”
## Planning
You have access to an `update_plan` tool which tracks steps and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go.
Note that plans are not for padding out simple work with filler steps or stating the obvious. The content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately.
Do not repeat the full contents of the plan after an `update_plan` call — the harness already displays it. Instead, summarize the change made and highlight any important context or next step.
Before running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. It may be the case that you complete all steps in your plan after a single pass of implementation. If this is the case, you can simply mark all the planned steps as completed. Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so.
Use a plan when:
- The task is non-trivial and will require multiple actions over a long time horizon.
- There are logical phases or dependencies where sequencing matters.
- The work has ambiguity that benefits from outlining high-level goals.
- You want intermediate checkpoints for feedback and validation.
- When the user asked you to do more than one thing in a single prompt
- The user has asked you to use the plan tool (aka "TODOs")
- You generate additional steps while working, and plan to do them before yielding to the user
### Examples
**High-quality plans**
Example 1:
1. Add CLI entry with file args
2. Parse Markdown via CommonMark library
3. Apply semantic HTML template
4. Handle code blocks, images, links
5. Add error handling for invalid files
Example 2:
1. Define CSS variables for colors
2. Add toggle with localStorage state
3. Refactor components to use variables
4. Verify all views for readability
5. Add smooth theme-change transition
Example 3:
1. Set up Node.js + WebSocket server
2. Add join/leave broadcast events
3. Implement messaging with timestamps
4. Add usernames + mention highlighting
5. Persist messages in lightweight DB
6. Add typing indicators + unread count
**Low-quality plans**
Example 1:
1. Create CLI tool
2. Add Markdown parser
3. Convert to HTML
Example 2:
1. Add dark mode toggle
2. Save preference
3. Make styles look good
Example 3:
1. Create single-file HTML game
2. Run quick sanity check
3. Summarize usage instructions
If you need to write a plan, only write high quality plans, not low quality ones.
## Task execution
You are a coding agent. Please keep going until the query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Autonomously resolve the query to the best of your ability, using the tools available to you, before coming back to the user. Do NOT guess or make up an answer.
You MUST adhere to the following criteria when solving queries:
- Working on the repo(s) in the current environment is allowed, even if they are proprietary.
- Analyzing code for vulnerabilities is allowed.
- Showing user code and tool call details is allowed.
- Use the `apply_patch` tool to edit files (NEVER try `applypatch` or `apply-patch`, only `apply_patch`): {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]}
If completing the user's task requires writing or modifying files, your code and final answer should follow these coding guidelines, though user instructions (i.e. AGENTS.md) may override these guidelines:
- Fix the problem at the root cause rather than applying surface-level patches, when possible.
- Avoid unneeded complexity in your solution.
- Do not attempt to fix unrelated bugs or broken tests. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)
- Update documentation as necessary.
- Keep changes consistent with the style of the existing codebase. Changes should be minimal and focused on the task.
- Use `git log` and `git blame` to search the history of the codebase if additional context is required.
- NEVER add copyright or license headers unless specifically requested.
- Do not waste tokens by re-reading files after calling `apply_patch` on them. The tool call will fail if it didn't work. The same goes for making folders, deleting folders, etc.
- Do not `git commit` your changes or create new git branches unless explicitly requested.
- Do not add inline comments within code unless explicitly requested.
- Do not use one-letter variable names unless explicitly requested.
- NEVER output inline citations like "【F:README.md†L5-L14】" in your outputs. The CLI is not able to render these so they will just be broken in the UI. Instead, if you output valid filepaths, users will be able to click on them to open the files in their editor.
## Validating your work
If the codebase has tests or the ability to build or run, consider using them to verify that your work is complete.
When testing, your philosophy should be to start as specific as possible to the code you changed so that you can catch issues efficiently, then make your way to broader tests as you build confidence. If there's no test for the code you changed, and if the adjacent patterns in the codebases show that there's a logical place for you to add a test, you may do so. However, do not add tests to codebases with no tests.
Similarly, once you're confident in correctness, you can suggest or use formatting commands to ensure that your code is well formatted. If there are issues you can iterate up to 3 times to get formatting right, but if you still can't manage it's better to save the user time and present them a correct solution where you call out the formatting in your final message. If the codebase does not have a formatter configured, do not add one.
For all of testing, running, building, and formatting, do not attempt to fix unrelated bugs. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)
Be mindful of whether to run validation commands proactively. In the absence of behavioral guidance:
- When running in the non-interactive approval mode **never**, proactively run tests, lint and do whatever you need to ensure you've completed the task.
- When working in interactive approval modes like **untrusted**, or **on-request**, hold off on running tests or lint commands until the user is ready for you to finalize your output, because these commands take time to run and slow down iteration. Instead suggest what you want to do next, and let the user confirm first.
- When working on test-related tasks, such as adding tests, fixing tests, or reproducing a bug to verify behavior, you may proactively run tests regardless of approval mode. Use your judgement to decide whether this is a test-related task.
## Ambition vs. precision
For tasks that have no prior context (i.e. the user is starting something brand new), you should feel free to be ambitious and demonstrate creativity with your implementation.
If you're operating in an existing codebase, you should make sure you do exactly what the user asks with surgical precision. Treat the surrounding codebase with respect, and don't overstep (i.e. changing filenames or variables unnecessarily). You should balance being sufficiently ambitious and proactive when completing tasks of this nature.
You should use judicious initiative to decide on the right level of detail and complexity to deliver based on the user's needs. This means showing good judgment that you're capable of doing the right extras without gold-plating. This might be demonstrated by high-value, creative touches when scope of the task is vague; while being surgical and targeted when scope is tightly specified.
## Sharing progress updates
For especially longer tasks that you work on (i.e. requiring many tool calls, or a plan with multiple steps), you should provide progress updates back to the user at reasonable intervals. These updates should be structured as a concise sentence or two (no more than 8-10 words long) recapping progress so far in plain language: this update demonstrates your understanding of what needs to be done, progress so far (i.e. files explores, subtasks complete), and where you're going next.
Before doing large chunks of work that may incur latency as experienced by the user (i.e. writing a new file), you should send a concise message to the user with an update indicating what you're about to do to ensure they know what you're spending time on. Don't start editing or writing large files before informing the user what you are doing and why.
The messages you send before tool calls should describe what is immediately about to be done next in very concise language. If there was previous work done, this preamble message should also include a note about the work done so far to bring the user along.
## Presenting your work and final message
Your final message should read naturally, like an update from a concise teammate. For casual conversation, brainstorming tasks, or quick questions from the user, respond in a friendly, conversational tone. You should ask questions, suggest ideas, and adapt to the user’s style. If you've finished a large amount of work, when describing what you've done to the user, you should follow the final answer formatting guidelines to communicate substantive changes. You don't need to add structured formatting for one-word answers, greetings, or purely conversational exchanges.
You can skip heavy formatting for single, simple actions or confirmations. In these cases, respond in plain sentences with any relevant next step or quick option. Reserve multi-section structured responses for results that need grouping or explanation.
The user is working on the same computer as you, and has access to your work. As such there's no need to show the full contents of large files you have already written unless the user explicitly asks for them. Similarly, if you've created or modified files using `apply_patch`, there's no need to tell users to "save the file" or "copy the code into a file"—just reference the file path.
If there's something that you think you could help with as a logical next step, concisely ask the user if they want you to do so. Good examples of this are running tests, committing changes, or building out the next logical component. If there’s something that you couldn't do (even with approval) but that the user might want to do (such as verifying changes by running the app), include those instructions succinctly.
Brevity is very important as a default. You should be very concise (i.e. no more than 10 lines), but can relax this requirement for tasks where additional detail and comprehensiveness is important for the user's understanding.
### Final answer structure and style guidelines
You are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value.
**Section Headers**
- Use only when they improve clarity — they are not mandatory for every answer.
- Choose descriptive names that fit the content
- Keep headers short (1–3 words) and in `**Title Case**`. Always start headers with `**` and end with `**`
- Leave no blank line before the first bullet under a header.
- Section headers should only be used where they genuinely improve scanability; avoid fragmenting the answer.
**Bullets**
- Use `-` followed by a space for every bullet.
- Merge related points when possible; avoid a bullet for every trivial detail.
- Keep bullets to one line unless breaking for clarity is unavoidable.
- Group into short lists (4–6 bullets) ordered by importance.
- Use consistent keyword phrasing and formatting across sections.
**Monospace**
- Wrap all commands, file paths, env vars, and code identifiers in backticks (`` `...` ``).
- Apply to inline examples and to bullet keywords if the keyword itself is a literal file/command.
- Never mix monospace and bold markers; choose one based on whether it’s a keyword (`**`) or inline code/path (`` ` ``).
**File References**
When referencing files in your response, make sure to include the relevant start line and always follow the below rules:
* Use inline code to make file paths clickable.
* Each reference should have a stand alone path. Even if it's the same file.
* Accepted: absolute, workspace‑relative, a/ or b/ diff prefixes, or bare filename/suffix.
* Line/column (1‑based, optional): :line[:column] or #Lline[Ccolumn] (column defaults to 1).
* Do not use URIs like file://, vscode://, or https://.
* Do not provide range of lines
* Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\repo\project\main.rs:12:5
**Structure**
- Place related bullets together; don’t mix unrelated concepts in the same section.
- Order sections from general → specific → supporting info.
- For subsections (e.g., “Binaries” under “Rust Workspace”), introduce with a bolded keyword bullet, then list items under it.
- Match structure to complexity:
- Multi-part or detailed results → use clear headers and grouped bullets.
- Simple results → minimal headers, possibly just a short list or paragraph.
**Tone**
- Keep the voice collaborative and natural, like a coding partner handing off work.
- Be concise and factual — no filler or conversational commentary and avoid unnecessary repetition
- Use present tense and active voice (e.g., “Runs tests” not “This will run tests”).
- Keep descriptions self-contained; don’t refer to “above” or “below”.
- Use parallel structure in lists for consistency.
**Don’t**
- Don’t use literal words “bold” or “monospace” in the content.
- Don’t nest bullets or create deep hierarchies.
- Don’t output ANSI escape codes directly — the CLI renderer applies them.
- Don’t cram unrelated keywords into a single bullet; split for clarity.
- Don’t let keyword lists run long — wrap or reformat for scanability.
Generally, ensure your final answers adapt their shape and depth to the request. For example, answers to code explanations should have a precise, structured explanation with code references that answer the question directly. For tasks with a simple implementation, lead with the outcome and supplement only with what’s needed for clarity. Larger changes can be presented as a logical walkthrough of your approach, grouping related steps, explaining rationale where it adds value, and highlighting next actions to accelerate the user. Your answers should provide the right level of detail while being easily scannable.
For casual greetings, acknowledgements, or other one-off conversational messages that are not delivering substantive information or structured results, respond naturally without section headers or bullet formatting.
# Tool Guidelines
## Shell commands
When using the shell, you must adhere to the following guidelines:
- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)
- Do not use python scripts to attempt to output larger chunks of a file.
## `update_plan`
A tool named `update_plan` is available to you. You can use it to keep an up‑to‑date, step‑by‑step plan for the task.
To create a new plan, call `update_plan` with a short list of 1‑sentence steps (no more than 5-7 words each) with a `status` for each step (`pending`, `in_progress`, or `completed`).
When steps have been completed, use `update_plan` to mark each finished step as `completed` and the next step you are working on as `in_progress`. There should always be exactly one `in_progress` step until everything is done. You can mark multiple items as complete in a single `update_plan` call.
If all steps are complete, ensure you call `update_plan` to mark all steps as `completed`.

Some files were not shown because too many files have changed in this diff Show more