scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 117

merge: origin/main into litellm_mcp_discovery_budget_exempt
Some checks failed
ai-gateway image / ai-gateway release image (push) Has been cancelled
Details
LiteLLM Rust / rust-lint (push) Has been cancelled
Details
LiteLLM Rust / rust-test (push) Has been cancelled
Details
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Details
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Details

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-15 23:53:47 +00:00
commit 8b85bbc812
336 changed files with 18210 additions and 3162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CLAUDE.md
View file

@ -25,6 +25,8 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
Never test structure of code only function of it
A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`

3
Makefile
View file

@ -299,6 +299,9 @@ test-rust-extension:
[ "$$#" -eq 1 ] && \
UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
"$$temporary/venv/bin/python" -I -m mypy.stubtest \
--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
litellm.rust_bridge._native && \
LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.67"
version = "0.1.68"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.67"
version = "0.1.68"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

14
litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql Normal file
View file

@ -0,0 +1,14 @@
-- AlterTable
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;

18
litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- DropIndex
DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_is_act_idx";
-- DropIndex
DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_key";
-- AlterTable
-- NOT NULL DEFAULT '' (not nullable): Postgres unique constraints treat every
-- NULL as distinct, so a nullable column would let multiple unscoped mappings
-- collide on the same claim without a constraint violation. The constant
-- default is a fast, metadata-only backfill for existing rows, not a rewrite.
ALTER TABLE "LiteLLM_JWTKeyMapping" ADD COLUMN IF NOT EXISTS "jwt_issuer" TEXT NOT NULL DEFAULT '';
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_idx" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value", "is_active");
-- CreateIndex
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_key" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value");

13
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_reset_at DateTime?
@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
@ -483,6 +487,10 @@ model LiteLLM_VerificationToken {
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_issuer String @default("") // Scopes the mapping to one configured issuer; "" matches any issuer.
// Not nullable: Postgres unique constraints treat every NULL as
// distinct, so a nullable column would let multiple unscoped
// mappings collide on the same claim without a constraint violation.
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
@ -495,8 +503,8 @@ model LiteLLM_JWTKeyMapping {
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
@@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
@@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
@ -534,6 +542,7 @@ model LiteLLM_DeletedVerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.97"
version = "0.4.98"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.97"
version = "0.4.98"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

1
litellm/__init__.py
View file

@ -343,6 +343,7 @@ _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
"1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
)
openai_system_messages_first: bool = False
disable_vertex_batch_output_transformation: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"

3
litellm/a2a_protocol/main.py
View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -507,7 +508,7 @@ async def asend_message(
prompt_tokens,
completion_tokens,
_,
) = A2ARequestUtils.calculate_usage_from_request_response(
) = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
request=request,
response_dict=response_dict,
)

5
litellm/a2a_protocol/streaming_iterator.py
View file

@ -11,6 +11,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
if TYPE_CHECKING:
@ -99,11 +100,11 @@ class A2AStreamingIterator:
# Calculate tokens from collected text
input_message: Final = A2ARequestUtils.get_input_message_from_request(self.request)
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
# Use the last (most complete) text from chunks
output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
completion_tokens: Final = A2ARequestUtils.count_tokens(output_text)
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
total_tokens: Final = prompt_tokens + completion_tokens

6
litellm/anthropic_beta_headers_config.json
View file

@ -22,6 +22,7 @@
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@ -52,6 +53,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@ -82,6 +84,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@ -113,6 +116,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
@ -144,6 +148,7 @@
"mcp-servers-2025-12-04": null,
"output-128k-2025-02-19": null,
"structured-output-2024-03-01": null,
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": null,
"skills-2025-10-02": null,
"structured-outputs-2025-11-13": null,
@ -176,6 +181,7 @@
"mcp-servers-2025-12-04": null,
"oauth-2025-04-20": "oauth-2025-04-20",
"output-128k-2025-02-19": "output-128k-2025-02-19",
"per-turn-control-2026-07-01": null,
"prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
"skills-2025-10-02": "skills-2025-10-02",
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",

35
litellm/caching/caching_handler.py
View file

@ -35,6 +35,7 @@ from litellm.litellm_core_utils.logging_utils import (
_assemble_complete_response_from_streaming_chunks,
)
from litellm.types.caching import CachedEmbedding
from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
@ -107,17 +108,31 @@ def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
return "choices" in cached_result
def _should_defer_streaming_cache_hit_callbacks(*, kwargs: dict[str, object]) -> bool:
def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool:
if kwargs.get("stream", False) is True:
return True
return converted_stream_requested(kwargs) and not kwargs.get("_agentic_loop_depth")
def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> bool:
"""
When stream=True, do not run success callbacks at cache-hit time.
When the cache hit is replayed as a stream, do not run success callbacks at cache-hit time.
Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
replay uses CachedResponsesAPIStreamingIterator; cached Anthropic Messages
replay uses CachedAnthropicMessagesStreamIterator. All invoke logging success
handlers when the stream finishes; firing them here too would double-count
spend and callback records.
spend and callback records. A plain (non-stream) replay logs here, since nothing
else will.
"""
return kwargs.get("stream", False) is True
from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
CachedAnthropicMessagesStreamIterator,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
return isinstance(
cached_result, (CustomStreamWrapper, BaseResponsesAPIStreamingIterator, CachedAnthropicMessagesStreamIterator)
)
def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]:
@ -267,7 +282,7 @@ class LLMCachingHandler:
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
# LOG SUCCESS
self._async_log_cache_hit_on_callbacks(
logging_obj=logging_obj,
@ -383,7 +398,7 @@ class LLMCachingHandler:
is_async=False,
)
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
@ -823,7 +838,7 @@ class LLMCachingHandler:
if (call_type == CallTypes.acompletion.value or call_type == CallTypes.completion.value) and isinstance(
cached_result, dict
):
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
cached_result = self._convert_cached_stream_response(
cached_result=cached_result,
call_type=call_type,
@ -838,7 +853,7 @@ class LLMCachingHandler:
if (
call_type == CallTypes.atext_completion.value or call_type == CallTypes.text_completion.value
) and isinstance(cached_result, dict):
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
cached_result = self._convert_cached_stream_response(
cached_result=cached_result,
call_type=call_type,
@ -893,7 +908,7 @@ class LLMCachingHandler:
elif (call_type == "aresponses" or call_type == "responses") and isinstance(cached_result, dict):
use_chat_completion_cache: Final = _is_chat_completion_cached_dict(cached_result)
if use_chat_completion_cache:
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
bridge_call_type: Final = (
CallTypes.acompletion.value if call_type == "aresponses" else CallTypes.completion.value
)
@ -921,7 +936,7 @@ class LLMCachingHandler:
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
if _stream_replay_requested(kwargs):
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,

3
litellm/caching/qdrant_semantic_cache.py
View file

@ -21,6 +21,7 @@ from litellm.constants import (
QDRANT_VECTOR_SIZE,
SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -255,7 +256,7 @@ class QdrantSemanticCache(BaseCache):
llm_router = None
router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
embedding_input: Final = self._embedding_input(prompt, router)
embedding_input: Final = await asyncify(self._embedding_input)(prompt, router)
embedding_call: Final = (
router.aembedding(
model=self.embedding_model,

3
litellm/caching/redis_semantic_cache.py
View file

@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -522,7 +523,7 @@ class RedisSemanticCache(BaseCache):
llm_router = None
router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
embedding_input: Final = self._embedding_input(prompt, router)
embedding_input: Final = await asyncify(self._embedding_input)(prompt, router)
embedding_call: Final = (
router.aembedding(
model=self.embedding_model,

19
litellm/compression/compress.py
View file

@ -214,26 +214,33 @@ def _message_has_cache_control(message: Mapping[str, object]) -> bool:
return False
def _cached_prefix_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
last_breakpoint: Final = max(
(index for index, msg in enumerate(messages) if _message_has_cache_control(msg)),
default=-1,
)
return tuple(range(last_breakpoint + 1))
def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
"""
Return indices of messages that must never be compressed:
- All system messages
- The last user message
- The last assistant message
- Any message carrying an Anthropic cache_control breakpoint
- Every message up to and including the last one carrying an Anthropic cache_control breakpoint
The last user message is what the model is being asked to act on right now,
so compressing it replaces the live instruction with a marker. Compression
guardrails share this policy; see the Headroom guardrail. A cache_control
breakpoint pins the provider's prompt-cache prefix to that row's exact
bytes, so rewriting a marked row anywhere in history turns the next
request's cache read into a cache write.
breakpoint pins the provider's prompt-cache prefix to the exact bytes of every
row up to it, so rewriting any row inside that prefix turns the next request's
cache read into a cache write.
"""
system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system")
last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:]
assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")
cache_control_indices: Final = tuple(index for index, msg in enumerate(messages) if _message_has_cache_control(msg))
return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + cache_control_indices))
return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + _cached_prefix_indices(messages)))
def _combine_scores(

6
litellm/constants.py
View file

@ -317,6 +317,8 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model"
MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved"
MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body"
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
@ -1567,6 +1569,8 @@ BASE_MCP_ROUTE: Final = "/mcp"
BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour
BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24)) # for 24 hours
BATCH_TPD_WINDOW_SECONDS: Final = 86400
BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd"
HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)) # 60 seconds
_background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS")
@ -1775,6 +1779,7 @@ DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_
LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"})
LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"default_internal_user_params",
"default_team_params",
@ -1792,6 +1797,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
# test_general_settings_ui_fields_are_db_overridable enforces that pairing.
"enable_anthropic_prompt_caching",
"anthropic_prompt_caching_ttl",
"openai_system_messages_first",
"max_ui_session_budget",
"budget_rollover",
"mcp_tool_search",

1
litellm/cost_calculator.py
View file

@ -814,6 +814,7 @@ def _select_model_name_for_cost_calc(
if (
entry.get("input_cost_per_token") is not None
or entry.get("input_cost_per_second") is not None
or entry.get("input_cost_per_query") is not None
or entry.get("tiered_pricing") is not None
):
return_model = router_model_id

3
litellm/integrations/compression_interception/handler.py
View file

@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
CompressionSavingsMetadata,
@ -153,7 +154,7 @@ class CompressionInterceptionLogger(CustomLogger):
self._prune_expired_cache()
compressed: Final = compress(
compressed: Final = await asyncify(compress)(
messages=messages,
model=model,
call_type=CallTypes.anthropic_messages,

49
litellm/integrations/custom_logger.py
View file

@ -822,46 +822,33 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
def truncate_standard_logging_payload_content(
self,
standard_logging_object: StandardLoggingPayload,
):
) -> StandardLoggingPayload:
"""
Truncate error strings and message content in logging payload
Return a copy of the logging payload with error_str, messages, and response truncated
Some loggers like DataDog/ GCS Bucket have a limit on the size of the payload. (1MB)
This function truncates the error string and the message content if they exceed a certain length.
Every callback of a request shares one standard logging object, so the payload passed in is left
untouched and the callbacks that run later (the prompt caching router check, spend logs) still see
the original fields.
"""
MAX_STR_LENGTH: Final = 10_000
max_str_length: Final = 10_000
candidates: Final = {
field: self._truncate_field(field_value=standard_logging_object.get(field), max_length=max_str_length)
for field in ("error_str", "messages", "response")
}
truncated_fields: Final = {field: text for field, text in candidates.items() if text is not None}
return {**standard_logging_object, **truncated_fields}
# Truncate fields that might exceed max length
fields_to_truncate: Final = ["error_str", "messages", "response"]
for field in fields_to_truncate:
self._truncate_field(
standard_logging_object=standard_logging_object,
field_name=field,
max_length=MAX_STR_LENGTH,
)
def _truncate_field(
self,
standard_logging_object: StandardLoggingPayload,
field_name: str,
max_length: int,
) -> None:
def _truncate_field(self, field_value: object, max_length: int) -> str | None:
"""
Helper function to truncate a field in the logging payload
Return the truncated text of a field that exceeds max_length, or None when the field fits
This converts the field to a string and then truncates it if it exceeds the max length.
Why convert to string ?
1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
- Converting to string and then truncating the logged content catches this
2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
The field is measured as a string because users send poorly formatted lists for `messages`, so there is
no fixed place the content would be.
"""
field_value: Final[object] = standard_logging_object.get(field_name)
if field_value:
str_value: Final = str(field_value)
if len(str_value) > max_length:
standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
text: Final = str(field_value or "")
return self._truncate_text(text=text, max_length=max_length) if len(text) > max_length else None
def _truncate_text(self, text: str, max_length: int) -> str:
"""Truncate text if it exceeds max_length"""

5
litellm/integrations/datadog/datadog.py
View file

@ -563,11 +563,10 @@ class DataDogLogger(
if standard_logging_object.get("status") == "failure":
status = DataDogStatus.ERROR
# Build the initial payload
self.truncate_standard_logging_payload_content(standard_logging_object)
truncated_payload: Final = self.truncate_standard_logging_payload_content(standard_logging_object)
dd_payload: Final = self._create_datadog_logging_payload_helper(
standard_logging_object=standard_logging_object,
standard_logging_object=truncated_payload,
status=status,
)
return dd_payload

2
litellm/integrations/langsmith.py
View file

@ -39,6 +39,8 @@ def is_serializable(value):
class LangsmithLogger(CustomBatchLogger):
preserve_events_added_during_flush = True
def __init__(
self,
langsmith_api_key: str | None = None,

8
litellm/integrations/prometheus.py
View file

@ -2610,12 +2610,6 @@ class PrometheusLogger(CustomLogger):
StandardLoggingPayloadSetup,
)
if self._should_skip_metrics_for_invalid_key(
user_api_key_dict=user_api_key_dict,
exception=original_exception,
):
return
status_code: Final = self._extract_status_code(exception=original_exception)
try:
@ -2633,7 +2627,7 @@ class PrometheusLogger(CustomLogger):
end_user=user_api_key_dict.end_user_id,
user=user_api_key_dict.user_id,
user_email=user_api_key_dict.user_email,
hashed_api_key=user_api_key_dict.api_key,
hashed_api_key=None if status_code == 401 else user_api_key_dict.api_key,
api_key_alias=user_api_key_dict.key_alias,
team=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -41,6 +41,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"azure_password",
"azure_scope",
"timeout",
"client_side_timeout",
"gcs_bucket_name",
"bucket_name",
"vertex_credentials",

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -238,6 +238,8 @@ def get_llm_provider(
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
raise Exception(f"dynamic_api_key needs to be a string. Got type={type(dynamic_api_key).__name__}")
return model, custom_llm_provider, dynamic_api_key, api_base
if "/" in model and is_registered_custom_provider(provider_prefix):
return model.split("/", 1)[1], provider_prefix, dynamic_api_key, api_base
# check if api base is a known openai compatible endpoint
if api_base:
for endpoint in litellm.openai_compatible_endpoints:
@ -536,6 +538,10 @@ def get_llm_provider(
)
def is_registered_custom_provider(custom_llm_provider: str | None) -> bool:
return any(item["provider"] == custom_llm_provider for item in litellm.custom_provider_map)
def _dashscope_family_chat_config(custom_llm_provider: str) -> "litellm.DashScopeChatConfig":
if custom_llm_provider == "qwencloud":
return litellm.QwenCloudChatConfig()

24
litellm/litellm_core_utils/litellm_logging.py
View file

@ -644,6 +644,24 @@ class Logging(LiteLLMLoggingBaseClass):
"""Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
self.response_timing_metrics = dict(timing_metrics) # mutable-ok: kept deep-copyable
def add_dynamic_callback(self, callback: CustomLogger) -> None:
self.dynamic_input_callbacks = self._with_dynamic_callback(self.dynamic_input_callbacks, callback)
self.dynamic_success_callbacks = self._with_dynamic_callback(self.dynamic_success_callbacks, callback)
self.dynamic_async_success_callbacks = self._with_dynamic_callback(
self.dynamic_async_success_callbacks, callback
)
self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, callback)
self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
self.dynamic_async_failure_callbacks, callback
)
@staticmethod
def _with_dynamic_callback(
callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
) -> list[str | Callable | CustomLogger]:
existing: Final = tuple(callbacks or ())
return [*existing, *(() if callback in existing else (callback,))]
def process_dynamic_callbacks(self):
"""
Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks
@ -1973,6 +1991,12 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
if isinstance(usage, Usage):
self.record_partial_usage_for_failure(usage, self._response_cost_calculator(result=assembled) or 0.0)
async def dispatch_failure_handlers(
self,
exception: Exception,

16
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -2256,6 +2256,22 @@ def drop_tool_reference_parts_from_tool_messages(
return [_drop_tool_reference_parts(message) for message in messages] # mutable-ok: pipelines mutate message lists
INSTRUCTION_MESSAGE_ROLES: Final = frozenset({"system", "developer"})
def _is_instruction_message(message: AllMessageValues) -> bool:
return message.get("role") in INSTRUCTION_MESSAGE_ROLES
def system_messages_first(
messages: list[AllMessageValues], # mutable-ok: message pipelines type messages as mutable lists
) -> list[AllMessageValues]: # mutable-ok: message pipelines type messages as mutable lists
return [ # mutable-ok: pipelines mutate message lists
*(message for message in messages if _is_instruction_message(message)),
*(message for message in messages if not _is_instruction_message(message)),
]
def _attempt_json_repair(s: str) -> object | None:
"""
Attempt to repair truncated JSON produced by LLM tool calls.

46
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1500,19 +1500,33 @@ def convert_to_gemini_tool_call_result(
return _part
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""
Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
_TOOL_USE_ID_FALLBACK: Final = "tool_use_id"
_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
_BEDROCK_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_.:-]")
_BEDROCK_TOOL_USE_ID_MAX_LEN: Final = 64
_BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8
Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
This function replaces any invalid characters with underscores.
def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str:
return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$."""
return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS)
def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
"""
# Replace any character that's not alphanumeric, underscore, or hyphen with underscore
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
# Ensure it's not empty (fallback to a default if needed)
if not sanitized:
sanitized = "tool_use_id"
return sanitized
Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars.
Ids that need rewriting get a short hash of the original appended so two ids that only
differ in a replaced char or past the cut still map to distinct values.
"""
sanitized: Final = _replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS)
if sanitized == tool_use_id and len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN:
return sanitized
digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN]
return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}"
_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
@ -3661,7 +3675,9 @@ def _convert_to_bedrock_tool_call_invoke(
if parsed_objects:
# First object keeps the original tool id.
for obj_idx, obj in enumerate(parsed_objects):
block_id = tool_id if obj_idx == 0 else f"{tool_id}_{obj_idx}"
block_id = _sanitize_bedrock_tool_use_id(
tool_id if obj_idx == 0 else f"{tool_id}_{obj_idx}"
)
bedrock_tool = BedrockToolUseBlock(input=obj, name=name, toolUseId=block_id)
_parts_list.append(BedrockContentBlock(toolUse=bedrock_tool))
# cache_control applies to the whole original
@ -3678,7 +3694,9 @@ def _convert_to_bedrock_tool_call_invoke(
# Fallback: no objects extracted — use empty dict.
arguments_dict = {}
bedrock_tool = BedrockToolUseBlock(input=arguments_dict, name=name, toolUseId=tool_id)
bedrock_tool = BedrockToolUseBlock(
input=arguments_dict, name=name, toolUseId=_sanitize_bedrock_tool_use_id(tool_id)
)
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
_parts_list.append(bedrock_content_block)
@ -3849,7 +3867,7 @@ def _convert_to_bedrock_tool_call_result(
tool_result_content_blocks, used_search_results = _build_bedrock_tool_result_content_blocks(message)
message.get("name", "")
id: Final = str(message.get("tool_call_id", str(uuid.uuid4())))
id: Final = _sanitize_bedrock_tool_use_id(str(message.get("tool_call_id", str(uuid.uuid4()))))
tool_result: Final = BedrockToolResultBlock(content=tool_result_content_blocks, toolUseId=id)
if used_search_results:

3
litellm/litellm_core_utils/streaming_handler.py
View file

@ -19,6 +19,7 @@ from typing_extensions import NotRequired, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
@ -2247,7 +2248,7 @@ class CustomStreamWrapper:
if self.sent_last_chunk is True:
# log the final chunk with accurate streaming values
try:
complete_streaming_response = litellm.stream_chunk_builder(
complete_streaming_response = await asyncify(litellm.stream_chunk_builder)(
chunks=self.chunks,
messages=self.messages,
logging_obj=self.logging_obj,

7
litellm/llms/anthropic/common_utils.py
View file

@ -539,6 +539,13 @@ class AnthropicModelInfo(BaseLLMModelInfo):
value: Final = litellm.model_cost.get(model, {}).get(key)
return value if isinstance(value, bool) else None
@staticmethod
def supports_fast_mode(model: str, custom_llm_provider: str) -> bool:
return (
custom_llm_provider == "anthropic"
and AnthropicModelInfo._get_exact_model_capability(model, "supports_fast_mode") is True
)
@staticmethod
def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> bool | None:
"""Resolve boolean capability ``key`` for ``model`` under the caller's provider.

9
litellm/llms/anthropic/experimental_pass_through/context_management/dispatcher.py
View file

@ -5,6 +5,7 @@ from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Final, TypeAlias
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import AppliedEdit
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
@ -82,9 +83,9 @@ async def apply_context_management(
"""Run edits in order; return a single ``PolyfillResult``.
The dispatcher is async so async editors (``compact_20260112``) can
``await`` the configured summarization model. Sync editors are called
inline — ``inspect.iscoroutinefunction`` decides how each editor is
invoked.
``await`` the configured summarization model. Sync editors run in a
worker thread so their token counts stay off the event loop;
``inspect.iscoroutinefunction`` decides how each editor is invoked.
"""
edits: Final = _normalize_spec(context_management_spec)
if not edits:
@ -121,7 +122,7 @@ async def apply_context_management(
user_api_key_auth=user_api_key_auth,
)
if editor_is_async
else editor(
else await asyncify(editor)(
model=model,
messages=current_messages,
tools=tools,

3
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -20,6 +20,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import (
AppliedEdit,
CompactionBlock,
@ -1157,7 +1158,7 @@ async def apply_compact_20260112(
# Phase B: threshold check.
try:
current_tokens = _count_effective_tokens(
current_tokens = await asyncify(_count_effective_tokens)(
model=model,
effective_messages=effective_messages,
# ``augmented_system`` already carries the prior compaction summary

2
litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
View file

@ -678,7 +678,7 @@ class BaseAnthropicMessagesStreamingIterator:
"""
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler
PassThroughStreamingHandler.schedule_stream_failure_logging(
await PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=self.litellm_logging_obj,
endpoint_type=EndpointType.ANTHROPIC,
request_body=self.request_body,

52
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -42,6 +42,10 @@ DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
)
def _messages_carry_output_config(messages: Sequence[object]) -> bool:
return any(isinstance(message, Mapping) and "output_config" in message for message in messages)
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
@property
def custom_llm_provider(self) -> str | None:
@ -331,6 +335,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=messages,
)
return headers, api_base
@ -664,6 +669,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
headers: dict,
optional_params: dict,
custom_llm_provider: str = "anthropic",
messages: Sequence[object] = (),
) -> dict:
"""
Auto-inject anthropic-beta headers based on features used.
@ -673,24 +679,30 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
- tool_search: adds provider-specific tool search header
- output_format: adds 'structured-outputs-2025-11-13'
- speed: adds 'fast-mode-2026-02-01'
- a message carrying output_config: adds 'per-turn-control-2026-07-01'
Args:
headers: Request headers dict
optional_params: Optional parameters including tools, context_management, output_format, speed
custom_llm_provider: Provider name for looking up correct tool search header
messages: Request messages, scanned for per-message output_config
"""
beta_values: Final[set] = set()
# Get existing beta headers if any
existing_beta: Final = headers.get("anthropic-beta")
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
existing_beta: Final = tuple(
piece.strip()
for key, value in headers.items()
if key.lower() == "anthropic-beta"
for piece in value.split(",")
if piece.strip()
)
beta_values.update(existing_beta)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:
# Check edits array for compact_20260112 type
edits: Final = context_management_param.get("edits", [])
edits: Final = context_management_param.get("edits", ())
has_compact = False
has_other = False
@ -722,24 +734,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if optional_params.get("speed") == "fast":
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
# Check for advisor tool
tools = optional_params.get("tools")
if tools:
for tool in tools:
if isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE:
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
break
if _messages_carry_output_config(messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
# Check for tool search tools
tools = optional_params.get("tools")
if tools:
anthropic_model_info: Final = AnthropicModelInfo()
if anthropic_model_info.is_tool_search_used(tools):
# Use provider-specific tool search header
tool_search_header: Final = get_tool_search_beta_header(custom_llm_provider)
beta_values.add(tool_search_header)
tools: Final = optional_params.get("tools")
if any(isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for tool in tools or ()):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
if beta_values:
headers["anthropic-beta"] = ",".join(sorted(beta_values))
if AnthropicModelInfo().is_tool_search_used(tools):
beta_values.add(get_tool_search_beta_header(custom_llm_provider))
return headers
if not beta_values:
return headers
merged: Final = {key: value for key, value in headers.items() if key.lower() != "anthropic-beta"}
merged["anthropic-beta"] = ",".join(sorted(beta_values))
return merged

4
litellm/llms/azure/chat/gpt_transformation.py
View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_tool_reference_parts_from_tool_messages,
flatten_combinators_and_drop_non_python_regex_patterns,
hoist_images_from_tool_messages,
system_messages_first,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -276,7 +277,8 @@ class AzureOpenAIConfig(BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
return {
"model": model,

1
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -68,6 +68,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=messages,
)
return headers, api_base

1
litellm/llms/bedrock/claude_platform/messages_transformation.py
View file

@ -46,6 +46,7 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
headers = self._update_headers_with_anthropic_beta(
headers=headers,
optional_params=optional_params,
messages=messages,
)
return headers, api_base

1
litellm/llms/deepseek/messages/transformation.py
View file

@ -66,6 +66,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
headers=headers,
optional_params=optional_params,
custom_llm_provider=self.custom_llm_provider or "deepseek",
messages=messages,
)
return headers, api_base

3
litellm/llms/fireworks_ai/rerank/transformation.py
View file

@ -250,8 +250,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig):
rerank_results.append(rerank_result)
# Use model name as id if no id is provided
response_id: Final = raw_response_json.get("id") or raw_response_json.get("model") or str(uuid.uuid4())
response_id: Final = raw_response_json.get("id") or str(uuid.uuid4())
return RerankResponse(
id=response_id,

2
litellm/llms/github_copilot/messages/transformation.py
View file

@ -92,7 +92,7 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig):
headers["anthropic-version"] = "2023-06-01"
headers = self._update_headers_with_anthropic_beta(
headers, optional_params, custom_llm_provider="github_copilot"
headers, optional_params, custom_llm_provider="github_copilot", messages=messages
)
return headers, dynamic_api_base

19
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -12,6 +12,7 @@ from urllib.parse import urlparse
import httpx
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_extract_reasoning_content,
@ -24,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
flatten_combinators_and_drop_non_python_regex_patterns,
get_tool_call_names,
hoist_images_from_tool_messages,
system_messages_first,
tool_with_sanitized_parameters,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -463,6 +465,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
]
return MappingProxyType({"tools": sanitized})
def _prompt_cache_ordered_messages(
self, messages: list[AllMessageValues], litellm_params: Mapping[str, object]
) -> list[AllMessageValues]:
if not litellm.openai_system_messages_first:
return messages
if litellm_params.get("custom_llm_provider") not in OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS:
return messages
return system_messages_first(messages)
def transform_request(
self,
model: str,
@ -477,7 +488,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
Returns:
dict: The transformed request. Sent as the body of the API call.
"""
messages = self._transform_messages(messages=messages, model=model)
messages = self._transform_messages(
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model
)
if not self._should_preserve_cache_control_for_endpoint(
litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
):
@ -506,7 +519,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
litellm_params: dict,
headers: dict,
) -> dict:
transformed_messages = await self._transform_messages(messages=messages, model=model, is_async=True)
transformed_messages = await self._transform_messages(
messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
)
if not self._should_preserve_cache_control_for_endpoint(
litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
):

1
litellm/llms/openai_like/messages/transformation.py
View file

@ -56,6 +56,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
merged: Final = self._update_headers_with_anthropic_beta(
headers=normalized,
optional_params=optional_params,
messages=messages,
)
return merged, api_base

11
litellm/llms/vertex_ai/rerank/transformation.py
View file

@ -4,6 +4,8 @@ Translates from Cohere's `/v1/rerank` input format to Vertex AI Discovery Engine
Why separate file? Make it easy to see how transformation works
"""
import math
import uuid
from collections.abc import Mapping
from typing import Any, Final
@ -32,6 +34,8 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
Reference: https://cloud.google.com/generative-ai-app-builder/docs/ranking#rank_or_rerank_a_set_of_records_according_to_a_query
"""
MAX_RECORDS_PER_SEARCH_UNIT = 100
def __init__(self) -> None:
super().__init__()
@ -208,10 +212,11 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
RerankResponseResult(index=result["index"], relevance_score=result["relevance_score"])
)
# Create meta object
meta: Final = RerankResponseMeta(billed_units=RerankBilledUnits(search_units=len(records)))
input_record_count: Final = len(request_data.get("records", ()))
search_units: Final = math.ceil(input_record_count / self.MAX_RECORDS_PER_SEARCH_UNIT)
meta: Final = RerankResponseMeta(billed_units=RerankBilledUnits(search_units=search_units))
return RerankResponse(id=f"vertex_ai_rerank_{model}", results=rerank_results, meta=meta)
return RerankResponse(id=f"vertex_ai_rerank_{uuid.uuid4()}", results=rerank_results, meta=meta)
def get_supported_cohere_rerank_params(self, model: str) -> list:
return [

3
litellm/llms/voyage/rerank/transformation.py
View file

@ -9,6 +9,7 @@ from typing import Any, Final
import httpx
from litellm._uuid import uuid
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.secret_managers.main import get_secret_str
@ -127,7 +128,7 @@ class VoyageRerankConfig(BaseRerankConfig):
rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)
return RerankResponse(
id=_json_response.get("id", f"voyage-rerank-{model}"),
id=_json_response.get("id") or str(uuid.uuid4()),
results=transformed_results,
meta=rerank_meta,
)

2
litellm/llms/watsonx/rerank/transformation.py
View file

@ -191,7 +191,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig):
transformed_results.append(transformed_result)
response_id: Final = raw_response_json.get("id") or raw_response_json.get("model_id") or str(uuid.uuid4())
response_id: Final = raw_response_json.get("id") or str(uuid.uuid4())
# Extract usage information
_tokens: Final = RerankTokens(

18
litellm/llms/xai/chat/transformation.py
View file

@ -219,13 +219,19 @@ class XAIChatConfig(OpenAIGPTConfig):
litellm_params: dict,
headers: dict,
) -> dict:
"""
Handle https://github.com/BerriAI/litellm/issues/9720
"""Handle https://github.com/BerriAI/litellm/issues/9720"""
if "web_search_options" in optional_params:
verbose_logger.warning(
"XAI no longer supports web search on /chat/completions (Live Search is deprecated). "
"Dropping 'web_search_options'. Use the Responses API for XAI web search."
)
Filter out 'name' from messages
"""
messages = strip_name_from_messages(messages)
return super().transform_request(model, messages, optional_params, litellm_params, headers)
chat_params: Final = { # mutable-ok: base transform_request takes a plain dict of optional params
key: value for key, value in optional_params.items() if key != "web_search_options"
}
return super().transform_request(
model, strip_name_from_messages(messages), chat_params, litellm_params, headers
)
@staticmethod
def _fix_choice_finish_reason_for_tool_calls(choice: Choices) -> None:

22
litellm/llms/xai/responses/transformation.py
View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -32,6 +33,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_STR_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _usage_restated_from_xai_ticks(usage: ResponseAPIUsage | None) -> ResponseAPIUsage | None:
reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None))
@ -81,30 +84,25 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- enable_image_understanding
XAI does NOT support search_context_size (OpenAI-specific).
Domains may come nested under 'filters' (the OpenAI/XAI documented shape) or flat on the tool.
"""
xai_tool: Final[dict[str, object]] = {"type": "web_search"}
# Remove search_context_size if present (not supported by XAI)
if "search_context_size" in tool:
verbose_logger.info(
"XAI does not support 'search_context_size' parameter. Removing it from web_search tool."
)
# Handle filters (XAI-specific structure)
filters: Final = {}
if "allowed_domains" in tool:
allowed_domains: Final = tool["allowed_domains"]
filters["allowed_domains"] = allowed_domains
nested_filters: Final = tool.get("filters")
domains: Final = (
_STR_MAPPING_ADAPTER.validate_python(nested_filters) if isinstance(nested_filters, Mapping) else tool
)
filters: Final = {key: domains[key] for key in ("allowed_domains", "excluded_domains") if key in domains}
if "excluded_domains" in tool:
excluded_domains: Final = tool["excluded_domains"]
filters["excluded_domains"] = excluded_domains
# Add filters if any were specified
if filters:
xai_tool["filters"] = filters
# Handle enable_image_understanding (top-level in XAI format)
if "enable_image_understanding" in tool:
xai_tool["enable_image_understanding"] = tool["enable_image_understanding"]

12
litellm/main.py
View file

@ -67,7 +67,7 @@ from litellm.constants import (
)
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.asyncify import asyncify, run_async_function
from litellm.litellm_core_utils.audio_utils.utils import (
calculate_request_duration,
get_audio_file_for_health_check,
@ -1072,10 +1072,6 @@ def responses_api_bridge_check(
mode = "responses"
model_info["mode"] = mode
if web_search_options is not None and custom_llm_provider == "xai":
model_info["mode"] = "responses"
model = model.replace("responses/", "")
except Exception as e:
verbose_logger.debug("Error getting model info: %s", e)
@ -1084,6 +1080,10 @@ def responses_api_bridge_check(
mode = "responses"
model_info["mode"] = mode
if web_search_options is not None and custom_llm_provider == "xai":
model_info["mode"] = "responses"
model = model.replace("responses/", "")
# OpenAI/Azure GPT-5 chat-completions that need Responses-only fields (e.g.
# ``reasoningSummary`` in ``extra_body``) must be bridged; Chat Completions rejects
# those keys.
@ -9127,7 +9127,7 @@ async def acount_tokens(
fallback_messages = messages or []
if system and fallback_messages:
fallback_messages = [{"role": "system", "content": system}] + fallback_messages
local_count: Final = litellm.token_counter(
local_count: Final = await asyncify(litellm.token_counter)(
model=model,
messages=fallback_messages,
tools=tools,

1127
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

1
litellm/models/budget.py
View file

@ -26,6 +26,7 @@ class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase):
max_parallel_requests: int | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
model_max_budget: dict | None = None
budget_duration: str | None = None
allowed_models: list[str] | None = None # per-member model scope; empty = inherit team models

1
litellm/models/team.py
View file

@ -71,6 +71,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
budget_duration: str | None = None

1
litellm/models/verification_token.py
View file

@ -31,6 +31,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
metadata: dict = {}
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
allowed_cache_controls: list | None = []

72
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -12968,18 +12968,24 @@
"PHONE_NUMBER",
"MEDICAL_LICENSE",
"URL",
"MAC_ADDRESS",
"UUID",
"US_BANK_NUMBER",
"US_DRIVER_LICENSE",
"US_ITIN",
"US_PASSPORT",
"US_SSN",
"US_MBI",
"US_NPI",
"UK_NHS",
"UK_NINO",
"UK_PASSPORT",
"UK_POSTCODE",
"UK_VEHICLE_REGISTRATION",
"UK_DRIVING_LICENCE",
"ES_NIF",
"ES_NIE",
"ES_PASSPORT",
"IT_FISCAL_CODE",
"IT_DRIVER_LICENSE",
"IT_VAT_CODE",
@ -12997,7 +13003,38 @@
"IN_VEHICLE_REGISTRATION",
"IN_VOTER",
"IN_PASSPORT",
"FI_PERSONAL_IDENTITY_CODE"
"IN_GSTIN",
"FI_PERSONAL_IDENTITY_CODE",
"DE_TAX_ID",
"DE_TAX_NUMBER",
"DE_VAT_ID",
"DE_PASSPORT",
"DE_ID_CARD",
"DE_FUEHRERSCHEIN",
"DE_SOCIAL_SECURITY",
"DE_HEALTH_INSURANCE",
"DE_LANR",
"DE_BSNR",
"DE_KFZ",
"DE_HANDELSREGISTER",
"DE_PLZ",
"KR_RRN",
"KR_FRN",
"KR_PASSPORT",
"KR_DRIVER_LICENSE",
"KR_BRN",
"CA_SIN",
"SE_PERSONNUMMER",
"SE_ORGANISATIONSNUMMER",
"TH_TNIN",
"TR_NATIONAL_ID",
"TR_LICENSE_PLATE",
"NG_NIN",
"NG_VEHICLE_REGISTRATION",
"PH_TIN",
"PH_UMID",
"PH_PASSPORT",
"ZA_ID_NUMBER"
],
"title": "PiiEntityType",
"type": "string"
@ -15189,6 +15226,17 @@
"title": "Jwt Claim Value",
"type": "string"
},
"jwt_issuer": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Jwt Issuer"
},
"key": {
"title": "Key",
"type": "string"
@ -15273,6 +15321,17 @@
"title": "Jwt Claim Value",
"type": "string"
},
"jwt_issuer": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Jwt Issuer"
},
"updated_at": {
"format": "date-time",
"title": "Updated At",
@ -15329,6 +15388,17 @@
],
"title": "Is Active"
},
"jwt_issuer": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"title": "Jwt Issuer"
},
"key": {
"anyOf": [
{

22
litellm/proxy/_types.py
View file

@ -286,6 +286,7 @@ class KeyManagementRoutes(str, enum.Enum):
# team's `team_member_permissions`, non-admin members of that team may set
# `access_group_ids` on keys they create/update. Default-deny.
KEY_ACCESS_GROUP_ASSIGNMENT = "/key/access_group_assignment"
AUTO_ROUTER_MANAGE = "/auto_router/manage"
# info and health routes
KEY_INFO = "/key/info"
@ -652,6 +653,7 @@ class LiteLLMRoutes(enum.Enum):
KeyManagementRoutes.KEY_RESET_SPEND.value,
KeyManagementRoutes.KEY_ALIASES.value,
KeyManagementRoutes.KEY_ACCESS_GROUP_ASSIGNMENT.value,
KeyManagementRoutes.AUTO_ROUTER_MANAGE.value,
]
management_routes = (
@ -1204,6 +1206,7 @@ class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase):
class KeyRequestBase(GenerateRequestBase):
key: str | None = None
tpd_limit: int | None = None
default_estimated_output_tokens: PositiveInt | None = None
default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None
budget_id: str | None = None
@ -1889,6 +1892,9 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
)
tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.")
rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.")
tpd_limit: int | None = Field(
default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id."
)
budget_duration: str | None = Field(
default=None,
description="Max duration budget should be set for (e.g. '1hr', '1d', '28d')",
@ -2065,6 +2071,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
models: list | None = None
@ -3020,6 +3027,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
team_alias: str | None = None
team_tpm_limit: int | None = None
team_rpm_limit: int | None = None
team_tpd_limit: int | None = None
team_max_budget: float | None = None
team_soft_budget: float | None = None
team_models: list = []
@ -3039,6 +3047,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
end_user_id: str | None = None
end_user_tpm_limit: int | None = None
end_user_rpm_limit: int | None = None
end_user_tpd_limit: int | None = None
end_user_max_budget: float | None = None
end_user_model_max_budget: dict | None = None
@ -3837,6 +3846,7 @@ class SpendLogsMetadata(TypedDict):
user_api_key_team_alias: str | None
spend_logs_metadata: dict | None # special param to log k,v pairs to spendlogs for a call
requester_ip_address: str | None
user_agent: ReadOnly[str | None]
litellm_call_id: str | None
applied_guardrails: list[str] | None
mcp_tool_call_metadata: StandardLoggingMCPToolCall | None
@ -4476,12 +4486,14 @@ class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
jwt_claim_name: str
jwt_claim_value: str
key: str
jwt_issuer: str | None = None
description: str | None = None
class UpdateJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
id: str
key: str | None = None
jwt_issuer: str | None = None
description: str | None = None
is_active: bool | None = None
@ -4492,6 +4504,7 @@ class DeleteJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
class JWTKeyMappingResponse(LiteLLMPydanticObjectBase):
id: str
jwt_issuer: str | None = None
jwt_claim_name: str
jwt_claim_value: str
description: str | None = None
@ -4714,6 +4727,7 @@ class JWTAuthBuilderResult(TypedDict):
org_id: str | None
team_membership: LiteLLM_TeamMembership | None
jwt_claims: dict # Decoded JWT token claims (avoids re-decoding)
agent_id: ReadOnly[str | None]
class ClientSideFallbackModel(TypedDict, total=False):
@ -4952,6 +4966,14 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
user_allowed_roles: list[str] | None = None
user_id_upsert: bool = Field(default=False, description="If user doesn't exist, upsert them into the db.")
end_user_id_jwt_field: str | None = None
agent_id_jwt_field: str | None = Field(
default=None,
description=(
"The field in the JWT token that identifies the calling agent (e.g. 'azp' for a Microsoft Entra ID "
"app token). Supports dot notation. The value is matched against a registered agent's agent_id, "
"then agent_name, and the request is rejected when it matches neither."
),
)
public_key_ttl: float = 600
public_key_stale_ttl: float = Field(
default=DEFAULT_JWKS_STALE_TTL,

45
litellm/proxy/auth/auth_checks.py
View file

@ -39,6 +39,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.models.project import LiteLLM_ProjectTable
from litellm.proxy._types import (
RBAC_ROLES,
CallInfo,
@ -109,7 +110,7 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import RowT_co
from litellm.repositories.prisma_protocols import DatabaseClient, RowT_co
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import (
AccessGroupRepository,
@ -147,6 +148,7 @@ class _PrismaDictableRow(Protocol):
class _PrismaJWTKeyMappingRow(Protocol):
token: str
jwt_issuer: str
jwt_claim_name: str
jwt_claim_value: str
@ -847,6 +849,7 @@ BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset(
"/health",
"/health/services",
"/health/test_connection",
"/auto_router/test_routing",
}
)
@ -3190,7 +3193,7 @@ async def _delete_cache_access_object(
@log_db_metrics
async def get_access_object(
access_group_id: str,
prisma_client: PrismaClient | None,
prisma_client: DatabaseClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_AccessGroupTable:
@ -3617,9 +3620,18 @@ async def _fetch_key_object_from_db_with_reconnect(
raise
def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str) -> str:
"""Cache key under which ``_resolve_jwt_to_virtual_key`` stores a JWT-claim-to-key mapping."""
return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str, jwt_issuer: str | None = None) -> str:
"""Cache key under which a JWT-claim-to-key mapping is stored, scoped to one
issuer (or the issuer-agnostic/global scope when ``jwt_issuer`` is falsy).
Scoped by issuer (when one is configured) so a cached hit or ``__NO_MAPPING__`` miss
for one issuer's claim value can never be served to a different issuer whose claim
value happens to collide. Unchanged for the global scope, keeping the single-issuer
(no ``litellm_jwtauth.issuers`` configured) cache key format stable across this fix.
"""
if not jwt_issuer:
return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
return f"jwt_key_mapping:{jwt_issuer}:{jwt_claim_name}:{jwt_claim_value}"
@log_db_metrics
@ -3631,7 +3643,7 @@ async def get_jwt_key_mapping_cache_keys_for_token(
mappings: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_many(
where={"token": hashed_token}
)
return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value) for m in mappings)
return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value, m.jwt_issuer) for m in mappings)
@log_db_metrics
@ -3639,9 +3651,14 @@ async def get_jwt_key_mapping_object(
jwt_claim_name: str,
jwt_claim_value: str,
prisma_client: PrismaClient,
jwt_issuer: str | None = None,
) -> str | None:
"""
Lookup a JWT-to-virtual-key mapping from the database.
Lookup a JWT-to-virtual-key mapping from the database for one exact scope:
``jwt_issuer`` (or the global/issuer-agnostic scope when falsy). Does not fall
back to the global scope itself -- a caller that wants "issuer-scoped mapping,
else the global one" queries both scopes itself, so each result can be cached
under its own scope's key (see ``_resolve_jwt_to_virtual_key``).
Returns the hashed token (str) if a matching active mapping is found, else None.
"""
@ -3649,6 +3666,7 @@ async def get_jwt_key_mapping_object(
where={
"jwt_claim_name": jwt_claim_name,
"jwt_claim_value": jwt_claim_value,
"jwt_issuer": jwt_issuer or "",
"is_active": True,
}
)
@ -3936,7 +3954,7 @@ async def get_org_object(
async def _get_resources_from_access_groups(
access_group_ids: Sequence[str],
resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
prisma_client: PrismaClient | None = None,
prisma_client: DatabaseClient | None = None,
user_api_key_cache: UserApiKeyCache | None = None,
proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
@ -3994,7 +4012,7 @@ async def _get_resources_from_access_groups(
async def _get_models_from_access_groups(
access_group_ids: Sequence[str],
prisma_client: PrismaClient | None = None,
prisma_client: DatabaseClient | None = None,
user_api_key_cache: UserApiKeyCache | None = None,
proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
@ -4493,6 +4511,7 @@ async def can_key_call_model(
llm_model_list: Sequence[object] | None,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
prisma_client: DatabaseClient | None = None,
) -> Literal[True]:
"""
Checks if token can call a given model
@ -4522,6 +4541,7 @@ async def can_key_call_model(
if key_access_group_ids:
models_from_groups: Final = await _get_models_from_access_groups(
access_group_ids=key_access_group_ids,
prisma_client=prisma_client,
)
if models_from_groups:
return _can_object_call_model(
@ -4650,6 +4670,7 @@ async def can_team_access_model(
team_object: LiteLLM_TeamTable | None,
llm_router: Router | None,
team_model_aliases: dict[str, str] | None = None,
prisma_client: DatabaseClient | None = None,
) -> Literal[True]:
"""
Returns True if the team can access a specific model.
@ -4672,6 +4693,7 @@ async def can_team_access_model(
if team_access_group_ids:
models_from_groups: Final = await _get_models_from_access_groups(
access_group_ids=team_access_group_ids,
prisma_client=prisma_client,
)
if models_from_groups:
return _can_object_call_model(
@ -4767,7 +4789,7 @@ async def _key_access_group_grants_model(
def can_project_access_model(
model: str | list[str],
project_object: LiteLLM_ProjectTableCachedObj,
project_object: LiteLLM_ProjectTable,
llm_router: Router | None,
) -> Literal[True]:
"""
@ -5785,8 +5807,7 @@ async def _organization_max_budget_check(
if org_table.litellm_budget_table is not None:
org_max_budget = org_table.litellm_budget_table.max_budget
# Only check if organization has a valid max_budget set
if org_max_budget is None or org_max_budget <= 0:
if org_max_budget is None:
return
# Read spend from cross-pod counter (Redis-first) or cached object (fallback)

29
litellm/proxy/auth/auth_exception_handler.py
View file

@ -23,6 +23,7 @@ from litellm.proxy.auth.auth_utils import (
_get_request_ip_address,
is_invalid_virtual_key_error,
mark_invalid_virtual_key_error,
normalize_request_route,
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes
@ -74,17 +75,28 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
)
def _with_requester_ip_address(request_data: dict[str, object], requester_ip: str | None) -> dict[str, object]:
def _get_user_agent(request: Request) -> str | None:
if "headers" not in request.scope:
return None
return request.headers.get("user-agent")
def _with_client_context(
request_data: dict[str, object], requester_ip: str | None, user_agent: str | None
) -> dict[str, object]:
"""Auth gate rejections are raised before `add_litellm_data_to_request` records the
caller IP, so their failure logs would otherwise carry no IP nor key/user identity."""
if not requester_ip:
return request_data
caller IP and User-Agent, so their failure logs would otherwise carry neither."""
key: Final = "litellm_metadata" if "litellm_metadata" in request_data else "metadata"
metadata: Final = request_data.get(key)
base: Final[Mapping[str, object]] = metadata if isinstance(metadata, Mapping) else EMPTY_MAPPING
if base.get("requester_ip_address"):
stamped: Final = {
name: value
for name, value in (("requester_ip_address", requester_ip), ("user_agent", user_agent))
if value and not base.get(name)
}
if not stamped:
return request_data
return {**request_data, key: {**base, "requester_ip_address": requester_ip}} # mutable-ok: logging needs dicts
return {**request_data, key: {**base, **stamped}} # mutable-ok: logging needs dicts
class UserAPIKeyAuthExceptionHandler:
@ -148,6 +160,7 @@ class UserAPIKeyAuthExceptionHandler:
request=request,
use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True,
)
user_agent: Final = _get_user_agent(request)
# Log authentication failures before identity seeding and callbacks, so the log
# survives a raising callback pipeline. Classify and route malformed virtual-key
@ -172,7 +185,7 @@ class UserAPIKeyAuthExceptionHandler:
# so the handler is side-effect-free for the caller's identity object.
user_api_key_dict = resolved_identity.model_copy() if resolved_identity is not None else UserAPIKeyAuth()
user_api_key_dict.parent_otel_span = parent_otel_span
user_api_key_dict.request_route = route
user_api_key_dict.request_route = normalize_request_route(route)
user_api_key_dict.api_key = user_api_key_dict.api_key or UserAPIKeyAuth(api_key=api_key).api_key
# Stamp identity onto the request's server span now, before the request
@ -200,7 +213,7 @@ class UserAPIKeyAuthExceptionHandler:
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
request_data=_with_requester_ip_address(request_data, requester_ip),
request_data=_with_client_context(request_data, requester_ip, user_agent),
original_exception=e,
user_api_key_dict=user_api_key_dict,
error_type=ProxyErrorTypes.auth_error,

5
litellm/proxy/auth/auth_utils.py
View file

@ -1967,6 +1967,11 @@ def request_dispatched_to_pass_through_endpoint(request: Request | None) -> bool
return getattr(endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, False) is True
def request_dispatched_to_provider_pass_through(request: Request) -> bool:
"""Built-in provider pass-through handlers (``/anthropic/{endpoint:path}``, ...) bind ``endpoint``."""
return "endpoint" in request.path_params
def get_model_from_request(
request_data: dict,
route: str,

136
litellm/proxy/auth/auto_router_checks.py Normal file
View file

@ -0,0 +1,136 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
if TYPE_CHECKING:
from litellm.router import Router
_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _mapping(value: object) -> Mapping[str, object] | None:
try:
return _MAPPING_ADAPTER.validate_python(value)
except ValidationError:
return None
async def authorize_member_auto_router_inference(
*,
deployment: Mapping[str, object] | None,
request_kwargs: Mapping[str, object],
llm_router: Router,
) -> None:
if deployment is None:
return
model_info: Final = _mapping(deployment.get("model_info"))
if model_info is None or model_info.get("member_auto_router") is not True:
return
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import (
OrganizationNotFoundError,
TeamNotFoundError,
get_org_object,
get_project_object,
get_team_membership,
get_team_object,
)
from litellm.proxy.management_helpers.auto_router_permissions import (
MemberAutoRouterDependencyObjects,
authorize_member_auto_router_dependencies,
validate_member_auto_router_config,
)
metadata: Final = _mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)))
actor: Final = metadata.get("user_api_key_auth") if metadata is not None else None
team_id: Final = model_info.get("team_id")
if not isinstance(actor, UserAPIKeyAuth) or not isinstance(team_id, str) or not team_id:
raise HTTPException(status_code=403, detail="Member auto-routers require authenticated team access")
if actor.team_id != team_id and actor.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="This auto-router belongs to a different team")
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
if prisma_client is None:
raise HTTPException(status_code=503, detail="Cannot verify auto-router model access without a database")
try:
team: Final = await get_team_object(
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=actor.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
except TeamNotFoundError as error:
raise HTTPException(status_code=403, detail="The auto-router team no longer exists") from error
if (
actor.user_role != LitellmUserRoles.PROXY_ADMIN
and actor.user_id is not None
and (not actor.user_id or not any(member.user_id == actor.user_id for member in team.members_with_roles))
):
raise HTTPException(status_code=403, detail="You are no longer a member of this auto-router's team")
if team.blocked:
raise HTTPException(status_code=403, detail="This auto router's team is blocked.")
params: Final = _mapping(deployment.get("litellm_params"))
if params is None:
raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
raw_config: Final = _mapping(params.get("complexity_router_config"))
if raw_config is None:
raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
default_model: Final = params.get("complexity_router_default_model")
config: Final = validate_member_auto_router_config(raw_config)
membership: Final = (
await get_team_membership(
user_id=actor.user_id,
team_id=team_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=actor.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if actor.user_id
else None
)
try:
organization: Final = (
await get_org_object(
org_id=team.organization_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=actor.parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if team.organization_id
else None
)
except OrganizationNotFoundError as error:
raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.") from error
project: Final = (
await get_project_object(
project_id=actor.project_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
if actor.project_id
else None
)
await authorize_member_auto_router_dependencies(
config=config,
default_model=default_model if isinstance(default_model, str) else None,
user_api_key_dict=actor,
team=team,
prisma_client=None,
llm_router=llm_router,
dependency_objects=MemberAutoRouterDependencyObjects(
membership=membership, organization=organization, project=project
),
)

71
litellm/proxy/auth/handle_jwt.py
View file

@ -14,7 +14,7 @@ import hashlib
import os
import re
import time
from collections.abc import Awaitable, Callable, Sequence
from collections.abc import Awaitable, Callable, Mapping, Sequence
from typing import Any, Final, Literal, NoReturn, Protocol, TypeVar, cast
import httpx
@ -61,6 +61,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
)
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.repositories.user_repository import UserRepository
from litellm.types.agents import AgentResponse
from .auth_checks import (
_allowed_routes_check,
@ -127,6 +128,26 @@ class _UserInfoResponse(Protocol):
def json(self) -> dict[str, object]: ...
class AgentLookup(Protocol):
"""The registered-agent lookups a JWT agent claim is matched against."""
def get_agent_by_id(self, agent_id: str) -> AgentResponse | None:
"""The agent registered under ``agent_id``, if any."""
def get_agent_by_name(self, agent_name: str) -> AgentResponse | None:
"""The agent registered under ``agent_name``, if any."""
class _NoRegisteredAgents:
"""The lookup in force until the proxy binds its agent registry: no agent is registered, so no claim matches."""
def get_agent_by_id(self, agent_id: str) -> None:
return None
def get_agent_by_name(self, agent_name: str) -> None:
return None
def _discovery_document(response: _OIDCDiscoveryResponse) -> _OIDCDiscoveryBody:
"""Decode an OIDC discovery response body."""
return response.json()
@ -198,6 +219,10 @@ class JWTHandler:
self.leeway = 0
# Per-cache-key locks so a TTL lapse triggers one refresh instead of one per in-flight request.
self._refresh_locks: dict[str, asyncio.Lock] = {} # mutable-ok: lock registry, keyed by JWKS url
self.agent_lookup: AgentLookup = _NoRegisteredAgents()
def bind_agent_lookup(self, agent_lookup: AgentLookup) -> None:
self.agent_lookup = agent_lookup
def update_environment(
self,
@ -623,6 +648,12 @@ class JWTHandler:
object_id = default_value
return object_id
def get_agent_claim(self, token: Mapping[str, object]) -> str | None:
if self.litellm_jwtauth.agent_id_jwt_field is None:
return None
claim: Final[object] = get_nested_value(data=token, key_path=self.litellm_jwtauth.agent_id_jwt_field)
return claim if isinstance(claim, str) and claim else None
def get_org_id(self, token: dict, default_value: str | None) -> str | None:
if self._has_trusted_issuer_normalized_claim(token=token, claim=self.LITELLM_ORG_ID_CLAIM):
return token.get(self.LITELLM_ORG_ID_CLAIM)
@ -1380,6 +1411,7 @@ class JWTAuthManager:
api_key: str,
jwt_valid_token: dict | None = None,
user_email: str | None = None,
agent_id: str | None = None,
) -> JWTAuthBuilderResult | None:
"""Check admin status and route access permissions"""
if not jwt_handler.is_admin(scopes=scopes):
@ -1409,8 +1441,28 @@ class JWTAuthManager:
org_id=org_id,
team_membership=None,
jwt_claims=jwt_valid_token or {},
agent_id=agent_id,
)
@staticmethod
def resolve_agent_id(
jwt_handler: JWTHandler,
jwt_valid_token: Mapping[str, object],
agent_registry: AgentLookup,
) -> str | None:
agent_claim: Final = jwt_handler.get_agent_claim(token=jwt_valid_token)
if agent_claim is None:
return None
agent: Final = agent_registry.get_agent_by_id(agent_id=agent_claim) or agent_registry.get_agent_by_name(
agent_name=agent_claim
)
if agent is None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"No registered agent matches JWT claim {jwt_handler.litellm_jwtauth.agent_id_jwt_field}={agent_claim}",
)
return agent.agent_id
@staticmethod
async def find_and_validate_specific_team_id(
jwt_handler: JWTHandler,
@ -2268,9 +2320,23 @@ class JWTAuthManager:
elif rbac_role == LitellmUserRoles.INTERNAL_USER:
user_id = object_id
agent_id: Final = JWTAuthManager.resolve_agent_id(
jwt_handler=jwt_handler,
jwt_valid_token=jwt_valid_token,
agent_registry=jwt_handler.agent_lookup,
)
# Check admin access
admin_result: Final = await JWTAuthManager.check_admin_access(
jwt_handler, scopes, route, user_id, org_id, api_key, jwt_valid_token, user_email=user_email
jwt_handler,
scopes,
route,
user_id,
org_id,
api_key,
jwt_valid_token,
user_email=user_email,
agent_id=agent_id,
)
if admin_result:
await JWTAuthManager._attach_team_from_header_for_admin(
@ -2514,4 +2580,5 @@ class JWTAuthManager:
token=api_key,
team_membership=team_membership_object,
jwt_claims=jwt_valid_token,
agent_id=agent_id,
)

2
litellm/proxy/auth/team_grants.py
View file

@ -56,6 +56,7 @@ class TeamGrants(TypedDict, total=False):
team_alias: ReadOnly[str | None]
team_tpm_limit: ReadOnly[int | None]
team_rpm_limit: ReadOnly[int | None]
team_tpd_limit: ReadOnly[int | None]
team_max_budget: ReadOnly[float | None]
team_soft_budget: ReadOnly[float | None]
team_spend: ReadOnly[float | None]
@ -97,6 +98,7 @@ def team_grants(
team_alias=team_object.team_alias,
team_tpm_limit=team_object.tpm_limit,
team_rpm_limit=team_object.rpm_limit,
team_tpd_limit=team_object.tpd_limit,
team_max_budget=team_object.max_budget,
team_soft_budget=team_object.soft_budget,
team_spend=team_object.spend,

158
litellm/proxy/auth/user_api_key_auth.py
View file

@ -26,11 +26,13 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm._service_logger import ServiceLogging
from litellm.caching.redis_cache import RedisCache
from litellm.constants import (
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
GLOBAL_PROXY_SPEND_CACHE_KEY,
INVALID_VIRTUAL_KEY_ERROR_MARKER,
INVALID_VIRTUAL_KEY_ERROR_MESSAGE,
LITELLM_PROXY_BUDGET_NAME,
LITELLM_PROXY_MASTER_KEY_ALIAS,
MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY,
)
from litellm.integrations.otel.model.config import is_otel_v2_enabled
from litellm.integrations.otel.runtime import phase_span, seed_request_identity
@ -77,6 +79,8 @@ from litellm.proxy.auth.auth_utils import (
iter_request_fallback_targets,
normalize_request_route,
pre_db_read_auth_checks,
request_dispatched_to_pass_through_endpoint,
request_dispatched_to_provider_pass_through,
route_in_additonal_public_routes,
)
from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
@ -104,6 +108,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
_safe_set_request_parsed_body,
populate_request_with_path_params,
read_raw_json_body,
rewrite_request_model,
)
from litellm.proxy.common_utils.model_listing_utils import claude_code_requested_group
from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
@ -125,6 +130,7 @@ from litellm.proxy.utils import (
normalize_route_for_root_path,
)
from litellm.repositories.table_repositories import TeamMembershipRepository
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.secret_managers.main import get_secret_bool
from litellm.types.services import ServiceTypes
@ -236,11 +242,45 @@ async def _normalize_claude_model(
request.scope[_CLAUDE_MODEL_NORMALIZED] = True
if source is None:
return
request_data["model"] = source
_safe_set_request_parsed_body(request=request, parsed_body=request_data)
if request is not None:
request._json = request_data
request._body = orjson.dumps(request_data)
rewrite_request_model(request_data, request, source)
async def _resolve_router_settings_model_group_alias(
request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader
valid_token: UserAPIKeyAuth,
request: Request | None,
route: str,
) -> None:
"""Rewrite the requested model through the key's or team's ``router_settings.model_group_alias``
before the allowlist checks, so they authorize the model group the request is routed to.
"""
from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj
if request is None or llm_router is None or not RouteChecks.is_llm_api_route(route=route):
return
if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is True:
return
request.scope[MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY] = True
if request_dispatched_to_pass_through_endpoint(request) or request_dispatched_to_provider_pass_through(request):
return
requested: Final = request_data.get("model")
if not isinstance(requested, str) or await read_raw_json_body(request=request) is None:
return
settings: Final = await proxy_config.get_hierarchical_router_settings(
user_api_key_dict=valid_token, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
)
if not isinstance(settings, Mapping):
return
target: Final = resolve_model_group_alias(settings.get("model_group_alias"), requested)
if target is None or target == requested:
return
verbose_proxy_logger.debug(
"router_settings.model_group_alias resolved %s -> %s before auth",
requested.replace("\r", "").replace("\n", ""),
target.replace("\r", "").replace("\n", ""),
)
request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested)
rewrite_request_model(request_data, request, target)
def _get_model_names_for_budget_checks(
@ -270,6 +310,17 @@ class _TokenTeamModels(Protocol):
def team_models(self) -> list[str]: ...
class _RawCacheRead(Protocol):
async def async_get_cache(self, *, key: str) -> object: ...
def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead:
"""View an untyped cache object's ``async_get_cache`` as returning ``object``
instead of ``Any``, so a caller can ``isinstance``-narrow it without paying
the ``reportAny`` cost of the underlying (unannotated) cache implementation."""
return cache
def _token_team_models(valid_token: _TokenTeamModels) -> list[str]:
return valid_token.team_models
@ -538,6 +589,9 @@ def _apply_budget_limits_to_end_user_params(
if budget_info.rpm_limit is not None:
end_user_params["end_user_rpm_limit"] = budget_info.rpm_limit
if budget_info.tpd_limit is not None:
end_user_params["end_user_tpd_limit"] = budget_info.tpd_limit
if budget_info.max_budget is not None:
end_user_params["end_user_max_budget"] = budget_info.max_budget
@ -622,6 +676,8 @@ def update_valid_token_with_end_user_params(valid_token: UserAPIKeyAuth, end_use
valid_token.end_user_tpm_limit = end_user_params["end_user_tpm_limit"]
if end_user_params.get("end_user_rpm_limit") is not None:
valid_token.end_user_rpm_limit = end_user_params["end_user_rpm_limit"]
if end_user_params.get("end_user_tpd_limit") is not None:
valid_token.end_user_tpd_limit = end_user_params["end_user_tpd_limit"]
if end_user_params.get("allowed_model_region") is not None:
valid_token.allowed_model_region = end_user_params["allowed_model_region"]
if end_user_params.get("end_user_model_max_budget") is not None:
@ -838,6 +894,7 @@ class _PendingAutoRegister(NamedTuple):
claim_field: str
claim_value: str
cache_key: str
jwt_issuer: str | None = None
async def _auto_register_jwt_mapping(
@ -849,10 +906,12 @@ async def _auto_register_jwt_mapping(
parent_otel_span: Span | None,
proxy_logging_obj: ProxyLogging,
cache_key: str,
jwt_issuer: str | None = None,
team_id: str | None = None,
user_id: str | None = None,
org_id: str | None = None,
end_user_id: str | None = None,
agent_id: str | None = None,
) -> UserAPIKeyAuth | None:
"""
Auto-register: create a new virtual key + mapping for an unrecognised JWT
@ -885,6 +944,7 @@ async def _auto_register_jwt_mapping(
team_id=team_id,
user_id=user_id,
organization_id=org_id,
agent_id=agent_id,
metadata={
"auto_registered": True,
"jwt_claim_field": virtual_key_claim_field,
@ -899,6 +959,7 @@ async def _auto_register_jwt_mapping(
try:
await prisma_client.db.litellm_jwtkeymapping.create(
data={
"jwt_issuer": jwt_issuer or "",
"jwt_claim_name": virtual_key_claim_field,
"jwt_claim_value": claim_value,
"token": token_hash,
@ -933,6 +994,7 @@ async def _auto_register_jwt_mapping(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=claim_value,
prisma_client=prisma_client,
jwt_issuer=jwt_issuer,
)
if token_hash is None:
# The winner's mapping vanished between the unique-constraint
@ -977,6 +1039,43 @@ async def _auto_register_jwt_mapping(
return auto_registered_key
async def _lookup_jwt_mapping_token_hash(
prisma_client: PrismaClient,
user_api_key_cache: UserApiKeyCache,
virtual_key_claim_field: str,
claim_value: str,
normalized_issuer: str | None,
cache_key: str,
ttl: float,
) -> str | None:
issuer_scoped: Final = await get_jwt_key_mapping_object(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=claim_value,
prisma_client=prisma_client,
jwt_issuer=normalized_issuer,
)
if issuer_scoped is not None:
await user_api_key_cache.async_set_cache(key=cache_key, value=issuer_scoped, ttl=ttl)
return issuer_scoped
if normalized_issuer is None:
return None
# Another issuer may have already resolved (and cached) this same
# global mapping -- check its cache entry before re-querying the DB.
global_cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, claim_value)
cached_global: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=global_cache_key)
if isinstance(cached_global, str) and cached_global != "__NO_MAPPING__":
return cached_global
global_row: Final = await get_jwt_key_mapping_object(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=claim_value,
prisma_client=prisma_client,
jwt_issuer=None,
)
if global_row is not None:
await user_api_key_cache.async_set_cache(key=global_cache_key, value=global_row, ttl=ttl)
return global_row
async def _resolve_jwt_to_virtual_key(
jwt_claims: dict,
jwt_handler: JWTHandler,
@ -1035,7 +1134,7 @@ async def _resolve_jwt_to_virtual_key(
)
return None
cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value), normalized_issuer)
raw_cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key)
sentinel_written_by_this_policy: Final = behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER
cached_mapping: Final = (
@ -1075,6 +1174,7 @@ async def _resolve_jwt_to_virtual_key(
claim_field=virtual_key_claim_field,
claim_value=str(claim_value),
cache_key=cache_key,
jwt_issuer=normalized_issuer,
)
return None
elif cached_mapping is not None:
@ -1088,21 +1188,30 @@ async def _resolve_jwt_to_virtual_key(
)
# Resolve the mapping from DB, or treat prisma_client=None as a definitive
# miss (no DB → no mapping can exist → apply no-match policy below).
token_hash: str | None = None
if prisma_client is not None:
token_hash = await get_jwt_key_mapping_object(
jwt_claim_name=virtual_key_claim_field,
jwt_claim_value=str(claim_value),
# miss (no DB → no mapping can exist → apply no-match policy below). An
# issuer-scoped row wins; falling back to the global (no-issuer) row keeps
# mappings created before issuer scoping existed working for every issuer.
# Each tier is cached under ITS OWN key (the global tier under the
# issuer-less cache key, not under `cache_key`/this issuer's key) so that
# updating or deleting either row invalidates exactly the cache entries it
# can affect. Caching a global-row hit under the requesting issuer's key
# would leave every OTHER issuer that had fallen back to that same global
# mapping serving its stale token until TTL after the row changes.
token_hash: Final = (
await _lookup_jwt_mapping_token_hash(
prisma_client=prisma_client,
)
if token_hash is not None:
await user_api_key_cache.async_set_cache(
key=cache_key,
value=token_hash,
user_api_key_cache=user_api_key_cache,
virtual_key_claim_field=virtual_key_claim_field,
claim_value=str(claim_value),
normalized_issuer=normalized_issuer,
cache_key=cache_key,
ttl=jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl,
)
if prisma_client is not None
else None
)
if token_hash is not None:
return IdentityStore.key_from_principal(
await IdentityStore(
prisma_client,
@ -1143,6 +1252,7 @@ async def _resolve_jwt_to_virtual_key(
claim_field=virtual_key_claim_field,
claim_value=str(claim_value),
cache_key=cache_key,
jwt_issuer=normalized_issuer,
)
# FALLBACK_TEAM_MAPPING (default): cache the miss and return None so the
@ -1568,6 +1678,7 @@ async def _user_api_key_auth_builder(
org_id: Final = result["org_id"]
team_membership: Final[LiteLLM_TeamMembership | None] = result.get("team_membership", None)
jwt_claims = result.get("jwt_claims", None)
agent_id: Final[str | None] = result.get("agent_id")
if is_proxy_admin:
# Proxy admins authenticate via auth_builder (full
@ -1593,6 +1704,7 @@ async def _user_api_key_auth_builder(
end_user_id=end_user_id,
parent_otel_span=parent_otel_span,
jwt_claims=jwt_claims,
agent_id=agent_id,
**team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id),
)
@ -1613,6 +1725,7 @@ async def _user_api_key_auth_builder(
user_rpm_limit=(user_object.rpm_limit if user_object is not None else None),
user_model_max_budget=(user_object.model_max_budget if user_object is not None else None),
jwt_claims=jwt_claims,
agent_id=agent_id,
**team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id),
)
@ -1632,10 +1745,12 @@ async def _user_api_key_auth_builder(
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
cache_key=pending_auto_register.cache_key,
jwt_issuer=pending_auto_register.jwt_issuer,
team_id=team_id,
user_id=user_id,
org_id=org_id,
end_user_id=end_user_id,
agent_id=agent_id,
)
if auto_registered is not None:
auto_registered.jwt_claims = jwt_claims
@ -2021,6 +2136,7 @@ async def _user_api_key_auth_builder(
valid_token.end_user_id = end_user_params.get("end_user_id")
valid_token.end_user_tpm_limit = end_user_params.get("end_user_tpm_limit")
valid_token.end_user_rpm_limit = end_user_params.get("end_user_rpm_limit")
valid_token.end_user_tpd_limit = end_user_params.get("end_user_tpd_limit")
valid_token.allowed_model_region = end_user_params.get("allowed_model_region")
if valid_token is not None:
@ -2297,6 +2413,7 @@ async def _user_api_key_auth_builder(
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
tpd_limit=valid_token.team_tpd_limit,
blocked=valid_token.team_blocked,
models=token_team_models,
metadata=valid_token.team_metadata,
@ -2450,6 +2567,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
spend=valid_token.team_spend,
tpm_limit=valid_token.team_tpm_limit,
rpm_limit=valid_token.team_rpm_limit,
tpd_limit=valid_token.team_tpd_limit,
blocked=valid_token.team_blocked,
models=token_team_models,
metadata=valid_token.team_metadata,
@ -2491,7 +2609,7 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc
async def _run_centralized_common_checks(
user_api_key_auth_obj: UserAPIKeyAuth,
request: Request,
request_data: dict,
request_data: dict[str, object],
route: str,
) -> None:
"""Run ``common_checks`` once at the ``user_api_key_auth`` wrapper
@ -2913,6 +3031,7 @@ async def _authorize_authenticated_request(
## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ##
RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request)
await _normalize_claude_model(request_data, user_api_key_auth_obj, request, route)
await _resolve_router_settings_model_group_alias(request_data, user_api_key_auth_obj, request, route)
# Single authorization point. Builder paths MUST NOT call common_checks.
# Route through the same exception handler the builder uses so
@ -3299,6 +3418,7 @@ async def _enforce_key_and_fallback_model_access(
Not included in common_checks — common_checks enforces team/user/project model access only.
"""
await _normalize_claude_model(request_data, valid_token, request, route)
await _resolve_router_settings_model_group_alias(request_data, valid_token, request, route)
config: Final = valid_token.config
if config != {}:

2
litellm/proxy/client/cli/README.md
View file

@ -490,7 +490,7 @@ lite codex exec "summarize the repo"
Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.

325
litellm/proxy/client/cli/commands/agents.py
View file

@ -4,15 +4,16 @@ import re
import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, TypeAlias
from typing import Final, Literal, TypeAlias
import click
import requests
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login
from .claude_settings import ClaudeSettingsError, install_statusline_script
@ -65,6 +66,10 @@ _INSTALL_DOCS: Final[dict[str, str]] = {
_HIDDEN_AGENTS: Final = frozenset({"pi"})
CODEX_PROXY_PROVIDER: Final = "litellm"
CODEX_HOME_ENV: Final = "CODEX_HOME"
CODEX_MODEL_CATALOG_FILENAME: Final = "litellm-models.json"
_CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
_CODEX_PREFLIGHT_TIMEOUT_SECONDS: Final = 10.0
class AgentRunError(Exception):
@ -252,7 +257,7 @@ def agent_launch_args(command: str, base_url: str) -> list[str]:
class ListedModel(BaseModel):
"""The fields of a /v1/models entry that an OpenCode model entry is built from."""
"""The fields of a /v1/models entry that an OpenCode or Codex model entry is built from."""
id: str
mode: str | None = None
@ -265,7 +270,7 @@ class _ModelListing(BaseModel):
_MODEL_LISTING: Final = TypeAdapter(_ModelListing)
_OPENCODE_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
_NO_EXTRA_ENV: Final[Mapping[str, str]] = MappingProxyType({})
@ -274,6 +279,40 @@ class ModelSyncSkipped:
reason: str
@dataclass(frozen=True, slots=True)
class ModelSyncArgs:
"""CLI args, placed before the user's own, that hand an agent the synced model list."""
args: tuple[str, ...]
ModelSyncResult: TypeAlias = Mapping[str, str] | ModelSyncArgs | ModelSyncSkipped
def _chat_models(models: Sequence[ListedModel]) -> tuple[ListedModel, ...]:
return tuple(m for m in models if m.mode is None or m.mode in _CHAT_MODES)
def _fetch_model_listing(
base_url: str,
api_key: str,
*,
get: Callable[..., requests.Response],
) -> tuple[ListedModel, ...] | ModelSyncSkipped:
url: Final = base_url.rstrip("/") + "/v1/models"
try:
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
except requests.RequestException as e:
return ModelSyncSkipped(f"could not reach {url}: {e}")
if resp.status_code != 200:
return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
try:
listing: Final = _MODEL_LISTING.validate_json(resp.content)
except ValidationError:
return ModelSyncSkipped(f"{url} returned an unexpected body")
return listing.data
class _OpenCodeLimit(BaseModel):
context: int
output: int
@ -317,7 +356,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
it never lands in the config text. OpenCode merges this inline config over
the user's own files, leaving unrelated keys and providers untouched.
"""
chat_models: Final = tuple(m for m in models if m.mode is None or m.mode in _OPENCODE_CHAT_MODES)
chat_models: Final = _chat_models(models)
provider: Final = _OpenCodeProvider(
npm=OPENCODE_PROVIDER_NPM,
name=OPENCODE_PROVIDER_NAME,
@ -347,40 +386,269 @@ def opencode_model_sync_env(
"""
if OPENCODE_CONFIG_CONTENT_ENV in base_env:
return ModelSyncSkipped(f"{OPENCODE_CONFIG_CONTENT_ENV} is already set")
url: Final = base_url.rstrip("/") + "/v1/models"
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
if isinstance(listing, ModelSyncSkipped):
return listing
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
class _CodexTruncationPolicy(BaseModel):
mode: Literal["bytes"] = "bytes"
limit: int = 10_000
class _CodexModel(BaseModel):
"""One `ModelInfo` entry of a Codex model catalog for a model the installed Codex does not know.
Every field that some Codex release since `model_catalog_json` appeared
(0.105.0) deserializes without a default is spelled out here, so one catalog
parses on all of them; the values match the fallback metadata Codex uses for
a model slug it does not know, so picking such a proxy model behaves the
same as `codex -m` did.
"""
slug: str
display_name: str
description: None = None
supported_reasoning_levels: tuple[()] = ()
shell_type: Literal["unified_exec"] = "unified_exec"
visibility: Literal["list"] = "list"
supported_in_api: Literal[True] = True
priority: int
availability_nux: None = None
upgrade: None = None
support_verbosity: Literal[False] = False
supports_reasoning_summaries: Literal[False] = False
supports_parallel_tool_calls: Literal[False] = False
default_verbosity: None = None
apply_patch_tool_type: None = None
truncation_policy: _CodexTruncationPolicy = _CodexTruncationPolicy()
experimental_supported_tools: tuple[()] = ()
context_window: int | None
base_instructions: str
class _StockCodexUpgrade(BaseModel):
model_config = ConfigDict(extra="allow")
model: str
class _StockCodexModel(BaseModel):
"""One `ModelInfo` entry as the installed Codex prints it from `codex debug models`.
Only the fields the sync rewrites are named; everything else that release
knows about the model (its reasoning levels, prompt, tool support) rides
along untouched, whatever the release's schema.
"""
model_config = ConfigDict(extra="allow")
slug: str
priority: int
visibility: str
supported_in_api: bool = True
upgrade: _StockCodexUpgrade | None = None
class _StockCodexCatalog(BaseModel):
models: tuple[_StockCodexModel, ...]
class _CodexCatalog(BaseModel):
models: tuple[_CodexModel | _StockCodexModel, ...]
def _codex_catalog_entry(
priority: int,
listed: ListedModel,
stock: _StockCodexModel | None,
served: frozenset[str],
instructions: str,
) -> _CodexModel | _StockCodexModel:
if stock is None:
return _CodexModel(
slug=listed.id,
display_name=listed.id,
priority=priority,
context_window=listed.max_input_tokens,
base_instructions=instructions,
)
upgrade: Final = stock.upgrade if stock.upgrade is not None and stock.upgrade.model in served else None
return stock.model_copy(
update={"priority": priority, "visibility": "list", "supported_in_api": True, "upgrade": upgrade}
)
def codex_model_catalog(
models: Sequence[ListedModel], stock: Sequence[_StockCodexModel], instructions: str
) -> str | None:
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
Codex refuses an empty catalog, hence None instead of `{"models": []}`.
Passing a catalog replaces Codex's built-in one, so a proxy model the
installed Codex knows keeps that Codex's own entry and the proxy only
decides its place in the picker: the listing orders it, lists it even when
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
when the model it points at is served too. A model Codex does not know gets the fallback
entry, with the same base instructions Codex itself uses so the agent never
runs without a system prompt.
"""
chat_models: Final = _chat_models(models)
if not chat_models:
return None
served: Final = frozenset(m.id for m in chat_models)
known: Final = MappingProxyType({m.slug: m for m in stock})
catalog: Final = _CodexCatalog(
models=tuple(
_codex_catalog_entry(index, m, known.get(m.id), served, instructions) for index, m in enumerate(chat_models)
)
)
return catalog.model_dump_json()
def codex_model_catalog_path(env: Mapping[str, str], *, home: Callable[[], Path] = Path.home) -> Path:
override: Final = env.get(CODEX_HOME_ENV)
root: Final = Path(override) if override else home() / ".codex"
return root / CODEX_MODEL_CATALOG_FILENAME
def _replace_file(path: Path, text: str) -> None:
with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
_ = tmp.write(text)
try:
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
except requests.RequestException as e:
return ModelSyncSkipped(f"could not reach {url}: {e}")
if resp.status_code != 200:
return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
os.replace(tmp.name, path)
except OSError:
Path(tmp.name).unlink(missing_ok=True)
raise
def _codex_debug_models(
binary: str,
args: Sequence[str],
env: Mapping[str, str],
*,
run: Callable[..., subprocess.CompletedProcess[str]],
) -> str | ModelSyncSkipped:
"""What `codex debug models` prints with `args` in front, or why the installed Codex could not run it.
The command prints the catalog Codex would launch with, without touching
the network, so it lists the installed Codex's own models and parses a
catalog override the way a launch does. Releases before 0.130.0 have no
such command and are reported the same way. A batch shim goes through
cmd.exe exactly as the launch will.
"""
name: Final = os.path.basename(binary)
command: Final = _windows_command(binary, (binary, *args, "debug", "models"))
try:
listing: Final = _MODEL_LISTING.validate_json(resp.content)
except ValidationError:
return ModelSyncSkipped(f"{url} returned an unexpected body")
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing.data)})
completed: Final = run(
command,
env=dict(env),
stdin=subprocess.DEVNULL,
capture_output=True,
encoding="utf-8",
timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
)
except (OSError, subprocess.TimeoutExpired) as e:
return ModelSyncSkipped(f"`{name} debug models` failed: {e}")
if completed.returncode == 0:
return completed.stdout
lines: Final = completed.stderr.strip().splitlines()
detail: Final = lines[0] if lines else "no output"
return ModelSyncSkipped(f"`{name} debug models` exited {completed.returncode}: {detail}")
def _stock_codex_models(
binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
) -> tuple[_StockCodexModel, ...] | ModelSyncSkipped:
printed: Final = _codex_debug_models(binary, (), env, run=run)
if isinstance(printed, ModelSyncSkipped):
return printed
try:
return _StockCodexCatalog.model_validate_json(printed).models
except ValidationError as e:
name: Final = os.path.basename(binary)
return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
def codex_model_sync_args(
base_env: Mapping[str, str],
base_url: str,
api_key: str,
*,
binary: str = "codex",
get: Callable[..., requests.Response] = requests.get,
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
home: Callable[[], Path] = Path.home,
instructions_path: Path = _CODEX_BASE_INSTRUCTIONS_PATH,
) -> ModelSyncArgs | ModelSyncSkipped:
"""`-c model_catalog_json=...` pointing Codex at the proxy's model list, or why it was skipped.
Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
atomically replaced on every launch. The Codex at `binary` first lists its
own models, so the ones the proxy serves keep that Codex's entries, and then
reads the file back once before it is handed over. The key never lands in
the file. A failed fetch, read, listing, write or read-back is reported
rather than raised: Codex still launches with its built-in catalog and takes
a proxy model by name via -m, and a rejected file stays on disk to be looked
at.
"""
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
if isinstance(listing, ModelSyncSkipped):
return listing
try:
instructions: Final = instructions_path.read_text(encoding="utf-8")
except OSError as e:
return ModelSyncSkipped(f"could not read {instructions_path}: {e}")
path: Final = codex_model_catalog_path(base_env, home=home)
try:
path.parent.mkdir(parents=True, exist_ok=True)
except OSError as e:
return ModelSyncSkipped(f"could not write {path}: {e}")
stock: Final = _stock_codex_models(binary, base_env, run=run)
if isinstance(stock, ModelSyncSkipped):
return stock
catalog: Final = codex_model_catalog(listing, stock, instructions)
if catalog is None:
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
try:
_replace_file(path, catalog)
except OSError as e:
return ModelSyncSkipped(f"could not write {path}: {e}")
override: Final = f"model_catalog_json={json.dumps(str(path))}"
read_back: Final = _codex_debug_models(binary, ("-c", override), base_env, run=run)
if isinstance(read_back, ModelSyncSkipped):
return read_back
return ModelSyncArgs(("-c", override))
def agent_model_sync_env(
command: str,
binary: str,
base_env: Mapping[str, str],
base_url: str,
api_key: str,
skip_verify: bool,
*,
get: Callable[..., requests.Response] = requests.get,
) -> Mapping[str, str] | ModelSyncSkipped:
"""Extra env an agent needs to see the proxy's model list.
run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
) -> ModelSyncResult:
"""Extra env or args an agent needs to see the proxy's model list.
Only OpenCode needs one: Claude Code discovers models through
CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY and Codex takes the model by name.
skip_verify means the caller wants no pre-launch proxy call at all, so the
listing is skipped too rather than hanging on an offline proxy.
binary is the resolved path the launch will run (`codex.cmd` on a Windows
npm install). OpenCode takes the list as env, Codex as a `-c` override that
binary has read back first; Claude Code discovers models itself through
CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY. skip_verify means the caller
wants no pre-launch proxy call at all, so the listing is skipped too rather
than hanging on an offline proxy.
"""
if os.path.basename(command) != "opencode":
agent: Final = os.path.splitext(os.path.basename(binary))[0]
if agent not in ("opencode", "codex"):
return _NO_EXTRA_ENV
if skip_verify:
return ModelSyncSkipped(f"{_SKIP_VERIFY_FLAG} was passed")
if agent == "codex":
return codex_model_sync_args(base_env, base_url, api_key, binary=binary, get=get, run=run)
return opencode_model_sync_env(base_env, base_url, api_key, get=get)
@ -508,9 +776,7 @@ def run_agent(
base_env: Mapping[str, str] | None = None,
which: Callable[[str], str | None] = shutil.which,
verify: Callable[[str, str], None] = verify_proxy_key,
sync_models: Callable[[str, Mapping[str, str], str, str, bool], Mapping[str, str] | ModelSyncSkipped] = (
agent_model_sync_env
),
sync_models: Callable[[str, Mapping[str, str], str, str, bool], ModelSyncResult] = agent_model_sync_env,
warn: Callable[[str], None] = _warn,
launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off,
reattach_terminal: Callable[[], None] | None = None,
@ -537,7 +803,7 @@ def run_agent(
verify(base_url, api_key)
env_before_sync: Final = base_env if base_env is not None else os.environ
synced: Final = sync_models(command[0], env_before_sync, base_url, api_key, skip_verify)
synced: Final = sync_models(binary, env_before_sync, base_url, api_key, skip_verify)
if isinstance(synced, ModelSyncSkipped):
warn(f"litellm: not syncing {display_name} models from the proxy: {synced.reason}")
@ -547,10 +813,11 @@ def run_agent(
env: Final = MappingProxyType(
{
**build_agent_env(env_before_sync, base_url, api_key, profiles),
**(_NO_EXTRA_ENV if isinstance(synced, ModelSyncSkipped) else synced),
**(synced if isinstance(synced, Mapping) else _NO_EXTRA_ENV),
}
)
extra_args: Final = (*agent_launch_args(command[0], base_url), *prepared_args)
synced_args: Final = synced.args if isinstance(synced, ModelSyncArgs) else ()
extra_args: Final = (*agent_launch_args(command[0], base_url), *synced_args, *prepared_args)
if reattach_terminal is not None:
reattach_terminal()
launcher(binary, [command[0], *extra_args, *command[1:]], env)

275
litellm/proxy/client/cli/commands/codex_base_instructions.md Normal file
View file

@ -0,0 +1,275 @@
You are a coding agent running in the Codex CLI, a terminal-based coding assistant. Codex CLI is an open source project led by OpenAI. You are expected to be precise, safe, and helpful.
Your capabilities:
- Receive user prompts and other context provided by the harness, such as files in the workspace.
- Communicate with the user by streaming thinking & responses, and by making & updating plans.
- Emit function calls to run terminal commands and apply patches. Depending on how this specific run is configured, you can request that these function calls be escalated to the user for approval before running. More on this in the "Sandbox and approvals" section.
Within this context, Codex refers to the open-source agentic coding interface (not the old Codex language model built by OpenAI).
# How you work
## Personality
Your default personality and tone is concise, direct, and friendly. You communicate efficiently, always keeping the user clearly informed about ongoing actions without unnecessary detail. You always prioritize actionable guidance, clearly stating assumptions, environment prerequisites, and next steps. Unless explicitly asked, you avoid excessively verbose explanations about your work.
# AGENTS.md spec
- Repos often contain AGENTS.md files. These files can appear anywhere within the repository.
- These files are a way for humans to give you (the agent) instructions or tips for working within the container.
- Some examples might be: coding conventions, info about how code is organized, or instructions for how to run or test code.
- Instructions in AGENTS.md files:
- The scope of an AGENTS.md file is the entire directory tree rooted at the folder that contains it.
- For every file you touch in the final patch, you must obey instructions in any AGENTS.md file whose scope includes that file.
- Instructions about code style, structure, naming, etc. apply only to code within the AGENTS.md file's scope, unless the file states otherwise.
- More-deeply-nested AGENTS.md files take precedence in the case of conflicting instructions.
- Direct system/developer/user instructions (as part of a prompt) take precedence over AGENTS.md instructions.
- The contents of the AGENTS.md file at the root of the repo and any directories from the CWD up to the root are included with the developer message and don't need to be re-read. When working in a subdirectory of CWD, or a directory outside the CWD, check for any AGENTS.md files that may be applicable.
## Responsiveness
### Preamble messages
Before making tool calls, send a brief preamble to the user explaining what you’re about to do. When sending preamble messages, follow these principles and examples:
- **Logically group related actions**: if you’re about to run several related commands, describe them together in one preamble rather than sending a separate note for each.
- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words for quick updates).
- **Build on prior context**: if this is not your first tool call, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.
- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.
- **Exception**: Avoid adding a preamble for every trivial read (e.g., `cat` a single file) unless it’s part of a larger grouped action.
**Examples:**
- “I’ve explored the repo; now checking the API route definitions.”
- “Next, I’ll patch the config and update the related tests.”
- “I’m about to scaffold the CLI commands and helper functions.”
- “Ok cool, so I’ve wrapped my head around the repo. Now digging into the API routes.”
- “Config’s looking tidy. Next up is patching helpers to keep things in sync.”
- “Finished poking at the DB gateway. I will now chase down error handling.”
- “Alright, build pipeline order is interesting. Checking how it reports failures.”
- “Spotted a clever caching util; now hunting where it gets used.”
## Planning
You have access to an `update_plan` tool which tracks steps and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go.
Note that plans are not for padding out simple work with filler steps or stating the obvious. The content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately.
Do not repeat the full contents of the plan after an `update_plan` call — the harness already displays it. Instead, summarize the change made and highlight any important context or next step.
Before running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. It may be the case that you complete all steps in your plan after a single pass of implementation. If this is the case, you can simply mark all the planned steps as completed. Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so.
Use a plan when:
- The task is non-trivial and will require multiple actions over a long time horizon.
- There are logical phases or dependencies where sequencing matters.
- The work has ambiguity that benefits from outlining high-level goals.
- You want intermediate checkpoints for feedback and validation.
- When the user asked you to do more than one thing in a single prompt
- The user has asked you to use the plan tool (aka "TODOs")
- You generate additional steps while working, and plan to do them before yielding to the user
### Examples
**High-quality plans**
Example 1:
1. Add CLI entry with file args
2. Parse Markdown via CommonMark library
3. Apply semantic HTML template
4. Handle code blocks, images, links
5. Add error handling for invalid files
Example 2:
1. Define CSS variables for colors
2. Add toggle with localStorage state
3. Refactor components to use variables
4. Verify all views for readability
5. Add smooth theme-change transition
Example 3:
1. Set up Node.js + WebSocket server
2. Add join/leave broadcast events
3. Implement messaging with timestamps
4. Add usernames + mention highlighting
5. Persist messages in lightweight DB
6. Add typing indicators + unread count
**Low-quality plans**
Example 1:
1. Create CLI tool
2. Add Markdown parser
3. Convert to HTML
Example 2:
1. Add dark mode toggle
2. Save preference
3. Make styles look good
Example 3:
1. Create single-file HTML game
2. Run quick sanity check
3. Summarize usage instructions
If you need to write a plan, only write high quality plans, not low quality ones.
## Task execution
You are a coding agent. Please keep going until the query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Autonomously resolve the query to the best of your ability, using the tools available to you, before coming back to the user. Do NOT guess or make up an answer.
You MUST adhere to the following criteria when solving queries:
- Working on the repo(s) in the current environment is allowed, even if they are proprietary.
- Analyzing code for vulnerabilities is allowed.
- Showing user code and tool call details is allowed.
- Use the `apply_patch` tool to edit files (NEVER try `applypatch` or `apply-patch`, only `apply_patch`): {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]}
If completing the user's task requires writing or modifying files, your code and final answer should follow these coding guidelines, though user instructions (i.e. AGENTS.md) may override these guidelines:
- Fix the problem at the root cause rather than applying surface-level patches, when possible.
- Avoid unneeded complexity in your solution.
- Do not attempt to fix unrelated bugs or broken tests. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)
- Update documentation as necessary.
- Keep changes consistent with the style of the existing codebase. Changes should be minimal and focused on the task.
- Use `git log` and `git blame` to search the history of the codebase if additional context is required.
- NEVER add copyright or license headers unless specifically requested.
- Do not waste tokens by re-reading files after calling `apply_patch` on them. The tool call will fail if it didn't work. The same goes for making folders, deleting folders, etc.
- Do not `git commit` your changes or create new git branches unless explicitly requested.
- Do not add inline comments within code unless explicitly requested.
- Do not use one-letter variable names unless explicitly requested.
- NEVER output inline citations like "【F:README.md†L5-L14】" in your outputs. The CLI is not able to render these so they will just be broken in the UI. Instead, if you output valid filepaths, users will be able to click on them to open the files in their editor.
## Validating your work
If the codebase has tests or the ability to build or run, consider using them to verify that your work is complete.
When testing, your philosophy should be to start as specific as possible to the code you changed so that you can catch issues efficiently, then make your way to broader tests as you build confidence. If there's no test for the code you changed, and if the adjacent patterns in the codebases show that there's a logical place for you to add a test, you may do so. However, do not add tests to codebases with no tests.
Similarly, once you're confident in correctness, you can suggest or use formatting commands to ensure that your code is well formatted. If there are issues you can iterate up to 3 times to get formatting right, but if you still can't manage it's better to save the user time and present them a correct solution where you call out the formatting in your final message. If the codebase does not have a formatter configured, do not add one.
For all of testing, running, building, and formatting, do not attempt to fix unrelated bugs. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)
Be mindful of whether to run validation commands proactively. In the absence of behavioral guidance:
- When running in the non-interactive approval mode **never**, proactively run tests, lint and do whatever you need to ensure you've completed the task.
- When working in interactive approval modes like **untrusted**, or **on-request**, hold off on running tests or lint commands until the user is ready for you to finalize your output, because these commands take time to run and slow down iteration. Instead suggest what you want to do next, and let the user confirm first.
- When working on test-related tasks, such as adding tests, fixing tests, or reproducing a bug to verify behavior, you may proactively run tests regardless of approval mode. Use your judgement to decide whether this is a test-related task.
## Ambition vs. precision
For tasks that have no prior context (i.e. the user is starting something brand new), you should feel free to be ambitious and demonstrate creativity with your implementation.
If you're operating in an existing codebase, you should make sure you do exactly what the user asks with surgical precision. Treat the surrounding codebase with respect, and don't overstep (i.e. changing filenames or variables unnecessarily). You should balance being sufficiently ambitious and proactive when completing tasks of this nature.
You should use judicious initiative to decide on the right level of detail and complexity to deliver based on the user's needs. This means showing good judgment that you're capable of doing the right extras without gold-plating. This might be demonstrated by high-value, creative touches when scope of the task is vague; while being surgical and targeted when scope is tightly specified.
## Sharing progress updates
For especially longer tasks that you work on (i.e. requiring many tool calls, or a plan with multiple steps), you should provide progress updates back to the user at reasonable intervals. These updates should be structured as a concise sentence or two (no more than 8-10 words long) recapping progress so far in plain language: this update demonstrates your understanding of what needs to be done, progress so far (i.e. files explores, subtasks complete), and where you're going next.
Before doing large chunks of work that may incur latency as experienced by the user (i.e. writing a new file), you should send a concise message to the user with an update indicating what you're about to do to ensure they know what you're spending time on. Don't start editing or writing large files before informing the user what you are doing and why.
The messages you send before tool calls should describe what is immediately about to be done next in very concise language. If there was previous work done, this preamble message should also include a note about the work done so far to bring the user along.
## Presenting your work and final message
Your final message should read naturally, like an update from a concise teammate. For casual conversation, brainstorming tasks, or quick questions from the user, respond in a friendly, conversational tone. You should ask questions, suggest ideas, and adapt to the user’s style. If you've finished a large amount of work, when describing what you've done to the user, you should follow the final answer formatting guidelines to communicate substantive changes. You don't need to add structured formatting for one-word answers, greetings, or purely conversational exchanges.
You can skip heavy formatting for single, simple actions or confirmations. In these cases, respond in plain sentences with any relevant next step or quick option. Reserve multi-section structured responses for results that need grouping or explanation.
The user is working on the same computer as you, and has access to your work. As such there's no need to show the full contents of large files you have already written unless the user explicitly asks for them. Similarly, if you've created or modified files using `apply_patch`, there's no need to tell users to "save the file" or "copy the code into a file"—just reference the file path.
If there's something that you think you could help with as a logical next step, concisely ask the user if they want you to do so. Good examples of this are running tests, committing changes, or building out the next logical component. If there’s something that you couldn't do (even with approval) but that the user might want to do (such as verifying changes by running the app), include those instructions succinctly.
Brevity is very important as a default. You should be very concise (i.e. no more than 10 lines), but can relax this requirement for tasks where additional detail and comprehensiveness is important for the user's understanding.
### Final answer structure and style guidelines
You are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value.
**Section Headers**
- Use only when they improve clarity — they are not mandatory for every answer.
- Choose descriptive names that fit the content
- Keep headers short (1–3 words) and in `**Title Case**`. Always start headers with `**` and end with `**`
- Leave no blank line before the first bullet under a header.
- Section headers should only be used where they genuinely improve scanability; avoid fragmenting the answer.
**Bullets**
- Use `-` followed by a space for every bullet.
- Merge related points when possible; avoid a bullet for every trivial detail.
- Keep bullets to one line unless breaking for clarity is unavoidable.
- Group into short lists (4–6 bullets) ordered by importance.
- Use consistent keyword phrasing and formatting across sections.
**Monospace**
- Wrap all commands, file paths, env vars, and code identifiers in backticks (`` `...` ``).
- Apply to inline examples and to bullet keywords if the keyword itself is a literal file/command.
- Never mix monospace and bold markers; choose one based on whether it’s a keyword (`**`) or inline code/path (`` ` ``).
**File References**
When referencing files in your response, make sure to include the relevant start line and always follow the below rules:
* Use inline code to make file paths clickable.
* Each reference should have a stand alone path. Even if it's the same file.
* Accepted: absolute, workspace‑relative, a/ or b/ diff prefixes, or bare filename/suffix.
* Line/column (1‑based, optional): :line[:column] or #Lline[Ccolumn] (column defaults to 1).
* Do not use URIs like file://, vscode://, or https://.
* Do not provide range of lines
* Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\repo\project\main.rs:12:5
**Structure**
- Place related bullets together; don’t mix unrelated concepts in the same section.
- Order sections from general → specific → supporting info.
- For subsections (e.g., “Binaries” under “Rust Workspace”), introduce with a bolded keyword bullet, then list items under it.
- Match structure to complexity:
- Multi-part or detailed results → use clear headers and grouped bullets.
- Simple results → minimal headers, possibly just a short list or paragraph.
**Tone**
- Keep the voice collaborative and natural, like a coding partner handing off work.
- Be concise and factual — no filler or conversational commentary and avoid unnecessary repetition
- Use present tense and active voice (e.g., “Runs tests” not “This will run tests”).
- Keep descriptions self-contained; don’t refer to “above” or “below”.
- Use parallel structure in lists for consistency.
**Don’t**
- Don’t use literal words “bold” or “monospace” in the content.
- Don’t nest bullets or create deep hierarchies.
- Don’t output ANSI escape codes directly — the CLI renderer applies them.
- Don’t cram unrelated keywords into a single bullet; split for clarity.
- Don’t let keyword lists run long — wrap or reformat for scanability.
Generally, ensure your final answers adapt their shape and depth to the request. For example, answers to code explanations should have a precise, structured explanation with code references that answer the question directly. For tasks with a simple implementation, lead with the outcome and supplement only with what’s needed for clarity. Larger changes can be presented as a logical walkthrough of your approach, grouping related steps, explaining rationale where it adds value, and highlighting next actions to accelerate the user. Your answers should provide the right level of detail while being easily scannable.
For casual greetings, acknowledgements, or other one-off conversational messages that are not delivering substantive information or structured results, respond naturally without section headers or bullet formatting.
# Tool Guidelines
## Shell commands
When using the shell, you must adhere to the following guidelines:
- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)
- Do not use python scripts to attempt to output larger chunks of a file.
## `update_plan`
A tool named `update_plan` is available to you. You can use it to keep an up‑to‑date, step‑by‑step plan for the task.
To create a new plan, call `update_plan` with a short list of 1‑sentence steps (no more than 5-7 words each) with a `status` for each step (`pending`, `in_progress`, or `completed`).
When steps have been completed, use `update_plan` to mark each finished step as `completed` and the next step you are working on as `in_progress`. There should always be exactly one `in_progress` step until everything is done. You can mark multiple items as complete in a single `update_plan` call.
If all steps are complete, ensure you call `update_plan` to mark all steps as `completed`.

32
litellm/proxy/common_request_processing.py
View file

@ -56,6 +56,7 @@ from litellm.proxy.common_utils.callback_utils import (
get_logging_caching_headers,
get_remaining_tokens_and_requests_from_request_data,
)
from litellm.proxy.common_utils.http_parsing_utils import get_client_requested_model
from litellm.proxy.common_utils.openai_error_payload import (
attribute_of,
error_status_code,
@ -622,9 +623,9 @@ async def _resolve_per_request_model_group_alias(
holds the global config map and is shared across requests, so a per-request
map has to be applied here instead of being forwarded to the Router.
Model access was authorized against the requested group, so the target is
authorized in its own right before the rewrite; a key that may not call the
target gets the usual 403 rather than being quietly served it.
Auth already rewrote the body through this map for LLM API routes, so this is
a fallback for callers that skipped it; the target is authorized in its own
right before the rewrite, so a key that may not call it gets the usual 403.
Returns the target model group, or None when no alias applies.
"""
@ -1451,10 +1452,13 @@ def _has_attribute_error_in_chain(exc: Exception) -> bool:
_CLIENT_DISCONNECT_DETAIL: Final = "Client disconnected the request"
def _log_llm_api_exception(e: Exception) -> None:
def _log_llm_api_exception(e: Exception, litellm_call_id: str | None) -> None:
if getattr(e, "status_code", None) == 499 and getattr(e, "detail", None) == _CLIENT_DISCONNECT_DETAIL:
verbose_proxy_logger.info(
"litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled"
"litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, "
"upstream LLM request cancelled - litellm_call_id=%s",
litellm_call_id,
extra=MappingProxyType({"litellm_call_id": litellm_call_id}),
)
return
log_fn: Final = (
@ -1462,7 +1466,12 @@ def _log_llm_api_exception(e: Exception) -> None:
if is_expected_client_error(e) and not litellm.log_client_error_tracebacks
else verbose_proxy_logger.exception
)
log_fn("litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - %s", e)
log_fn(
"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm_call_id=%s - %s",
litellm_call_id,
e,
extra=MappingProxyType({"litellm_call_id": litellm_call_id}),
)
async def _cancel_llm_call_on_client_disconnect(
@ -2338,9 +2347,8 @@ class ProxyBaseLLMRequestProcessing:
"""
Common request processing logic for both chat completions and responses API endpoints
"""
requested_model_from_client: Final[str | None] = (
self.data.get("model") if isinstance(self.data.get("model"), str) else None
)
client_model: Final = get_client_requested_model(request) or self.data.get("model")
requested_model_from_client: Final[str | None] = client_model if isinstance(client_model, str) else None
self._debug_log_request_payload()
if skip_pre_call_logic:
@ -3421,7 +3429,11 @@ class ProxyBaseLLMRequestProcessing:
version: str | None = None,
):
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
_log_llm_api_exception(e)
logging_obj: Final[LiteLLMLoggingObj | None] = self.data.get("litellm_logging_obj", None)
_log_llm_api_exception(
e,
(logging_obj.litellm_call_id if logging_obj is not None else None) or self.data.get("litellm_call_id"),
)
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,

2
litellm/proxy/common_utils/encrypt_decrypt_utils.py
View file

@ -124,7 +124,7 @@ def decrypt_value_helper(
key: str, # this is just for debug purposes, showing the k,v pair that's invalid. not a signing key.
exception_type: Literal["debug", "error"] = "error",
return_original_value: bool = False,
):
) -> str | None:
signing_key: Final = _get_salt_key()
try:

32
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -9,7 +9,7 @@ from fastapi import Request, UploadFile, status
from typing_extensions import NotRequired, ReadOnly, Required
from litellm._logging import verbose_proxy_logger
from litellm.constants import MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
from litellm.constants import CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
from litellm.proxy._types import ProxyException
from litellm.proxy.common_utils.callback_utils import (
get_metadata_variable_name_from_kwargs,
@ -189,8 +189,9 @@ async def _read_request_body(request: Request | None) -> dict:
try:
parsed_body = json.loads(body_str)
except json.JSONDecodeError:
# If both orjson and json.loads fail, throw a proper error
json.dumps(parsed_body, ensure_ascii=False).encode("utf-8")
except (json.JSONDecodeError, UnicodeEncodeError):
# json.loads accepts lone surrogate escapes that no provider can encode
verbose_proxy_logger.error("Invalid JSON payload received: %s", e)
raise ProxyException(
message=f"Invalid JSON payload: {e}",
@ -234,6 +235,13 @@ def _safe_get_request_parsed_body(request: Request | None) -> dict | None:
return None
def get_client_requested_model(request: Request | None) -> str | None:
if request is None or not hasattr(request, "scope"):
return None
model: Final = request.scope.get(CLIENT_REQUESTED_MODEL_SCOPE_KEY)
return model if isinstance(model, str) else None
def _safe_get_request_query_params(request: Request | None) -> dict:
if request is None:
return {}
@ -258,6 +266,24 @@ def _safe_set_request_parsed_body(
verbose_proxy_logger.debug("Unexpected error setting request parsed body - %s", e)
def rewrite_request_model(
request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader
request: Request | None,
model: str,
) -> None:
"""Point the auth-time payload, the parsed-body cache, ``request.json()`` and ``request.body()`` at ``model``.
The cache and raw body keep only the keys the client sent, not params auth merged into ``request_data``.
"""
request_data["model"] = model
if request is None:
return
cached_body: Final = _safe_get_request_parsed_body(request=request)
body: Final = {**cached_body, "model": model} if cached_body is not None else request_data
_safe_set_request_parsed_body(request=request, parsed_body=body)
request._json = body
request._body = orjson.dumps(body)
def _safe_get_request_headers(request: Request | None) -> dict:
"""
[Non-Blocking] Safely get the request headers.

104
litellm/proxy/common_utils/reset_budget_job.py
View file

@ -7,7 +7,7 @@ from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from enum import Enum
from types import MappingProxyType
from typing import Final, Literal, Protocol, TypeVar
from typing import Final, Generic, Literal, Protocol, TypeVar
from typing_extensions import assert_never
@ -68,6 +68,13 @@ from litellm.types.services import ServiceTypes
_RowT = TypeVar("_RowT")
@dataclass(frozen=True, slots=True)
class _RowReset(Generic[_RowT]):
row: _RowT
spend_decrement: float
_LINKED_KEYS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"budget_duration": None, "spend": {"gt": 0}})
_SPENT_ROWS_WHERE: Final[Mapping[str, object]] = MappingProxyType({"spend": {"gt": 0}})
@ -530,10 +537,9 @@ class ResetBudgetJob:
)
@staticmethod
async def _invalidate_spend_counter(counter_key: str, new_spend: float = 0.0) -> None:
"""Overwrite a spend counter with the post-reset value (0, or the carried
overage when budget rollover is enabled) so a DB-row reset takes effect
immediately.
async def _invalidate_spend_counter(counter_key: str) -> None:
"""Drop a spend counter so the next read reseeds from the committed DB
row, the only value that includes increments that raced the reset.
Call AFTER the DB write commits. Clearing Redis before the DB
commit opens a window where get_current_spend reads 0 from Redis
@ -542,10 +548,10 @@ class ResetBudgetJob:
try:
from litellm.proxy.proxy_server import spend_counter_cache
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_spend, ttl=60)
spend_counter_cache.in_memory_cache.delete_cache(key=counter_key)
if spend_counter_cache.redis_cache is not None:
try:
await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_spend, ttl=60)
await spend_counter_cache.redis_cache.async_delete_cache(key=counter_key)
except Exception as redis_err:
verbose_proxy_logger.warning(
"Failed to reset spend counter %s in Redis: %s. "
@ -730,8 +736,8 @@ class ResetBudgetJob:
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
for counter_key, new_spend in cascade.counter_resets:
await self._invalidate_spend_counter(counter_key, new_spend=new_spend)
for counter_key, _ in cascade.counter_resets:
await self._invalidate_spend_counter(counter_key)
for cache_key in cascade.cache_keys:
await self._invalidate_user_api_key_cache_entry(cache_key)
@ -842,7 +848,7 @@ class ResetBudgetJob:
)
return [LiteLLM_EndUserTable.model_validate(row.model_dump()) for row in rows]
async def _write_key_reset_updates(self, updated_keys: list[LiteLLM_VerificationToken]) -> None:
async def _write_key_reset_updates(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None:
"""
Write per-row {spend, budget_reset_at} updates for keys.
@ -858,18 +864,18 @@ class ResetBudgetJob:
reason="reset_budget_write_keys_failure",
)
async def _write_key_reset_updates_once(self, updated_keys: list[LiteLLM_VerificationToken]) -> None:
async def _write_key_reset_updates_once(self, updated_keys: Sequence[_RowReset[LiteLLM_VerificationToken]]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for k in updated_keys:
if k.token is None:
if k.row.token is None:
continue
uow.keys.queue_spend_reset(
token=k.token,
budget_reset_at=k.budget_reset_at,
spend_decrement=k.max_budget if (k.spend or 0.0) > 0.0 else None,
token=k.row.token,
budget_reset_at=k.row.budget_reset_at,
spend_decrement=k.spend_decrement,
)
async def _write_user_reset_updates(self, updated_users: list[LiteLLM_UserTable]) -> None:
async def _write_user_reset_updates(self, updated_users: Sequence[_RowReset[LiteLLM_UserTable]]) -> None:
"""
Write per-row {spend, budget_reset_at} updates for users.
@ -882,16 +888,16 @@ class ResetBudgetJob:
reason="reset_budget_write_users_failure",
)
async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None:
async def _write_user_reset_updates_once(self, updated_users: Sequence[_RowReset[LiteLLM_UserTable]]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for u in updated_users:
uow.users.queue_spend_reset(
user_id=u.user_id,
budget_reset_at=u.budget_reset_at,
spend_decrement=u.max_budget if (u.spend or 0.0) > 0.0 else None,
user_id=u.row.user_id,
budget_reset_at=u.row.budget_reset_at,
spend_decrement=u.spend_decrement,
)
async def _write_team_reset_updates(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
async def _write_team_reset_updates(self, updated_teams: Sequence[_RowReset[LiteLLM_TeamTable]]) -> None:
"""
Write per-row {spend, budget_reset_at} updates for teams.
@ -904,13 +910,13 @@ class ResetBudgetJob:
reason="reset_budget_write_teams_failure",
)
async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
async def _write_team_reset_updates_once(self, updated_teams: Sequence[_RowReset[LiteLLM_TeamTable]]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for t in updated_teams:
uow.teams.queue_spend_reset(
team_id=t.team_id,
budget_reset_at=t.budget_reset_at,
spend_decrement=t.max_budget if (t.spend or 0.0) > 0.0 else None,
team_id=t.row.team_id,
budget_reset_at=t.row.budget_reset_at,
spend_decrement=t.spend_decrement,
)
def _emit_phase_failure(
@ -962,18 +968,24 @@ class ResetBudgetJob:
reason="reset_budget_read_keys_failure",
)
verbose_proxy_logger.debug("Keys to reset %s", _LazyJson(keys_to_reset))
updated_keys: Final[list[LiteLLM_VerificationToken]] = []
updated_keys: Final[list[_RowReset[LiteLLM_VerificationToken]]] = []
failed_keys: Final = []
if keys_to_reset is not None and len(keys_to_reset) > 0:
for key in keys_to_reset:
try:
pre_reset_spend = float(key.spend or 0.0)
updated_key = await ResetBudgetJob._reset_budget_for_key(
key=key,
current_time=now,
reset_settings=self.reset_settings,
)
if updated_key is not None:
updated_keys.append(updated_key)
updated_keys.append(
_RowReset(
row=updated_key,
spend_decrement=pre_reset_spend - float(updated_key.spend or 0.0),
)
)
else:
failed_keys.append({"key": key, "error": "Returned None without exception"})
except Exception as e:
@ -985,15 +997,15 @@ class ResetBudgetJob:
if updated_keys:
await self._write_key_reset_updates(updated_keys=updated_keys)
for k in updated_keys:
token = getattr(k, "token", None)
token = getattr(k.row, "token", None)
if token:
await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0)
await self._invalidate_spend_counter(f"spend:key:{token}")
end_time = time.time()
outcome: Final = _ChunkOutcome(
fetched=len(keys_to_reset) if keys_to_reset else 0,
advanced=_count_advanced(
(k.budget_reset_at for k in updated_keys),
(k.row.budget_reset_at for k in updated_keys),
cutoff=datetime.now(timezone.utc),
),
)
@ -1063,18 +1075,24 @@ class ResetBudgetJob:
),
reason="reset_budget_read_users_failure",
)
updated_users: Final[list[LiteLLM_UserTable]] = []
updated_users: Final[list[_RowReset[LiteLLM_UserTable]]] = []
failed_users: Final = []
if users_to_reset is not None and len(users_to_reset) > 0:
for user in users_to_reset:
try:
pre_reset_spend = float(user.spend or 0.0)
updated_user = await ResetBudgetJob._reset_budget_for_user(
user=user,
current_time=now,
reset_settings=self.reset_settings,
)
if updated_user is not None:
updated_users.append(updated_user)
updated_users.append(
_RowReset(
row=updated_user,
spend_decrement=pre_reset_spend - float(updated_user.spend or 0.0),
)
)
else:
failed_users.append(
{
@ -1090,9 +1108,9 @@ class ResetBudgetJob:
if updated_users:
await self._write_user_reset_updates(updated_users=updated_users)
for u in updated_users:
user_id = getattr(u, "user_id", None)
user_id = getattr(u.row, "user_id", None)
if user_id:
await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0)
await self._invalidate_spend_counter(f"spend:user:{user_id}")
if user_id == LITELLM_PROXY_BUDGET_NAME:
await self._invalidate_global_proxy_spend_cache()
@ -1100,7 +1118,7 @@ class ResetBudgetJob:
outcome: Final = _ChunkOutcome(
fetched=len(users_to_reset) if users_to_reset else 0,
advanced=_count_advanced(
(u.budget_reset_at for u in updated_users),
(u.row.budget_reset_at for u in updated_users),
cutoff=datetime.now(timezone.utc),
),
)
@ -1172,18 +1190,24 @@ class ResetBudgetJob:
),
reason="reset_budget_read_teams_failure",
)
updated_teams: Final[list[LiteLLM_TeamTable]] = []
updated_teams: Final[list[_RowReset[LiteLLM_TeamTable]]] = []
failed_teams: Final = []
if teams_to_reset is not None and len(teams_to_reset) > 0:
for team in teams_to_reset:
try:
pre_reset_spend = float(team.spend or 0.0)
updated_team = await ResetBudgetJob._reset_budget_for_team(
team=team,
current_time=now,
reset_settings=self.reset_settings,
)
if updated_team is not None:
updated_teams.append(updated_team)
updated_teams.append(
_RowReset(
row=updated_team,
spend_decrement=pre_reset_spend - float(updated_team.spend or 0.0),
)
)
else:
failed_teams.append(
{
@ -1199,15 +1223,15 @@ class ResetBudgetJob:
if updated_teams:
await self._write_team_reset_updates(updated_teams=updated_teams)
for t in updated_teams:
team_id = getattr(t, "team_id", None)
team_id = getattr(t.row, "team_id", None)
if team_id:
await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0)
await self._invalidate_spend_counter(f"spend:team:{team_id}")
end_time = time.time()
outcome: Final = _ChunkOutcome(
fetched=len(teams_to_reset) if teams_to_reset else 0,
advanced=_count_advanced(
(t.budget_reset_at for t in updated_teams),
(t.row.budget_reset_at for t in updated_teams),
cutoff=datetime.now(timezone.utc),
),
)

1
litellm/proxy/db/create_views.py
View file

@ -80,6 +80,7 @@ async def create_missing_views(db: SupportsRawQueries) -> None:
t.max_budget AS team_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit,
p.project_alias AS project_alias
FROM "LiteLLM_VerificationToken" v
LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id

5
litellm/proxy/db/routing_prisma_wrapper.py
View file

@ -81,6 +81,11 @@ class WriterPinnedClient:
self.db: Final = db.writer if isinstance(db, RoutingPrismaWrapper) and not db.writer_unavailable else db
def writer_wrapper(db: "PrismaWrapper | RoutingPrismaWrapper") -> PrismaWrapper:
"""Unlike `WriterPinnedClient`, ignores `writer_unavailable`: a raw SQL write has no replica fallback."""
return db.writer if isinstance(db, RoutingPrismaWrapper) else db
class RoutingPrismaWrapper:
"""
Routes Prisma operations between a writer and a reader Prisma client.

136
litellm/proxy/hooks/batch_rate_limiter.py
View file

@ -18,12 +18,13 @@ Quick summary:
"""
import json
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, TypeAlias
from fastapi import HTTPException
from pydantic import BaseModel, Field, TypeAdapter
from pydantic import BaseModel, Field, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
@ -33,6 +34,7 @@ from litellm.batches.batch_utils import (
_extract_file_access_credentials,
_iter_batch_input_lines,
)
from litellm.constants import BATCH_TPD_DESCRIPTOR_SUFFIX, BATCH_TPD_WINDOW_SECONDS
from litellm.exceptions import RateLimitErrorCategory
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import (
@ -55,6 +57,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
from litellm.proxy.hooks.parallel_request_limiter_v3 import (
PROJECT_ITPM_DESCRIPTOR_KEY,
PROJECT_OTPM_DESCRIPTOR_KEY,
ReservationAwareIncrementOperation,
get_or_create_request_stash,
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
@ -92,6 +95,7 @@ else:
_BATCH_BODY_ADAPTER: Final = TypeAdapter(dict[str, object])
_WINDOW_START_ADAPTER: Final[TypeAdapter[int | float | str | None]] = TypeAdapter(int | float | str | None)
IncrementAmounts: TypeAlias = dict[Literal["requests", "tokens"], int]
@ -128,6 +132,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
self,
internal_usage_cache: InternalUsageCache,
parallel_request_limiter: ParallelRequestLimiter,
time_provider: Callable[[], datetime] | None = None,
):
"""
Initialize the batch rate limiter.
@ -138,9 +143,11 @@ class _PROXY_BatchRateLimiter(CustomLogger):
Args:
internal_usage_cache: Cache for storing rate limit data (auto-injected)
parallel_request_limiter: Existing rate limiter to integrate with (needs custom injection)
time_provider: Clock used for rate limit reset times (defaults to ``datetime.now``)
"""
self.internal_usage_cache = internal_usage_cache
self.parallel_request_limiter = parallel_request_limiter
self._time_provider: Final = time_provider or datetime.now
self._warned_unsupported_model_skip = False
def _get_file_bound_batch_model(self, data: dict) -> str | None:
@ -236,14 +243,48 @@ class _PROXY_BatchRateLimiter(CustomLogger):
file-bound/top-level routing model this function resolves. Charging
project quotas here would let a caller bind the file to a model
without a quota while rows execute against a quota-limited model.
Scopes with a ``tpd_limit`` (key, team, end user) are charged against a
daily token descriptor instead of their per-minute RPM/TPM descriptor,
because a batch's rows are scheduled by the provider and never share a
minute with the submission. The daily descriptor uses its own key so
its 24h window never collides with the online limiter's counters.
"""
return self.parallel_request_limiter._create_rate_limit_descriptors(
descriptors: Final = self.parallel_request_limiter._create_rate_limit_descriptors(
user_api_key_dict=user_api_key_dict,
data=data,
rpm_limit_type=None,
tpm_limit_type=None,
model_has_failures=False,
)
tpd_limits: Final[Mapping[str, tuple[str, int]]] = MappingProxyType(
{
key: (value, limit)
for key, value, limit in (
("api_key", user_api_key_dict.api_key, user_api_key_dict.tpd_limit),
("team", user_api_key_dict.team_id, user_api_key_dict.team_tpd_limit),
("end_user", user_api_key_dict.end_user_id, user_api_key_dict.end_user_tpd_limit),
)
if value and limit is not None
}
)
if not tpd_limits:
return descriptors
return [
*(d for d in descriptors if d["key"] not in tpd_limits),
*(
RateLimitDescriptor(
key=f"{key}{BATCH_TPD_DESCRIPTOR_SUFFIX}",
value=value,
rate_limit={
"requests_per_unit": None,
"tokens_per_unit": limit,
"window_size": BATCH_TPD_WINDOW_SECONDS,
},
)
for key, (value, limit) in tpd_limits.items()
),
]
@staticmethod
def _project_has_any_io_token_limits(user_api_key_dict: UserAPIKeyAuth) -> bool:
@ -583,9 +624,14 @@ class _PROXY_BatchRateLimiter(CustomLogger):
batch_usage: BatchFileUsage,
limit_type: str,
requested_model: str | None = None,
window_start: int | None = None,
) -> NoReturn:
"""Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded."""
from datetime import datetime
"""Raise :class:`ProxyRateLimitError` (a 429) for batch rate limit exceeded.
``window_start`` is the active counter window's start (unix seconds) when
known, so the reset time reflects that window's actual end rather than a
full window from now.
"""
# Find the descriptor for this status. Matching on (key, value) is
# required, not key alone: a batch can carry several project ITPM/OTPM
@ -609,9 +655,12 @@ class _PROXY_BatchRateLimiter(CustomLogger):
descriptors[descriptor_index] if descriptors else {"key": "", "value": "", "rate_limit": None}
)
now: Final = datetime.now().timestamp()
window_size: Final = self.parallel_request_limiter.window_size
reset_time: Final = now + window_size
now: Final = self._time_provider().timestamp()
window_size: Final = (descriptor.get("rate_limit") or {}).get(
"window_size"
) or self.parallel_request_limiter.window_size
reset_time: Final = now + window_size if window_start is None else window_start + window_size
retry_after: Final = max(0, int(reset_time - now))
reset_time_formatted: Final = datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S UTC")
remaining_display: Final = max(0, status["limit_remaining"])
@ -643,10 +692,13 @@ class _PROXY_BatchRateLimiter(CustomLogger):
if descriptor.get("key") == PROJECT_ITPM_DESCRIPTOR_KEY
else batch_usage.total_tokens
)
token_limit_label: Final = (
"TPD" if descriptor.get("key", "").endswith(BATCH_TPD_DESCRIPTOR_SUFFIX) else "TPM"
)
detail = (
f"Batch rate limit exceeded for {descriptor.get('key', 'unknown')}: {descriptor.get('value', 'unknown')}. "
f"Batch contains {batch_token_count} tokens but only {remaining_display} tokens remaining "
f"out of {current_limit} TPM limit. "
f"out of {current_limit} {token_limit_label} limit. "
f"Limit resets at: {reset_time_formatted}"
)
@ -654,7 +706,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
raise ProxyRateLimitError(
detail=detail,
headers={
"retry-after": str(window_size),
"retry-after": str(retry_after),
"rate_limit_type": limit_type,
"reset_at": reset_time_formatted,
},
@ -712,6 +764,8 @@ class _PROXY_BatchRateLimiter(CustomLogger):
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash: Final = get_or_create_request_stash()
stash.batch_tpd_refund_ops = ()
if rate_limit_response["overall_code"] == "OVER_LIMIT":
requested_model: Final = data.get("model") if data else None
for status in rate_limit_response["statuses"]:
@ -722,8 +776,70 @@ class _PROXY_BatchRateLimiter(CustomLogger):
batch_usage,
status["rate_limit_type"],
requested_model=requested_model,
window_start=await self._read_tpd_window_start(
status=status, parent_otel_span=user_api_key_dict.parent_otel_span
),
)
stash.batch_tpd_refund_ops = self._build_tpd_refund_ops(
descriptors=descriptors,
tokens=batch_usage.total_tokens,
reservation_windows=rate_limit_response.get("reservation_windows", frozenset()),
)
async def _read_tpd_window_start(self, status: "RateLimitStatus", parent_otel_span: "Span | None") -> int | None:
descriptor_key: Final = status.get("descriptor_key") or ""
if not descriptor_key.endswith(BATCH_TPD_DESCRIPTOR_SUFFIX):
return None
try:
window_start: Final = _WINDOW_START_ADAPTER.validate_python(
await self.parallel_request_limiter.internal_usage_cache.async_get_cache(
key=f"{{{descriptor_key}:{status.get('descriptor_value') or ''}}}:window",
litellm_parent_otel_span=parent_otel_span,
),
strict=True,
)
return None if window_start is None else int(float(window_start))
except (ValidationError, ValueError):
return None
def _build_tpd_refund_ops(
self,
descriptors: Sequence["RateLimitDescriptor"],
tokens: int,
reservation_windows: frozenset[tuple[str, str, Literal["redis", "local"]]],
) -> tuple[ReservationAwareIncrementOperation, ...]:
"""Refund operations for the daily token counters this batch charged.
The v3 limiter's failure hook applies them when the submission fails
after the counters were incremented. Each operation carries the window
identity the charge landed in, so the refund is skipped once that
window has rolled over.
"""
if tokens <= 0 or not reservation_windows:
return ()
tpd_descriptors_by_counter: Final[Mapping[str, RateLimitDescriptor]] = MappingProxyType(
{
self.parallel_request_limiter.create_rate_limit_keys(
descriptor["key"], descriptor["value"], "tokens"
): descriptor
for descriptor in descriptors
if descriptor["key"].endswith(BATCH_TPD_DESCRIPTOR_SUFFIX)
}
)
return tuple(
ReservationAwareIncrementOperation(
key=counter_key,
increment_value=-tokens,
ttl=BATCH_TPD_WINDOW_SECONDS,
window_key=f"{{{descriptor['key']}:{descriptor['value']}}}:window",
expected_window_start=window_start,
reservation_backend=backend,
)
for counter_key, window_start, backend in sorted(reservation_windows)
if (descriptor := tpd_descriptors_by_counter.get(counter_key)) is not None
)
async def count_input_file_usage(
self,
file_id: str,

15
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -396,6 +396,8 @@ CacheCounterValue: TypeAlias = int | float | str | bytes
CacheCounterValues: TypeAlias = Sequence[CacheCounterValue | None]
ReservationWindowIdentity: TypeAlias = tuple[str, str, Literal["redis", "local"]]
ParallelGaugeCacheValue: TypeAlias = dict[str, object] | int | float | str | bytes
@ -542,6 +544,7 @@ class RequestRateLimiterStash:
default_factory=frozenset
)
batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
reservation_released: bool = False
@ -683,6 +686,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._batch_rate_limiter = _PROXY_BatchRateLimiter(
internal_usage_cache=self.internal_usage_cache,
parallel_request_limiter=self,
time_provider=self._time_provider,
)
except Exception as e:
verbose_proxy_logger.debug("Could not load batch rate limiter: %s", e)
@ -1823,6 +1827,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
applied: Final[list[list[AtomicCounterMeta]]] = []
statuses: Final[list[RateLimitStatus]] = []
reservation_windows: Final[set[ReservationWindowIdentity]] = set() # mutable-ok: filled by the group loop
raw: list[CacheCounterValue]
for _idx, (keys, args, meta) in enumerate(descriptor_groups):
@ -1860,11 +1865,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return response
applied.append(meta)
statuses.extend(response["statuses"])
reservation_windows.update(response.get("reservation_windows", frozenset()))
return RateLimitResponse(
overall_code="OK",
statuses=statuses,
reservation_windows=frozenset(),
reservation_windows=frozenset(reservation_windows),
)
async def _refund_applied_descriptor_groups(
@ -4824,6 +4830,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
stash.batch_enqueued_reservation = None
if stash.batch_tpd_refund_ops:
await self.async_increment_reservation_aware_tokens(
pipeline_operations=stash.batch_tpd_refund_ops,
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.batch_tpd_refund_ops = ()
if stash.reservation_released:
return
reserved_tokens: Final = stash.reserved_tokens

28
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -652,6 +652,10 @@ async def _update_database_and_spend_counters(
request_tags: list[str] | None = None,
model_access_groups: Sequence[str] | None = None,
) -> bool:
if budget_reservation is not None:
await _reconcile_budget_reservation_before_db_update(
budget_reservation=budget_reservation, response_cost=response_cost
)
try:
charged: Final = await proxy_logging_obj.db_spend_update_writer.update_database(
token=user_api_key,
@ -709,6 +713,30 @@ async def _update_database_and_spend_counters(
return True
async def _reconcile_budget_reservation_before_db_update(
budget_reservation: dict, # mutable-ok: reconcile_budget_reservation stamps applied_adjustment on the caller's shared reservation dict
response_cost: float,
) -> None:
from litellm.proxy.spend_tracking.budget_reservation import reconcile_budget_reservation
try:
await reconcile_budget_reservation(
budget_reservation=budget_reservation, actual_cost=response_cost, finalize=False
)
except Exception: # noqa: BLE001 # a failed reconcile must not block the spend write; the counters are dropped instead
verbose_proxy_logger.warning(
"Failed to reconcile budget reservation before persisting spend; invalidating reserved counters"
)
try:
await _invalidate_budget_reservation_counters(budget_reservation=budget_reservation)
except Exception: # noqa: BLE001 # nothing left to try; the finalized stamp below keeps it from being reprocessed
verbose_proxy_logger.exception(
"Failed to invalidate budget reservation counters after pre-persist reconcile failed"
)
finally:
budget_reservation["finalized"] = True # rebind-ok: the counter update reads the stamp off the shared dict
async def _release_budget_reservation(budget_reservation: dict | None) -> None:
if budget_reservation is None:
return

117
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -40,6 +40,14 @@ from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
refresh_proxy_server_request_body_snapshot,
)
from litellm.proxy.management_endpoints.common_utils import (
_is_user_team_admin, # pyright: ignore[reportPrivateUsage] # shared owner of team-admin membership
)
from litellm.proxy.management_helpers.auto_router_permissions import (
authorize_member_auto_router_dependencies,
authorize_member_auto_router_team,
validate_member_auto_router_config,
)
from litellm.repositories.autorouter_session_repository import AutoRouterSessionRepository
from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.team_repository import TeamRepository
@ -72,13 +80,13 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
)
if TYPE_CHECKING:
from fastapi import APIRouter, Depends, HTTPException, Query, status
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
else:
try:
from fastapi import APIRouter, Depends, HTTPException, Query, status
from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
except ImportError:
# fastapi is only required for proxy, not for SDK usage
pass
@ -201,21 +209,14 @@ async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -
return await prisma_client.db.query_raw(query, *args)
async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> None:
"""Allow exactly the callers who could create this router.
Both dry runs are gated like the write they rehearse rather than as reads: a proxy
admin, or a team admin naming their own team, matching /model/new. Routing a test
prompt can also spend money (an `llm` classifier config calls its classifier, a
semantic config embeds the prompt), so a read-level gate would be too loose anyway.
"""
async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None:
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelManagementAuthChecks,
)
from litellm.proxy.proxy_server import premium_user, prisma_client
if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
return
return None
if team_id is None:
raise HTTPException(
@ -244,12 +245,47 @@ async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id:
},
)
ModelManagementAuthChecks.can_user_make_team_model_call(
team_id=team_id,
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
if _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team):
ModelManagementAuthChecks.can_user_make_team_model_call(
team_id=team_id,
user_api_key_dict=user_api_key_dict,
team_obj=team,
premium_user=premium_user,
)
return None
authorize_member_auto_router_team(
user_api_key_dict=user_api_key_dict,
team_obj=LiteLLM_TeamTable.model_validate(team_row.model_dump()),
team=team,
premium_user=premium_user,
)
return team
async def _authorize_member_dry_run_config(
*,
config: Mapping[str, object],
default_model: str | None,
user_api_key_dict: UserAPIKeyAuth,
team: LiteLLM_TeamTable,
) -> UserAPIKeyAuth:
from litellm.proxy.proxy_server import llm_router, prisma_client
if prisma_client is None or llm_router is None:
raise HTTPException(status_code=503, detail="Cannot verify auto-router model access")
validated: Final = validate_member_auto_router_config(config)
scoped_actor: Final = user_api_key_dict.model_copy(
update=MappingProxyType({"team_id": team.team_id, "team_models": team.models, "org_id": team.organization_id})
)
await authorize_member_auto_router_dependencies(
config=validated,
default_model=default_model,
user_api_key_dict=scoped_actor,
team=team,
prisma_client=prisma_client,
llm_router=llm_router,
)
return scoped_actor
def _models_this_test_can_call(config: RequestComplexityRouterConfig) -> tuple[str, ...]:
@ -326,16 +362,23 @@ async def validate_complexity_router_config(
Runs the same check every write path runs (the router's own pydantic model), so a form can
show the backend's exact verdict while the operator is still editing rather than after a
rejected save. Gated exactly like the save it rehearses: a proxy admin, or a team admin
naming their own team. Nothing is created, routed, or billed.
rejected save. Uses the same team opt-in and model-access checks as configuration
writes for members. Nothing is created, routed, or billed.
"""
await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
from litellm.router_utils.auto_router_model_naming import (
validate_complexity_router_config_write,
)
error: Final = validate_complexity_router_config_write(data.complexity_router_config)
if error is None and member_team is not None:
await _authorize_member_dry_run_config(
config=data.complexity_router_config,
default_model=None,
user_api_key_dict=user_api_key_dict,
team=member_team,
)
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
@ -349,6 +392,7 @@ async def validate_complexity_router_config(
async def preview_auto_router_routing(
data: AutoRouterRoutingTestRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
http_request: Request,
) -> AutoRouterRoutingTestResponse:
"""
Route a single request through a complexity-router config and report where it landed.
@ -392,7 +436,34 @@ async def preview_auto_router_routing(
)
from litellm.proxy.utils import get_available_models_for_user
await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
actor: Final = (
await _authorize_member_dry_run_config(
config=data.complexity_router_config.model_dump(exclude_none=True),
default_model=data.default_model,
user_api_key_dict=user_api_key_dict,
team=member_team,
)
if member_team is not None
else user_api_key_dict
)
request_data: Final[dict[str, object]] = { # mutable-ok: auth and routing enrich this request in place
**data.wire_body(),
"metadata": {}, # mutable-ok: centralized auth and identity stamping share this metadata bucket
"proxy_server_request": {"body": None}, # mutable-ok: the snapshot owner fills this body in place
}
if member_team is not None and _models_this_test_can_call(data.complexity_router_config):
from litellm.proxy.auth.user_api_key_auth import (
_run_centralized_common_checks, # pyright: ignore[reportPrivateUsage] # reuse the serving admission policy
)
await _run_centralized_common_checks(
user_api_key_auth_obj=actor,
request=http_request,
request_data=request_data,
route="/auto_router/test_routing",
)
if llm_router is None:
raise HTTPException(
@ -404,7 +475,7 @@ async def preview_auto_router_routing(
await _authorize_models_this_test_can_call(
config=data.complexity_router_config,
user_api_key_dict=user_api_key_dict,
user_api_key_dict=actor,
llm_router=llm_router,
)
@ -417,12 +488,8 @@ async def preview_auto_router_routing(
)
request_kwargs: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
data={ # mutable-ok: the request-metadata helper takes and returns request kwargs as a dict
**data.wire_body(),
"metadata": {}, # mutable-ok: the request-metadata helper writes the auth fields into this dict
"proxy_server_request": {"body": None}, # mutable-ok: the snapshot owner fills body in place
},
user_api_key_dict=user_api_key_dict,
data=request_data,
user_api_key_dict=actor,
_metadata_variable_name="metadata",
)
refresh_proxy_server_request_body_snapshot(request_kwargs)

3
litellm/proxy/management_endpoints/budget_management_endpoints.py
View file

@ -52,6 +52,7 @@ async def new_budget(
- max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
- tpm_limit: Optional[int] - The tokens per minute limit for the budget.
- rpm_limit: Optional[int] - The requests per minute limit for the budget.
- tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
- model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
- budget_reset_at: Optional[datetime] - Datetime when the initial budget is reset. Default is now.
"""
@ -135,6 +136,7 @@ async def update_budget(
- max_parallel_requests: Optional[int] - The max number of parallel requests for the budget.
- tpm_limit: Optional[int] - The tokens per minute limit for the budget.
- rpm_limit: Optional[int] - The requests per minute limit for the budget.
- tpd_limit: Optional[int] - The tokens per day limit for the budget. Charged by batch submissions instead of tpm_limit/rpm_limit.
- model_max_budget: Optional[dict] - Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d", "tpm_limit": 100000, "rpm_limit": 100000}}
- budget_reset_at: Optional[datetime] - Update the Datetime when the budget was last reset.
"""
@ -272,6 +274,7 @@ async def budget_settings(
"max_parallel_requests": {"type": "Integer"},
"tpm_limit": {"type": "Integer"},
"rpm_limit": {"type": "Integer"},
"tpd_limit": {"type": "Integer"},
"budget_duration": {"type": "String"},
"max_budget": {"type": "Float"},
"soft_budget": {"type": "Float"},

1
litellm/proxy/management_endpoints/customer_endpoints.py
View file

@ -335,6 +335,7 @@ async def new_end_user(
- budget_duration: Optional[str] - Budget is reset at the end of specified duration. If not set, budget is never reset. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
- tpm_limit: Optional[int] - [Not Implemented Yet] Specify tpm limit for a given customer (Tokens per minute)
- rpm_limit: Optional[int] - [Not Implemented Yet] Specify rpm limit for a given customer (Requests per minute)
- tpd_limit: Optional[int] - Specify tpd limit for a given customer (Tokens per day). Batch submissions are charged against it instead of tpm_limit/rpm_limit
- model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}}
- max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer.
- soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests.

25
litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
View file

@ -28,6 +28,9 @@ class _JWTKeyMappingRecord(Protocol):
@property
def id(self) -> str: ...
@property
def jwt_issuer(self) -> str: ...
@property
def jwt_claim_name(self) -> str: ...
@ -78,6 +81,7 @@ def _to_response(mapping: _JWTKeyMappingRecord) -> JWTKeyMappingResponse:
"""Convert a Prisma mapping object to a safe response (no hashed token)."""
return JWTKeyMappingResponse(
id=mapping.id,
jwt_issuer=mapping.jwt_issuer or None,
jwt_claim_name=mapping.jwt_claim_name,
jwt_claim_value=mapping.jwt_claim_value,
description=mapping.description,
@ -109,6 +113,7 @@ async def create_jwt_key_mapping(
try:
hashed_key: Final = hash_token(data.key)
create_data: Final = {
"jwt_issuer": data.jwt_issuer or "",
"jwt_claim_name": data.jwt_claim_name,
"jwt_claim_value": data.jwt_claim_value,
"token": hashed_key,
@ -120,7 +125,7 @@ async def create_jwt_key_mapping(
new_mapping: Final = await _mapping_table(prisma_client).create(data=create_data)
cache_key: Final = jwt_key_mapping_cache_key(data.jwt_claim_name, data.jwt_claim_value)
cache_key: Final = jwt_key_mapping_cache_key(data.jwt_claim_name, data.jwt_claim_value, data.jwt_issuer)
await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache)
return _to_response(new_mapping)
@ -131,7 +136,10 @@ async def create_jwt_key_mapping(
if "unique" in error_str or "p2002" in error_str:
raise HTTPException(
status_code=409,
detail=f"A mapping for claim '{data.jwt_claim_name}' = '{data.jwt_claim_value}' already exists.",
detail=(
f"A mapping for claim '{data.jwt_claim_name}' = '{data.jwt_claim_value}' "
f"already exists for issuer '{data.jwt_issuer}'."
),
)
if "foreign" in error_str or "p2003" in error_str:
raise HTTPException(
@ -161,6 +169,9 @@ async def update_jwt_key_mapping(
update_data: Final = data.model_dump(exclude_unset=True, exclude={"id", "key"})
if data.key is not None:
update_data["token"] = hash_token(data.key)
if "jwt_issuer" in update_data:
# DB column is NOT NULL (see schema.prisma); "" is the global/unscoped sentinel.
update_data["jwt_issuer"] = update_data["jwt_issuer"] or ""
update_data["updated_by"] = user_api_key_dict.user_id
try:
@ -178,9 +189,11 @@ async def update_jwt_key_mapping(
# Evict only after the write commits: a concurrent request between an
# early eviction and the commit would re-cache the old mapping and keep
# it authorized until TTL.
old_cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value)
old_cache_key: Final = jwt_key_mapping_cache_key(
old_mapping.jwt_claim_name, old_mapping.jwt_claim_value, old_mapping.jwt_issuer
)
new_cache_key: Final = jwt_key_mapping_cache_key(
updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value
updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value, updated_mapping.jwt_issuer
)
cache_keys: Final = (old_cache_key,) if old_cache_key == new_cache_key else (old_cache_key, new_cache_key)
await evict_and_broadcast(cache_keys=cache_keys, user_api_key_cache=user_api_key_cache)
@ -227,7 +240,9 @@ async def delete_jwt_key_mapping(
# Evict only after the row is gone, else a concurrent request can
# re-cache the deleted mapping and keep it authorized until TTL.
cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value)
cache_key: Final = jwt_key_mapping_cache_key(
old_mapping.jwt_claim_name, old_mapping.jwt_claim_value, old_mapping.jwt_issuer
)
await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache)
return {"status": "success"}
except HTTPException:

9
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -1078,7 +1078,9 @@ async def validate_team_id_used_in_service_account_request(
return True
_BUDGET_NUMERIC_KEYS = frozenset(["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit"])
_BUDGET_NUMERIC_KEYS = frozenset(
["max_budget", "soft_budget", "max_parallel_requests", "tpm_limit", "rpm_limit", "tpd_limit"]
)
def _enforce_upperbound_key_params(
@ -1957,6 +1959,7 @@ async def generate_key_fn(
- blocked: Optional[bool] - Whether the key is blocked.
- rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
- tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
- tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
- soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
- tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
- prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
@ -2163,6 +2166,7 @@ async def generate_service_account_key_fn(
- blocked: Optional[bool] - Whether the key is blocked.
- rpm_limit: Optional[int] - Specify rpm limit for a given key (Requests per minute)
- tpm_limit: Optional[int] - Specify tpm limit for a given key (Tokens per minute)
- tpd_limit: Optional[int] - Specify tpd limit for a given key (Tokens per day). Charged by batch submissions instead of tpm_limit/rpm_limit.
- soft_budget: Optional[float] - Specify soft budget for a given key. Will trigger a slack alert when this soft budget is reached.
- tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
- enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
@ -3192,6 +3196,7 @@ async def update_key_fn(
- metadata: Optional[dict] - Metadata for key. Example {"team": "core-infra", "app": "app2"}
- tpm_limit: Optional[int] - Tokens per minute limit
- rpm_limit: Optional[int] - Requests per minute limit
- tpd_limit: Optional[int] - Tokens per day limit, charged by batch submissions instead of tpm_limit/rpm_limit
- model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200}
- mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200}
- tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit.
@ -4355,6 +4360,7 @@ async def generate_key_helper_fn(
metadata: dict | None = {},
tpm_limit: int | None = None,
rpm_limit: int | None = None,
tpd_limit: int | None = None,
query_type: Literal["insert_data", "update_data"] = "insert_data",
update_key_values: dict | None = None,
key_alias: str | None = None,
@ -4503,6 +4509,7 @@ async def generate_key_helper_fn(
"metadata": metadata_json,
"tpm_limit": tpm_limit,
"rpm_limit": rpm_limit,
"tpd_limit": tpd_limit,
"budget_duration": key_budget_duration,
"budget_reset_at": key_reset_at,
"allowed_cache_controls": allowed_cache_controls,

5
litellm/proxy/management_endpoints/management_v1/budgets.py
View file

@ -58,6 +58,7 @@ class BudgetListItem(BaseModel):
soft_budget: float | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpd_limit: int | None = None
budget_duration: str | None = None
budget_reset_at: datetime | None = None
created_at: datetime
@ -123,7 +124,7 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
resource="budgets",
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "created_at")),
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
searchable=frozenset(("budget_id",)),
filters=BUDGET_FILTERS,
default_sort=(SortKey(field="created_at", descending=True),),
@ -154,7 +155,7 @@ async def list_budgets(
way to page, sort or filter it.
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
`rpm_limit` or `created_at`, each optionally prefixed with `-` for descending,
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. `budget_id` is appended to every sort as the
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
`page_size` defaults to 50 and is capped at 100. Filters are

273
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -15,13 +15,16 @@ import datetime
import json
from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass
from fnmatch import fnmatchcase
from json import JSONDecodeError
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
@ -51,6 +54,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY
from litellm.proxy.auth.team_grants import team_model_aliases
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.config_sync_pubsub import (
coordination_redis_cache,
@ -65,6 +69,7 @@ from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin
from litellm.proxy.management_endpoints.team_endpoints import (
_refresh_cached_team,
append_team_models,
team_model_add,
team_model_delete,
)
@ -76,6 +81,13 @@ from litellm.proxy.management_helpers.access_group_model_sync import (
sync_access_groups_for_renamed_model,
)
from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
from litellm.proxy.management_helpers.auto_router_permissions import (
MemberAutoRouterWrite,
StoredAutoRouterIdentity,
authorize_member_auto_router_dependencies,
authorize_member_auto_router_team,
authorize_member_auto_router_write,
)
from litellm.proxy.spend_tracking.ptu_feature_flag import (
PTU_COST_ATTRIBUTION_ENV_VAR,
is_ptu_cost_attribution_enabled,
@ -122,12 +134,14 @@ from litellm.types.router import (
GenericLiteLLMParams,
ModelInfo,
updateDeployment,
updateLiteLLMParams,
)
from litellm.types.utils import without_server_derived_pricing
from litellm.utils import get_utc_datetime
if TYPE_CHECKING:
from prisma import models as prisma_models
from prisma import types as prisma_types
router: Final = APIRouter()
@ -180,6 +194,24 @@ class _ProxyModelTable(Protocol):
class _TxModelTables(Protocol):
litellm_proxymodeltable: _ProxyModelTable
async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
@runtime_checkable
class _TransactionFactory(Protocol):
def __call__(self, *, timeout: datetime.timedelta = ...) -> AbstractAsyncContextManager[_TxModelTables]: ...
class _ModelTransactionClient(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True)
tx: _TransactionFactory
@dataclass(frozen=True, slots=True)
class _TransactionClient:
db: _TxModelTables
_RowT = TypeVar("_RowT")
@ -213,7 +245,7 @@ def _proxy_model_table(prisma_client: PrismaClient) -> _ProxyModelTable:
def _repo_team_table(prisma_client: PrismaClient) -> _TeamLookupTable:
return TeamRepository(prisma_client).table
return TeamRepository(WriterPinnedClient(prisma_client.db)).table
def _db_team_table(prisma_client: PrismaClient) -> _TeamTable:
@ -353,6 +385,25 @@ def _effective_complexity_router_params(
)
def _member_auto_router_marker_for_update(
*,
incoming_params: updateLiteLLMParams | None,
existing: Deployment,
member_write: MemberAutoRouterWrite | None,
) -> bool | None:
if member_write is not None:
return True
if not existing.model_info.member_auto_router:
return None
if incoming_params is None:
return True
if any(getattr(incoming_params, field, None) is not None for field in STRATEGY_ROUTER_PARAM_FIELDS):
return False
if incoming_params.model is not None and incoming_params.model != _effective_model(None, existing.litellm_params):
return False
return True
def _decrypted_model(stored_model: object) -> str | None:
if not isinstance(stored_model, str):
return None
@ -385,7 +436,11 @@ def _raise_on_tuning_quota_violation(
@asynccontextmanager
async def _auto_router_capability_slot(
prisma_client: PrismaClient, *, effective_params: Mapping[str, object], model_id: str | None
prisma_client: PrismaClient,
*,
effective_params: Mapping[str, object],
model_id: str | None,
member_write: MemberAutoRouterWrite | None = None,
) -> AsyncGenerator[_ProxyModelTable, None]:
"""Hand out the model table to write through while the row's claim on a licensed capability is settled.
@ -394,9 +449,8 @@ async def _auto_router_capability_slot(
(a statement's snapshot predates anything it locks), so pods cannot both pass the count:
the DB rows (any pod, either JSON shape) plus this proxy's config.yaml routers are judged
against the license limit and the write is refused with a 403 before it happens. The row
being edited keeps its own slot through ``model_id``. Every other write, and every write on
an unlimited license, goes through the repository table with no lock. Only the row write
itself may run inside: anything that needs a second connection (the team model bookkeeping)
being edited keeps its own slot through ``model_id``. Member writes also recheck their
authorization under this lock. Team model bookkeeping needs a second connection and
must wait until the transaction has committed and the lock is released. The transaction
writes bypass the repository's publish-on-write, so the config change is published once
after commit, the way delete_team_models does.
@ -408,6 +462,7 @@ async def _auto_router_capability_slot(
_license_check, # pyright: ignore[reportPrivateUsage] # existing capability slot reads the proxy license singleton
heuristic_v1_tuning_baselines,
llm_router,
premium_user,
)
limit: Final = _license_check.auto_router_capability_limit()
@ -415,13 +470,96 @@ async def _auto_router_capability_slot(
baselines: Final = heuristic_v1_tuning_baselines
tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
if limit is None or (capability is None and not judges_tuning):
if member_write is None and (limit is None or (capability is None and not judges_tuning)):
yield _proxy_model_table(prisma_client)
return
async with prisma_client.db.tx() as tx_ctx:
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
transaction: Final = (
transaction_client.tx(timeout=datetime.timedelta(seconds=30))
if member_write is not None
else transaction_client.tx()
)
async with transaction as tx_ctx:
tables: Final[_TxModelTables] = tx_ctx
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
if member_write is not None:
if member_write.model_id is not None:
await tx_ctx.query_raw(
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE',
member_write.model_id,
)
pinned_client: Final = _TransactionClient(tx_ctx)
team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id}
team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True}
team_row: Final = await TeamRepository(pinned_client).table.find_unique(
where=team_where, include=team_include
)
if team_row is None or llm_router is None:
raise HTTPException(status_code=403, detail="The auto router's team or model catalog is unavailable.")
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
authorize_member_auto_router_team(
user_api_key_dict=member_write.actor, team=team, premium_user=premium_user
)
if member_write.model_id is not None:
model_where: Final[prisma_types.LiteLLM_ProxyModelTableWhereInput] = {"model_id": member_write.model_id}
current_row: Final = await tables.litellm_proxymodeltable.find_unique(where=model_where)
current_identity: Final = (
StoredAutoRouterIdentity.model_validate(current_row.model_dump())
if current_row is not None
else None
)
current_model: Final = (
Deployment.model_validate(current_row.model_dump()) if current_row is not None else None
)
if (
current_identity is None
or current_identity.created_by != member_write.actor.user_id
or current_model is None
or current_model.model_info.team_id != member_write.team_id
):
raise HTTPException(status_code=403, detail="Team members can update only their own auto routers.")
if current_identity.updated_at != member_write.updated_at:
raise HTTPException(status_code=409, detail="This auto router changed. Reload it before updating.")
else:
all_models: Final[prisma_types.LiteLLM_ProxyModelTableWhereInput] = {}
rows_for_names: Final = await tables.litellm_proxymodeltable.find_many(where=all_models)
stored_names: Final = tuple(
(
row.model_name,
model_info_as_mapping(row.model_info),
)
for row in rows_for_names
)
config_names: Final = tuple(
(str(row.get("model_name", "")), model_info_as_mapping(row.get("model_info")))
for row in config_rows
)
team_aliases: Final = team_model_aliases(team)
aliases: Final = (
*(llm_router.model_group_alias or ()),
*(litellm.model_alias_map or ()),
*(team_aliases or ()),
)
if member_write.public_name in aliases or any(
fnmatchcase(
member_write.public_name,
str(info.get("team_public_model_name") or name)
if info is not None and info.get("team_id") == member_write.team_id
else name,
)
for name, info in (*stored_names, *config_names)
if info is None or info.get("team_id") in (None, member_write.team_id)
):
raise HTTPException(status_code=409, detail="This auto-router name is already used by a model.")
await authorize_member_auto_router_dependencies(
config=member_write.config,
default_model=member_write.default_model,
user_api_key_dict=member_write.actor,
team=team,
prisma_client=pinned_client,
llm_router=llm_router,
)
if capability is not None:
rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(
_CAPABILITY_DB_ROWS_SQL[capability.key], model_id or ""
@ -434,7 +572,7 @@ async def _auto_router_capability_slot(
status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
)
if judges_tuning and baselines is not None:
model_rows: Final = await ModelRepository(WriterPinnedClient(tx_ctx)).find_all_except(model_id or "")
model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "")
_raise_on_tuning_quota_violation(
candidate=tuning_candidate,
others=tuple(
@ -883,11 +1021,39 @@ async def patch_model(
param=None,
)
await ModelManagementAuthChecks.can_user_make_model_call(
write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
model_params=db_model,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
premium_user=premium_user,
member_operation="update",
incoming_model_params=patch_data,
)
member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
member_marker: Final = _member_auto_router_marker_for_update(
incoming_params=patch_data.litellm_params, existing=db_model, member_write=member_write
)
marker_info: Final = (
ModelInfo(id=db_model.model_info.id)
if member_write is not None
else patch_data.model_info or ModelInfo(id=db_model.model_info.id)
)
effective_info: Final = (
marker_info.model_copy(update=MappingProxyType({"member_auto_router": member_marker}))
if member_marker is not None
else patch_data.model_info
)
effective_patch: Final = (
patch_data.model_copy(
update=MappingProxyType(
{
"model_name": None if member_write is not None else patch_data.model_name,
"model_info": effective_info,
}
)
)
if member_marker is not None
else patch_data
)
# Pause/resume (`blocked`) is a proxy-admin-only privilege. Team admins
@ -933,13 +1099,14 @@ async def patch_model(
prisma_client,
effective_params=effective_params,
model_id=model_id,
member_write=member_write,
) as table:
return await table.update(where={"model_id": model_id}, data=update_data)
# Handle team model updates with proper alias management
updated_model: Final = await _update_team_model_in_db(
db_model=db_model,
patch_data=patch_data,
patch_data=effective_patch,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
write_row=write_row,
@ -1218,7 +1385,7 @@ async def _add_team_model_to_db(
user_api_key_dict: UserAPIKeyAuth,
prisma_client: PrismaClient,
slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
) -> "_ProxyModelRow | LiteLLM_ProxyModelTable | None":
"""
If 'team_id' is provided,
@ -1226,6 +1393,8 @@ async def _add_team_model_to_db(
- store the model in the db with the unique 'model_name'
- add the public model name to the team's allowed models list
"""
from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
_team_id: Final = model_params.model_info.team_id
if _team_id is None:
return None
@ -1253,13 +1422,14 @@ async def _add_team_model_to_db(
)
if original_model_name:
await team_model_add(
await append_team_models(
data=TeamModelAddRequest(
team_id=_team_id,
models=[original_model_name],
),
http_request=Request(scope={"type": "http"}),
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
return model_response
@ -1787,9 +1957,16 @@ class ModelManagementAuthChecks:
prisma_client: PrismaClient,
premium_user: bool,
allow_missing_team: bool = False,
) -> Literal[True]:
member_operation: Literal["create", "update"] | None = None,
incoming_model_params: updateDeployment | None = None,
) -> Literal[True] | MemberAutoRouterWrite:
if user_api_key_dict.user_role in (
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
):
raise HTTPException(status_code=403, detail="View-only users cannot manage models.")
## Check team model auth
if model_params.model_info is not None and model_params.model_info.team_id is not None:
if model_params.model_info.team_id is not None:
team_obj_row: Final = await _repo_team_table(prisma_client).find_unique(
where={"team_id": model_params.model_info.team_id}
)
@ -1810,6 +1987,27 @@ class ModelManagementAuthChecks:
)
team_obj: Final = LiteLLM_TeamTable.model_validate(team_obj_row.model_dump())
if (
member_operation is not None
and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj)
):
from litellm.proxy.proxy_server import llm_router
if llm_router is None or (member_operation == "update" and incoming_model_params is None):
raise HTTPException(
status_code=400, detail="An auto-router configuration and model catalog are required."
)
return await authorize_member_auto_router_write(
incoming=incoming_model_params if incoming_model_params is not None else model_params,
existing=model_params if member_operation == "update" else None,
user_api_key_dict=user_api_key_dict,
team=team_obj,
premium_user=premium_user,
prisma_client=prisma_client,
llm_router=llm_router,
)
return ModelManagementAuthChecks.can_user_make_team_model_call(
team_id=model_params.model_info.team_id,
user_api_key_dict=user_api_key_dict,
@ -2067,12 +2265,14 @@ async def add_new_model(
)
## Auth check
await ModelManagementAuthChecks.can_user_make_model_call(
write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
model_params=model_params,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
premium_user=premium_user,
member_operation="create",
)
member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=model_params.litellm_params,
@ -2094,9 +2294,14 @@ async def add_new_model(
enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)),
)
model_params.model_info = ModelInfo( # rebind-ok: downstream team-model handling mutates this same object
clean_model_info: Final = ModelInfo(
**without_server_derived_pricing(model_params.model_info.model_dump(exclude_none=True))
)
model_params.model_info = ( # rebind-ok: downstream team-model handling mutates this same object
clean_model_info.model_copy(update=MappingProxyType({"member_auto_router": True}))
if member_write is not None
else clean_model_info
)
model_response: prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None = None
# update DB
@ -2129,6 +2334,7 @@ async def add_new_model(
None,
),
model_id=priced_model_params.model_info.id,
member_write=member_write,
),
)
reload_outcome = await proxy_config.add_deployment(
@ -2259,12 +2465,15 @@ async def update_model(
raise Exception("model not found")
deployment: Final = Deployment(**_existing_litellm_params.model_dump())
await ModelManagementAuthChecks.can_user_make_model_call(
write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
model_params=deployment,
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
premium_user=premium_user,
member_operation="update",
incoming_model_params=model_params,
)
member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
ModelManagementAuthChecks.can_user_attach_credential(
litellm_params=model_params.litellm_params,
@ -2285,6 +2494,9 @@ async def update_model(
effective_params: Final = _effective_complexity_router_params(
model_params.litellm_params, deployment.litellm_params
)
member_marker: Final = _member_auto_router_marker_for_update(
incoming_params=model_params.litellm_params, existing=deployment, member_write=member_write
)
# update DB
if store_model_in_db is True:
@ -2317,15 +2529,30 @@ async def update_model(
and deployment.model_info.team_id is None
else None
)
_data: Final[dict[str, str]] = {
base_update: Final[PrismaCompatibleUpdateDBModel] = {
"litellm_params": json.dumps(merged_dictionary),
"updated_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
**({} if renamed_to is None else {"model_name": renamed_to}),
}
renamed_update: Final[PrismaCompatibleUpdateDBModel] = (
{**base_update, "model_name": renamed_to} # mutable-ok: Prisma serializes only concrete update dicts
if renamed_to is not None
else base_update
)
_data: Final[PrismaCompatibleUpdateDBModel] = (
{ # mutable-ok: Prisma serializes only concrete update dicts
**renamed_update,
"model_info": deployment.model_info.model_copy(
update=MappingProxyType({"member_auto_router": member_marker})
).model_dump_json(exclude_none=True),
}
if member_marker is not None
else renamed_update
)
async with _auto_router_capability_slot(
prisma_client,
effective_params=effective_params,
model_id=_model_id,
member_write=member_write,
) as table:
model_response: Final = await table.update(
where={"model_id": _model_id},
@ -2421,7 +2648,6 @@ async def update_public_model_groups(
"""
try:
# Update the public model groups
import litellm
from litellm.proxy.proxy_server import proxy_config, store_model_in_db
# Check if user has admin permissions
@ -2496,7 +2722,6 @@ async def update_useful_links(
"""
try:
# Update the public model groups
import litellm
from litellm.proxy.proxy_server import proxy_config
# Check if user has admin permissions

1
litellm/proxy/management_endpoints/organization_endpoints.py
View file

@ -362,6 +362,7 @@ async def new_organization(
- max_budget: *Optional[float]* - Max budget for org
- tpm_limit: *Optional[int]* - Max tpm limit for org
- rpm_limit: *Optional[int]* - Max rpm limit for org
- tpd_limit: *Optional[int]* - Max tokens per day stored on the org budget. Batch submissions enforce tpd_limit at the key, team and end user scopes only.
- model_rpm_limit: *Optional[Dict[str, int]]* - The RPM (Requests Per Minute) limit per model for this organization.
- model_tpm_limit: *Optional[Dict[str, int]]* - The TPM (Tokens Per Minute) limit per model for this organization.
- max_parallel_requests: *Optional[int]* - [Not Implemented Yet] Max parallel requests for org

45
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -1217,6 +1217,7 @@ async def new_team(
- mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team.
- tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
- rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
- tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
- rpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of RPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating RPM, or "best_effort_throughput" for best effort enforcement.
- tpm_limit_type: Optional[Literal["guaranteed_throughput", "best_effort_throughput"]] - The type of TPM limit enforcement. Use "guaranteed_throughput" to raise an error if overallocating TPM, or "best_effort_throughput" for best effort enforcement.
- max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
@ -1856,9 +1857,9 @@ def validate_team_org_change(
# Check if the team's budget is less than the org's max_budget
if (
team.max_budget
and organization.litellm_budget_table
and organization.litellm_budget_table.max_budget
team.max_budget is not None
and organization.litellm_budget_table is not None
and organization.litellm_budget_table.max_budget is not None
and team.max_budget > organization.litellm_budget_table.max_budget
):
raise HTTPException(
@ -1969,6 +1970,7 @@ async def update_team(
- metadata: Optional[dict] - Metadata for team, store information for team. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
- tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
- rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
- tpd_limit: Optional[int] - The TPD (Tokens Per Day) limit for this team. Batch submissions are charged against it instead of tpm_limit/rpm_limit
- max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
- soft_budget: Optional[float] - The soft budget threshold for the team. If max_budget is set (either in the request or existing), soft_budget must be strictly lower than max_budget. Can be set independently if max_budget is not set.
- budget_duration: Optional[str] - The duration of the budget for the team. Doc [here](https://docs.litellm.ai/docs/proxy/team_budgets)
@ -3325,7 +3327,8 @@ async def team_member_delete(
}'
```
"""
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
@ -3463,6 +3466,25 @@ async def team_member_delete(
}
)
await delete_cache_team_object(
team_id=data.team_id,
team_alias=existing_team_row.team_alias,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
await delete_cache_key_objects(
hashed_tokens=tuple(key.token for key in keys_to_delete),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
await evict_and_broadcast(cache_keys=tuple(sorted(user_ids_to_delete)), user_api_key_cache=user_api_key_cache)
for user_id in sorted(user_ids_to_delete):
await invalidate_team_member_spend_state(
user_id=user_id,
team_id=data.team_id,
user_api_key_cache=user_api_key_cache,
)
_emit_team_members_metric(existing_team_row)
return existing_team_row
@ -5684,6 +5706,21 @@ async def team_model_add(
detail={"error": "Only proxy admin or team admin can modify team models"},
)
return await append_team_models(
data=data,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
async def append_team_models(
*,
data: TeamModelAddRequest,
prisma_client: PrismaClient,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
) -> "prisma_models.LiteLLM_TeamTable":
# Atomic array append with dedup at the database level so concurrent
# BYOK model creates don't overwrite each other's team.models entries.
# When the team currently has models=[] (unrestricted access), the

4
litellm/proxy/management_helpers/access_group_key_sync.py
View file

@ -38,7 +38,7 @@ from litellm.proxy._types import (
from litellm.proxy.auth.auth_checks import (
_delete_cache_access_object, # pyright: ignore[reportPrivateUsage] # the access-group endpoints reach for this same cache primitive
)
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
from litellm.repositories.table_repositories import AccessGroupRepository
@ -75,7 +75,7 @@ _REPOINT_KEY_SQL: Final = (
def _raw_executor(prisma_client: object) -> _RawExecutor:
"""Narrow the untyped Prisma client down to the raw-query call this module makes, pinned to the writer."""
db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client
return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin
return writer_wrapper(db) # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin
async def _invalidate_access_group_cache(access_group_id: str) -> None:

4
litellm/proxy/management_helpers/access_group_model_sync.py
View file

@ -10,7 +10,7 @@ from typing import Final, Protocol
from pydantic import BaseModel
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
from litellm.proxy.management_helpers.access_group_team_sync import invalidate_access_group_caches
from litellm.repositories.table_repositories import AccessGroupRepository
from litellm.router import Router
@ -56,7 +56,7 @@ _REMOVE_MODEL_NAME_SQL: Final = (
def _raw_executor(prisma_client: object) -> _RawExecutor:
db: Final = AccessGroupRepository(prisma_client).prisma_client.db # pyright: ignore[reportAny] # untyped Prisma client
return WriterPinnedClient(db).db # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin
return writer_wrapper(db) # pyright: ignore[reportAny, reportReturnType] # untyped Prisma client behind the pin
def _config_sourced_sibling(llm_router: Router, deployment_id: str, model_id: str) -> bool:

345
litellm/proxy/management_helpers/auto_router_permissions.py Normal file
View file

@ -0,0 +1,345 @@
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
from fastapi import HTTPException
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm.models.organization import LiteLLM_OrganizationTable
from litellm.models.project import LiteLLM_ProjectTable
from litellm.proxy._types import (
UI_TEAM_ID,
CommonProxyErrors,
KeyManagementRoutes,
LiteLLM_TeamMembership,
LiteLLM_TeamTable,
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.proxy.auth.auth_checks import (
_check_team_member_model_access, # pyright: ignore[reportPrivateUsage] # shared membership authorization owner
can_key_call_model,
can_org_access_model,
can_project_access_model,
can_team_access_model,
)
from litellm.proxy.auth.team_grants import team_model_aliases
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import DatabaseClient
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import TeamMembershipRepository
from litellm.router import Router
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies
from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig
from litellm.types.router import Deployment, updateDeployment
if TYPE_CHECKING:
from prisma import types as prisma_types
class _MemberRouterThinking(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
type: Literal["enabled", "disabled", "adaptive"]
budget_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
class _MemberRouterGenerationParams(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
reasoning_effort: str | None = None
thinking: _MemberRouterThinking | None = None
verbosity: Literal["low", "medium", "high"] | None = None
max_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
max_completion_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
max_output_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
temperature: float | None = Field(default=None, ge=0, le=2, allow_inf_nan=False)
top_p: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False)
frequency_penalty: float | None = Field(default=None, ge=-2, le=2, allow_inf_nan=False)
presence_penalty: float | None = Field(default=None, ge=-2, le=2, allow_inf_nan=False)
seed: int | None = None
stop: str | tuple[str, ...] | None = None
class _MemberComplexityRouterConfig(RequestComplexityRouterConfig):
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
class _RouterConfigSource(BaseModel):
model: str | None = None
complexity_router_config: Mapping[str, object] | None = None
class _MembershipKey(TypedDict):
user_id: ReadOnly[str]
team_id: ReadOnly[str]
class _MembershipWhere(TypedDict):
user_id_team_id: ReadOnly[_MembershipKey]
@dataclass(frozen=True, slots=True)
class MemberAutoRouterDependencyObjects:
membership: LiteLLM_TeamMembership | None
organization: LiteLLM_OrganizationTable | None
project: LiteLLM_ProjectTable | None
def authorize_member_auto_router_team(
*, user_api_key_dict: UserAPIKeyAuth, team: LiteLLM_TeamTable, premium_user: bool
) -> None:
if not premium_user:
raise HTTPException(status_code=403, detail=CommonProxyErrors.not_premium_user.value)
if (
user_api_key_dict.user_role
not in (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.ORG_ADMIN)
or not user_api_key_dict.user_id
or not any(member.user_id == user_api_key_dict.user_id for member in team.members_with_roles)
or user_api_key_dict.team_id not in (None, UI_TEAM_ID, team.team_id)
or team.blocked
or KeyManagementRoutes.AUTO_ROUTER_MANAGE.value not in (team.team_member_permissions or ())
):
raise HTTPException(status_code=403, detail="This team does not allow you to manage your own auto routers.")
def validate_member_auto_router_config(config: Mapping[str, object]) -> RequestComplexityRouterConfig:
try:
validated: Final = _MemberComplexityRouterConfig.model_validate(config)
for entries in validated.tier_model_configs.values():
for entry in entries:
_MemberRouterGenerationParams.model_validate(entry.litellm_params)
return validated
except ValidationError as exc:
location: Final = ".".join(str(part) for part in exc.errors()[0]["loc"])
raise HTTPException(status_code=400, detail=f"Invalid member auto-router configuration at {location}.") from exc
async def authorize_member_auto_router_dependencies(
*,
config: RequestComplexityRouterConfig,
default_model: str | None,
user_api_key_dict: UserAPIKeyAuth,
team: LiteLLM_TeamTable,
prisma_client: DatabaseClient | None,
llm_router: Router,
dependency_objects: MemberAutoRouterDependencyObjects | None = None,
) -> None:
from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
if team.blocked:
raise HTTPException(status_code=403, detail="This auto router's team is blocked.")
aliases: Final = team_model_aliases(team)
alias_dict: Final = (
dict(aliases) if aliases is not None else None # mutable-ok: auth model and helpers require dict
)
scoped_actor: Final = user_api_key_dict.model_copy(
update=MappingProxyType({"team_id": team.team_id, "team_models": team.models, "team_model_aliases": alias_dict})
)
objects: Final = (
dependency_objects
if dependency_objects is not None
else await _load_member_auto_router_dependency_objects(
user_api_key_dict=scoped_actor, team=team, prisma_client=prisma_client
)
)
if team.organization_id and objects.organization is None:
raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.")
if scoped_actor.project_id and (
objects.project is None or objects.project.team_id != team.team_id or objects.project.blocked
):
raise HTTPException(status_code=403, detail="The auto router's project is unavailable.")
dependencies: Final = strategy_router_dependencies(
MappingProxyType(
{
"model": "auto_router/complexity_router",
"complexity_router_config": config.model_dump(exclude_none=True),
"complexity_router_default_model": default_model,
}
)
)
for model, deployments in (
(dependency.model_name, llm_router.get_model_list(model_name=dependency.model_name, team_id=team.team_id))
for dependency in dependencies
):
if not deployments or any(
classify_strategy_router_model(_RouterConfigSource.model_validate(deployment["litellm_params"]).model or "")
is not None
for deployment in deployments
):
raise HTTPException(status_code=400, detail=f"Auto-router target {model!r} must be a configured model.")
await can_team_access_model(
model=model,
team_object=team,
llm_router=llm_router,
team_model_aliases=alias_dict,
prisma_client=prisma_client,
)
await can_key_call_model(
model=model,
llm_model_list=None,
valid_token=scoped_actor,
llm_router=llm_router,
prisma_client=prisma_client,
)
await _check_team_member_model_access(
model=model,
team_object=team,
valid_token=scoped_actor,
llm_router=llm_router,
prisma_client=None,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
team_membership=objects.membership,
team_membership_loaded=True,
)
if objects.organization is not None:
can_org_access_model(model=model, org_object=objects.organization, llm_router=llm_router)
if objects.project is not None:
can_project_access_model(model=model, project_object=objects.project, llm_router=llm_router)
async def _load_member_auto_router_dependency_objects(
*, user_api_key_dict: UserAPIKeyAuth, team: LiteLLM_TeamTable, prisma_client: DatabaseClient | None
) -> MemberAutoRouterDependencyObjects:
if prisma_client is None:
raise HTTPException(status_code=503, detail="Cannot verify auto-router model access without a database")
membership_where: Final[_MembershipWhere] = {
"user_id_team_id": {"user_id": user_api_key_dict.user_id or "", "team_id": team.team_id}
}
membership_include: Final[prisma_types.LiteLLM_TeamMembershipInclude] = {"litellm_budget_table": True}
membership_row: Final = (
await TeamMembershipRepository(prisma_client).table.find_unique(
where=membership_where, include=membership_include
)
if user_api_key_dict.user_id
else None
)
membership: Final = (
LiteLLM_TeamMembership.model_validate(membership_row.model_dump()) if membership_row is not None else None
)
organization: Final = (
await OrganizationRepository(prisma_client).find_by_id(team.organization_id) if team.organization_id else None
)
if team.organization_id and organization is None:
raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.")
project: Final = (
await ProjectRepository(prisma_client).find_by_id(user_api_key_dict.project_id)
if user_api_key_dict.project_id
else None
)
return MemberAutoRouterDependencyObjects(membership=membership, organization=organization, project=project)
class StoredAutoRouterIdentity(BaseModel):
created_by: str | None = None
updated_at: datetime | None = None
@dataclass(frozen=True, slots=True)
class MemberAutoRouterWrite:
actor: UserAPIKeyAuth
team_id: str
model_id: str | None
public_name: str
updated_at: datetime | None
config: RequestComplexityRouterConfig
default_model: str | None
async def authorize_member_auto_router_write(
*,
incoming: Deployment | updateDeployment,
existing: Deployment | None,
user_api_key_dict: UserAPIKeyAuth,
team: LiteLLM_TeamTable,
premium_user: bool,
prisma_client: DatabaseClient,
llm_router: Router,
) -> MemberAutoRouterWrite:
authorize_member_auto_router_team(user_api_key_dict=user_api_key_dict, team=team, premium_user=premium_user)
stored: Final = StoredAutoRouterIdentity.model_validate(existing.model_dump()) if existing is not None else None
if stored is not None and stored.created_by != user_api_key_dict.user_id:
raise HTTPException(status_code=403, detail="Team members can update only their own auto routers.")
params: Final = incoming.litellm_params
if params is None or incoming.model_fields_set - frozenset({"model_name", "litellm_params", "model_info"}):
raise HTTPException(status_code=403, detail="Team members may change only auto-router configuration.")
if params.model_fields_set - frozenset({"model", "complexity_router_config", "complexity_router_default_model"}):
raise HTTPException(status_code=403, detail="Team members may change only auto-router configuration.")
info: Final = incoming.model_info
if info is not None and (
info.model_fields_set - frozenset({"id", "team_id"})
or info.team_id not in (None, team.team_id)
or (existing is not None and "id" in info.model_fields_set and info.id != existing.model_info.id)
):
raise HTTPException(
status_code=403, detail="Team members cannot change model ownership or administrative settings."
)
existing_model: Final = (
decrypt_value_helper(existing.litellm_params.model, key="model", return_original_value=True)
if existing is not None
else None
)
effective_model: Final = params.model or existing_model
if (
not isinstance(effective_model, str)
or classify_strategy_router_model(effective_model) != "complexity"
or (existing is not None and effective_model != existing_model)
):
raise HTTPException(status_code=403, detail="Team members may manage only complexity auto routers.")
public_name: Final = (
existing.model_info.team_public_model_name or existing.model_name
if existing is not None
else incoming.model_name
)
if (
not public_name
or public_name != public_name.strip()
or any(character in public_name for character in "*?[]")
or public_name.startswith("model_name_")
):
raise HTTPException(
status_code=400, detail="Choose a non-empty auto-router name without wildcards or internal prefixes."
)
if existing is not None and incoming.model_name not in (None, public_name, existing.model_name):
raise HTTPException(status_code=403, detail="Team members cannot rename an auto router.")
supplied_config: Final = _RouterConfigSource.model_validate(params.model_dump()).complexity_router_config
raw_config: Final = (
supplied_config
if supplied_config is not None
else _RouterConfigSource.model_validate(existing.litellm_params.model_dump()).complexity_router_config
if existing is not None
else None
)
if raw_config is None:
raise HTTPException(status_code=400, detail="A complexity_router_config is required.")
config: Final = validate_member_auto_router_config(raw_config)
stored_default: Final = existing.litellm_params.complexity_router_default_model if existing is not None else None
default_model: Final = (
params.complexity_router_default_model
if params.complexity_router_default_model is not None
else decrypt_value_helper(stored_default, key="complexity_router_default_model", return_original_value=True)
if stored_default is not None
else None
)
await authorize_member_auto_router_dependencies(
config=config,
default_model=default_model,
user_api_key_dict=user_api_key_dict,
team=team,
prisma_client=prisma_client,
llm_router=llm_router,
)
return MemberAutoRouterWrite(
actor=user_api_key_dict,
team_id=team.team_id,
model_id=existing.model_info.id if existing is not None else None,
public_name=public_name,
updated_at=stored.updated_at if stored is not None else None,
config=config,
default_model=default_model,
)

20
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -77,6 +77,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
from litellm.secret_managers.main import get_secret_str, str_to_bool
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
)
from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
@ -1322,7 +1323,7 @@ def _resolve_vertex_model_from_router(
endpoint: str,
vertex_project: str | None,
vertex_location: str | None,
) -> tuple[str, str, str | None, str | None]:
) -> tuple[str, str, str | None, str | None, Mapping[str, object] | None]:
"""
Resolve Vertex AI model configuration from router.
@ -1335,18 +1336,21 @@ def _resolve_vertex_model_from_router(
vertex_location: Current vertex location (may be from URL)
Returns:
tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location)
with resolved values from router config
tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location, deployment_model_info)
with resolved values from router config; deployment_model_info is the resolved
deployment's `model_info`, or None when no deployment matched
"""
if not llm_router:
return encoded_endpoint, endpoint, vertex_project, vertex_location
return encoded_endpoint, endpoint, vertex_project, vertex_location, None
try:
deployment: Final = llm_router.get_available_deployment_for_pass_through(model=model_id)
if not deployment:
return encoded_endpoint, endpoint, vertex_project, vertex_location
return encoded_endpoint, endpoint, vertex_project, vertex_location, None
litellm_params: Final = deployment.get("litellm_params", {})
model_info: Final = deployment.get("model_info")
deployment_model_info: Final = model_info if isinstance(model_info, Mapping) else None
# Always override with router config values (they take precedence over URL values)
config_vertex_project: Final = litellm_params.get("vertex_project")
@ -1387,10 +1391,11 @@ def _resolve_vertex_model_from_router(
encoded_endpoint = encoded_endpoint.replace(model_id, actual_model)
endpoint = endpoint.replace(model_id, actual_model)
return encoded_endpoint, endpoint, vertex_project, vertex_location, deployment_model_info
except Exception as e:
verbose_proxy_logger.debug("Error resolving vertex model from router for model %s: %s", model_id, e)
return encoded_endpoint, endpoint, vertex_project, vertex_location
return encoded_endpoint, endpoint, vertex_project, vertex_location, None
def _is_bedrock_agent_runtime_route(endpoint: str) -> bool:
@ -2134,6 +2139,7 @@ async def _base_vertex_proxy_route(
endpoint,
vertex_project,
vertex_location,
deployment_model_info,
) = _resolve_vertex_model_from_router(
model_id=model_id,
llm_router=llm_router,
@ -2142,6 +2148,8 @@ async def _base_vertex_proxy_route(
vertex_project=vertex_project,
vertex_location=vertex_location,
)
if deployment_model_info:
setattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, deployment_model_info)
vertex_credentials: Final = passthrough_endpoint_router.get_vertex_credentials(
project_id=vertex_project,

9
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -96,6 +96,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
EndpointType,
@ -613,6 +614,12 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
_metadata.update(
LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
)
_request_state: Final = getattr(request, "state", None)
deployment_model_info: Final = getattr(
_request_state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, None
)
if isinstance(deployment_model_info, Mapping):
_metadata["model_info"] = dict(deployment_model_info)
kwargs: Final = {
"litellm_params": {
@ -2002,6 +2009,8 @@ def create_pass_through_route(
delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY)
if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY):
delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY)
if hasattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY):
delattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY)
# The upstream withholds its response headers until its first token, so
# the whole time-to-first-token is spent inside _relay with nothing on

13
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -8,6 +8,7 @@ import httpx
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.proxy._types import PassThroughEndpointLoggingResultValues
@ -60,7 +61,7 @@ class PassThroughStreamingHandler:
litellm_logging_obj._update_completion_start_time(completion_start_time=datetime.now())
@staticmethod
def schedule_stream_failure_logging(
async def schedule_stream_failure_logging(
litellm_logging_obj: LiteLLMLoggingObj,
endpoint_type: EndpointType,
request_body: dict[str, object],
@ -68,7 +69,7 @@ class PassThroughStreamingHandler:
exception: Exception,
stream_context: PassThroughStreamContext | None = None,
) -> None:
PassThroughStreamingHandler._record_partial_usage_for_failure(
await asyncify(PassThroughStreamingHandler._record_partial_usage_for_failure)(
litellm_logging_obj=litellm_logging_obj,
endpoint_type=endpoint_type,
request_body=request_body,
@ -222,7 +223,7 @@ class PassThroughStreamingHandler:
verbose_proxy_logger.error("Error in chunk_processor: %s", e)
if response.status_code < 400:
logging_scheduled = True
PassThroughStreamingHandler.schedule_stream_failure_logging(
await PassThroughStreamingHandler.schedule_stream_failure_logging(
litellm_logging_obj=litellm_logging_obj,
endpoint_type=endpoint_type,
request_body=resolved_request_body,
@ -292,7 +293,7 @@ class PassThroughStreamingHandler:
(
standard_logging_response_object,
kwargs,
) = PassThroughStreamingHandler._build_passthrough_logging_result(
) = await asyncify(PassThroughStreamingHandler._build_passthrough_logging_result)(
litellm_logging_obj=litellm_logging_obj,
passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route,
@ -334,8 +335,8 @@ class PassThroughStreamingHandler:
Synchronous, CPU-bound reconstruction of the standard logging payload
from collected raw SSE bytes. Extracted from
_route_streaming_logging_to_handler so the per-endpoint dispatch can
be unit-tested in isolation. Still invoked synchronously on the event
loop; an off-loop dispatch is a future change, not part of this PR.
be unit-tested in isolation. The async callers run it in a worker
thread so the token counts inside stay off the event loop.
"""
all_chunks: Final = PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(raw_bytes)
standard_logging_response_object: PassThroughEndpointLoggingResultValues | None = None

30
litellm/proxy/proxy_server.py
View file

@ -3061,7 +3061,7 @@ async def _reconcile_budget_reservation_for_counter_update(
budget_reservation: dict | None,
response_cost: float | None,
) -> set[str]:
if budget_reservation is None:
if budget_reservation is None or budget_reservation.get("finalized") is True:
return set()
from litellm.proxy.spend_tracking.budget_reservation import (
@ -7080,8 +7080,19 @@ class ProxyConfig:
## PASS-THROUGH ENDPOINTS ##
if "pass_through_endpoints" in _general_settings:
general_settings["pass_through_endpoints"] = _general_settings["pass_through_endpoints"]
await initialize_pass_through_endpoints(pass_through_endpoints=general_settings["pass_through_endpoints"])
db_pass_through_endpoints: Final = _general_settings["pass_through_endpoints"]
db_pass_through_paths: Final = frozenset(
endpoint.get("path") for endpoint in db_pass_through_endpoints if isinstance(endpoint, dict)
)
general_settings["pass_through_endpoints"] = [
*db_pass_through_endpoints,
*(
endpoint
for endpoint in config_passthrough_endpoints or ()
if endpoint.get("path") not in db_pass_through_paths
),
]
await initialize_pass_through_endpoints(pass_through_endpoints=db_pass_through_endpoints)
## UI ACCESS MODE ##
if "ui_access_mode" in _general_settings:
@ -9516,6 +9527,9 @@ class ProxyStartupEvent:
user_api_key_cache=user_api_key_cache,
litellm_jwtauth=litellm_jwtauth,
)
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
jwt_handler.bind_agent_lookup(global_agent_registry)
@classmethod
def _add_proxy_budget_to_db(cls):
@ -17462,6 +17476,16 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
"tab": "prompt_caching",
"description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.",
},
"openai_system_messages_first": {
"type": "Boolean",
"tab": "prompt_caching",
"description": (
"Moves system and developer messages to the front of the messages array on OpenAI and "
"Azure OpenAI chat completions requests, keeping their relative order. OpenAI's prompt cache "
"matches on the exact prefix, so a system message that arrives mid-conversation otherwise "
"breaks the cached prefix on every turn."
),
},
"budget_rollover": { # mutable-ok: registry literal, frozen with its siblings below
"type": "Boolean",
"description": (

13
litellm/proxy/schema.prisma
View file

@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_reset_at DateTime?
@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
max_parallel_requests Int?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
budget_duration String?
budget_reset_at DateTime?
blocked Boolean @default(false)
@ -438,6 +441,7 @@ model LiteLLM_VerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?
@ -483,6 +487,10 @@ model LiteLLM_VerificationToken {
model LiteLLM_JWTKeyMapping {
id String @id @default(uuid())
jwt_issuer String @default("") // Scopes the mapping to one configured issuer; "" matches any issuer.
// Not nullable: Postgres unique constraints treat every NULL as
// distinct, so a nullable column would let multiple unscoped
// mappings collide on the same claim without a constraint violation.
jwt_claim_name String // e.g. "sub", "email"
jwt_claim_value String // The claim value to match
token String // Hashed virtual key (FK)
@ -495,8 +503,8 @@ model LiteLLM_JWTKeyMapping {
litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
@@unique([jwt_claim_name, jwt_claim_value])
@@index([jwt_claim_name, jwt_claim_value, is_active])
@@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
@@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
}
// Deprecated keys during grace period - allows old key to work until revoke_at
@ -534,6 +542,7 @@ model LiteLLM_DeletedVerificationToken {
blocked Boolean?
tpm_limit BigInt?
rpm_limit BigInt?
tpd_limit BigInt?
max_budget Float?
budget_duration String?
budget_reset_at DateTime?

5
litellm/proxy/spend_tracking/budget_reservation.py
View file

@ -959,8 +959,9 @@ async def _set_reserved_entries_actual_cost(
async def _reseed_reserved_entry(item: _EntryAdjustment, actual_cost: float) -> None:
"""Post-call reconcile / release of a counter that was flushed, expired or reseeded between reservation and
reconcile: the optimistic delta no longer applies, so reseed from the DB floor (which cannot include this
request's cost yet) and add the settled cost, since increment_spend_counters skips reserved keys."""
reconcile: the optimistic delta no longer applies, so reseed from the DB floor and add the settled cost, since
increment_spend_counters skips reserved keys. The reconcile runs before this request's spend is enqueued to the
DB, so the reseeded floor excludes it."""
from litellm.proxy.proxy_server import _increment_spend_counter_cache, reseed_spend_counter_from_db
reseeded: Final = await reseed_spend_counter_from_db(counter_key=item.counter_key)

1
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -157,6 +157,7 @@ def _get_spend_logs_metadata(
user_api_key_team_alias=None,
spend_logs_metadata=None,
requester_ip_address=None,
user_agent=None,
additional_usage_values=None,
applied_guardrails=None,
status="success",

58
litellm/proxy/utils.py
View file

@ -85,7 +85,11 @@ from litellm._logging import _redact_string, verbose_proxy_logger
from litellm._service_logger import ServiceLogging, ServiceTypes
from litellm.caching.caching import DualCache, RedisCache
from litellm.caching.dual_cache import LimitedSizeOrderedDict
from litellm.exceptions import RejectedRequestError, SensitiveDataRouteException
from litellm.exceptions import (
GuardrailRaisedException,
RejectedRequestError,
SensitiveDataRouteException,
)
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
ModifyResponseException,
@ -430,6 +434,14 @@ def _enrich_http_exception_with_guardrail_context(exc: BaseException, callback:
detail.setdefault("guardrail_mode", event_hook)
def _is_client_error_exception(exc: Exception) -> bool:
if isinstance(exc, HTTPException):
return exc.status_code < 500
if isinstance(exc, ProxyException):
return not (exc.code.isdigit() and int(exc.code) >= 500)
return False
def _exception_changes_request_flow(exc: BaseException) -> bool:
"""
True for guardrail exceptions the proxy turns into an alternate request flow
@ -893,6 +905,9 @@ def _call_type_for_route(route: str | None) -> str | None:
return call_types[0].value if len(operations) == 1 else None
_PROXY_ONLY_LLM_API_ERRORS: Final = (HTTPException, ProxyException, GuardrailRaisedException)
def _failure_fields_to_lift(request_data: Mapping[str, object]) -> Mapping[str, object]:
"""Failure-path callbacks run after ``litellm_logging_obj`` is popped from
request_data (it is not serialisable), so the caller merges these fields
@ -2886,9 +2901,7 @@ class ProxyLogging:
### ALERTING ###
await self.update_request_status(litellm_call_id=request_data.get("litellm_call_id", ""), status="fail")
if AlertType.llm_exceptions in self.alert_types and not isinstance(
original_exception, (HTTPException, ProxyException)
):
if AlertType.llm_exceptions in self.alert_types and not _is_client_error_exception(original_exception):
"""
Just alert on LLM API exceptions. Do not alert on user errors
@ -2985,6 +2998,7 @@ class ProxyLogging:
- Authentication Errors from user_api_key_auth
- HTTP HTTPException (rate limit errors)
- ProxyException (guardrail blocks, budget / rate-limit errors)
- GuardrailRaisedException (guardrail blocks / guardrail failures)
"""
#########################################################
@ -2999,9 +3013,7 @@ class ProxyLogging:
if not (RouteChecks.is_llm_api_route(route) or RouteChecks.is_info_route(route)):
return False
return isinstance(original_exception, (HTTPException, ProxyException)) or (
error_type == ProxyErrorTypes.auth_error
)
return isinstance(original_exception, _PROXY_ONLY_LLM_API_ERRORS) or (error_type == ProxyErrorTypes.auth_error)
async def _handle_logging_proxy_only_error(
self,
@ -3557,8 +3569,9 @@ class ProxyLogging:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
raise
except Exception:
ProxyLogging._fire_deferred_stream_logging(request_data)
except Exception as e:
if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
ProxyLogging._fire_deferred_stream_logging(request_data)
raise
ProxyLogging._fire_deferred_stream_logging(request_data)
return
@ -3632,8 +3645,9 @@ class ProxyLogging:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
raise
except Exception:
ProxyLogging._fire_deferred_stream_logging(request_data)
except Exception as e:
if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, e):
ProxyLogging._fire_deferred_stream_logging(request_data)
raise
# Fire deferred logging AFTER all guardrail end-of-stream blocks
@ -3729,6 +3743,23 @@ class ProxyLogging:
logging_obj._deferred_stream_complete_args = None
asyncio.create_task(_deferred_cb(*_args))
@staticmethod
def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool:
"""Drop the parked success dispatch for an assembled chat stream that ends in an error
``post_call_failure_hook`` logs as a failure, billing its usage on the failure row instead.
Returns False when the parked dispatch should still be flushed by the caller."""
logging_obj: Final = request_data.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return False
_args: Final[tuple[object, ...] | None] = getattr(logging_obj, "_deferred_stream_complete_args", None)
assembled: Final = _args[0] if _args else None
if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(assembled, ModelResponse):
return False
logging_obj._on_deferred_stream_complete = None
logging_obj._deferred_stream_complete_args = None
logging_obj.record_assembled_response_for_failure(assembled)
return True
async def _arelease_max_parallel_requests_on_disconnect(
self,
user_api_key_dict: UserAPIKeyAuth,
@ -4289,7 +4320,8 @@ class PrismaClient:
t.spend AS team_spend,
t.max_budget AS team_max_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit
FROM "LiteLLM_VerificationToken" v
LEFT JOIN "LiteLLM_TeamTable" t ON v.team_id = t.team_id;
""",
@ -4728,6 +4760,7 @@ class PrismaClient:
t.soft_budget AS team_soft_budget,
t.tpm_limit AS team_tpm_limit,
t.rpm_limit AS team_rpm_limit,
t.tpd_limit AS team_tpd_limit,
t.models AS team_models,
t.metadata AS team_metadata,
t.blocked AS team_blocked,
@ -4745,6 +4778,7 @@ class PrismaClient:
b.max_budget AS litellm_budget_table_max_budget,
b.tpm_limit AS litellm_budget_table_tpm_limit,
b.rpm_limit AS litellm_budget_table_rpm_limit,
b.tpd_limit AS litellm_budget_table_tpd_limit,
b.model_max_budget as litellm_budget_table_model_max_budget,
b.soft_budget as litellm_budget_table_soft_budget,
o.metadata as organization_metadata,

5
litellm/repositories/prisma_protocols.py
View file

@ -12,6 +12,11 @@ from typing import Protocol, TypeVar
RowT_co = TypeVar("RowT_co", covariant=True)
class DatabaseClient(Protocol):
@property
def db(self) -> object: ...
class TableActions(Protocol[RowT_co]):
"""The prisma-client-py per-model action surface, keyed to the row it returns.

20
litellm/repositories/unit_of_work.py
View file

@ -24,12 +24,8 @@ from typing import Final
from litellm.repositories.prisma_protocols import BatchTable, PrismaBatch
def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float | None) -> Mapping[str, object]:
spend: Final[object] = (
{"decrement": spend_decrement} # mutable-ok: prisma update payload must be a dict
if spend_decrement is not None
else 0
)
def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float) -> Mapping[str, object]:
spend: Final[object] = {"decrement": spend_decrement} # mutable-ok: prisma update payload must be a dict
return {"spend": spend, "budget_reset_at": budget_reset_at} # mutable-ok: prisma update payload must be a dict
@ -37,9 +33,7 @@ def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float |
class KeySpendResetWrites:
table: BatchTable
def queue_spend_reset(
self, token: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
) -> None:
def queue_spend_reset(self, token: str, budget_reset_at: datetime | None, spend_decrement: float) -> None:
self.table.update(
where={"token": token}, # mutable-ok: prisma where filter must be a dict
data=_spend_reset_data(budget_reset_at, spend_decrement),
@ -50,9 +44,7 @@ class KeySpendResetWrites:
class UserSpendResetWrites:
table: BatchTable
def queue_spend_reset(
self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
) -> None:
def queue_spend_reset(self, user_id: str, budget_reset_at: datetime | None, spend_decrement: float) -> None:
self.table.update(
where={"user_id": user_id}, # mutable-ok: prisma where filter must be a dict
data=_spend_reset_data(budget_reset_at, spend_decrement),
@ -63,9 +55,7 @@ class UserSpendResetWrites:
class TeamSpendResetWrites:
table: BatchTable
def queue_spend_reset(
self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float | None = None
) -> None:
def queue_spend_reset(self, team_id: str, budget_reset_at: datetime | None, spend_decrement: float) -> None:
self.table.update(
where={"team_id": team_id}, # mutable-ok: prisma where filter must be a dict
data=_spend_reset_data(budget_reset_at, spend_decrement),

28
litellm/responses/main.py
View file

@ -1,9 +1,10 @@
import asyncio
import contextvars
from collections.abc import Coroutine, Generator, Iterable, Mapping
from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
import httpx
@ -15,7 +16,7 @@ from litellm._logging import verbose_logger
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
from litellm.constants import request_timeout
from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, request_timeout
from litellm.integrations.anthropic_cache_control_hook import CARRY_UNMATCHED_MESSAGE_POINTS
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
@ -52,6 +53,7 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.secret_managers.main import get_secret_str
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import all_litellm_params
from litellm.utils import (
ProviderConfigManager,
client,
@ -408,6 +410,25 @@ def _bridges_to_chat_completions(
return responses_api_provider_config is None or use_chat_completions_api is True
def _bridge_kwargs(
kwargs: Mapping[str, object],
responses_api_provider_config: BaseResponsesAPIConfig | None,
allowed_openai_params: Sequence[str] | None,
) -> Mapping[str, object]:
if responses_api_provider_config is None:
return kwargs
forwarded_keys: Final = frozenset(
(
*litellm.OPENAI_CHAT_COMPLETION_PARAMS,
*DEFAULT_CHAT_COMPLETION_PARAM_VALUES,
*all_litellm_params,
*GenericLiteLLMParams.model_fields,
*(allowed_openai_params or ()),
)
)
return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys})
_ResponsesCompatibilityFailure: TypeAlias = Literal["encrypted_task_unsupported"]
@ -1281,6 +1302,7 @@ def responses(
return _file_search_dispatch
if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api):
bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params)
return litellm_completion_transformation_handler.response_api_handler(
model=model,
input=input,
@ -1292,7 +1314,7 @@ def responses(
extra_body=extra_body,
timeout=timeout if timeout is not None else request_timeout,
allowed_openai_params=allowed_openai_params,
**kwargs,
**bridge_kwargs,
)
# Get optional parameters for the responses API

5
litellm/responses/streaming_iterator.py
View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.openai import (
PART_UNION_TYPES,
ResponseAPIUsage,
@ -626,7 +627,9 @@ class BaseResponsesAPIStreamingIterator:
return
request_kwargs = getattr(caching_handler, "request_kwargs", None)
if not _is_json_object(request_kwargs) or request_kwargs.get("stream") is not True:
if not _is_json_object(request_kwargs):
return
if request_kwargs.get("stream") is not True and not converted_stream_requested(request_kwargs):
return
request_kwargs = request_kwargs.copy()
preset_cache_key = getattr(caching_handler, "preset_cache_key", None)

160
litellm/router.py
View file

@ -79,7 +79,10 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
is_registered_custom_provider,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
@ -99,6 +102,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
mask_sensitive_structure,
)
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
from litellm.llms.base_llm.vector_store.transformation import (
RouterVectorStoreEmbeddingExecutor,
@ -167,6 +171,7 @@ from litellm.router_utils.cooldown_handlers import (
_get_cooldown_deployments,
_set_cooldown_deployments,
is_advisor_orchestration_failure,
is_caller_timeout_408,
)
from litellm.router_utils.fallback_event_handlers import (
AttemptedFallbackTargets,
@ -1622,6 +1627,24 @@ class Router:
return
await selector.async_pre_call_check(deployment, parent_otel_span)
def _bind_override_selector_to_request(
self, strategy: str, selector: RouterStrategySelector | None, request_kwargs: Mapping[str, object] | None
) -> None:
if selector is None or request_kwargs is None or strategy in self._globally_registered_strategies():
return
logging_obj: Final = request_kwargs.get("litellm_logging_obj")
if isinstance(logging_obj, LiteLLMLogging):
logging_obj.add_dynamic_callback(selector)
def _globally_registered_strategies(self) -> frozenset[str]:
configured: Final = (
self.routing_strategy,
*(group.routing_strategy for group in self._routing_groups.values()),
)
return frozenset(
normalized for normalized in map(self._normalize_strategy, configured) if normalized is not None
)
def _get_routing_context(
self, model: str, request_kwargs: dict | None = None
) -> tuple[str | None, RouterStrategySelector | None]:
@ -1647,7 +1670,9 @@ class Router:
override: Final = self._get_request_routing_strategy_override(request_kwargs)
if override is not None:
verbose_router_logger.debug("routing_group=request-override model=%s strategy=%s", model, override)
return override, self._get_override_strategy_selector(override)
override_selector: Final = self._get_override_strategy_selector(override)
self._bind_override_selector_to_request(override, override_selector, request_kwargs)
return override, override_selector
group_name: Final = model if self.get_routing_group(model) is not None else self._model_to_group.get(model)
if group_name is None:
@ -2461,7 +2486,7 @@ class Router:
### DEPLOYMENT-SPECIFIC PRE-CALL CHECKS ### (e.g. update rpm pre-call. Raise error, if deployment over limit)
## only run if model group given, not model id
if not self.has_model_id(model):
if model in self.model_names or not self.has_model_id(model):
self.routing_strategy_pre_call_checks(deployment=deployment)
input_kwargs: Final = {
@ -3753,7 +3778,16 @@ class Router:
self, deployment: dict, kwargs: dict, function_name: str | None = None
) -> Deployment:
"""
Handle clientside credential
Build a per-request Deployment carrying the caller-supplied api_key/api_base,
with its own stable id for cooldown, logging, and cost-map identity.
This deployment is deliberately never registered with the router (no
upsert_deployment/add_deployment call): doing so used to add it to
self.model_list under the shared model_name, which made a request-scoped,
caller-supplied provider credential a permanent, load-balanced deployment
that every other caller of that model group could be routed onto. Its
pricing is still registered directly, so a custom price configured on the
underlying deployment still applies to this call.
"""
model_info: Final = deployment.get("model_info", {}).copy()
litellm_params: Final = deployment["litellm_params"].copy()
@ -3772,7 +3806,7 @@ class Router:
litellm_params=LiteLLM_Params(**dynamic_litellm_params),
model_info=model_info,
)
self.upsert_deployment(deployment=deployment_pydantic_obj) # add new deployment to router
Router._register_deployment_pricing(deployment=deployment_pydantic_obj)
return deployment_pydantic_obj
@staticmethod
@ -8277,6 +8311,13 @@ class Router:
litellm_params: Final = kwargs.get("litellm_params", {})
_model_info: Final = litellm_params.get("model_info", {})
if is_caller_timeout_408(kwargs, exception_status):
verbose_router_logger.debug(
"Router: Exiting 'deployment_callback_on_failure' without cooldown. "
"A timeout the caller set caused this 408, not the deployment's health."
)
return False
exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers(
original_exception=exception
)
@ -9526,8 +9567,10 @@ class Router:
)
# done reading model["litellm_params"]
# Check if provider is supported: either in enum or JSON-configured
if custom_llm_provider not in litellm.provider_list and not JSONProviderRegistry.exists(
custom_llm_provider
if (
custom_llm_provider not in litellm.provider_list
and not JSONProviderRegistry.exists(custom_llm_provider)
and not is_registered_custom_provider(custom_llm_provider)
):
raise Exception(f"Unsupported provider - {custom_llm_provider}")
@ -9673,40 +9716,7 @@ class Router:
# initialize client
self._add_deployment(deployment=deployment)
_model_info_dict: Final[dict] = deployment.model_info.model_dump(exclude_none=True)
for field in CustomPricingLiteLLMParams.model_fields:
field_value = deployment.litellm_params.get(field)
if field_value is not None:
_model_info_dict[field] = field_value
Router._inherit_builtin_base_rates_for_off_peak(
model_info=_model_info_dict,
backend_model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)
if _model_info_dict.get("input_cost_per_token") is not None:
Router._inherit_builtin_cache_pricing(
model_info=_model_info_dict,
backend_model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)
Router._inherit_builtin_tiered_output_rate(
model_info=_model_info_dict,
backend_model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)
# Register custom pricing in litellm.model_cost.
# Mirrors _create_deployment() logic to ensure dynamically-added deployments
# (e.g., loaded from DB) also have their custom pricing registered.
# Without this, _is_model_cost_zero() cannot detect explicitly-configured
# zero-cost models, causing budget checks to block free models.
Router._register_deployment_in_model_cost(
model_id=deployment.model_info.id,
model_info=_model_info_dict,
model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)
Router._register_deployment_pricing(deployment=deployment)
# add to model names
self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id)
@ -9968,6 +9978,21 @@ class Router:
)
return model_info
@staticmethod
def _register_deployment_pricing(deployment: Deployment) -> None:
"""Register a deployment's custom/inherited pricing in ``litellm.model_cost``.
Takes only a ``Deployment``, so it registers pricing for a deployment that
is never added to ``self.model_list`` (a per-request client-side-credential
deployment) just as readily as one that is.
"""
Router._register_deployment_in_model_cost(
model_id=deployment.model_info.id,
model_info=Router._deployment_model_cost_payload(deployment),
model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)
@staticmethod
def _register_deployment_in_model_cost(
*,
@ -10794,6 +10819,7 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
**model_info,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
}
)
@ -10872,6 +10898,9 @@ class Router:
if model_info.get("rpm", None) is not None and _deployment_rpm is None:
_deployment_rpm = model_info.get("rpm")
model_group_info.supports_fast_mode = model_group_info.supports_fast_mode and (
AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider)
)
deployment_reasoning_efforts = (
resolve_supported_reasoning_efforts( # rebind-ok: recalculated per deployment
model_info, deployment_is_mapped=deployment_is_mapped
@ -12512,7 +12541,7 @@ class Router:
# check if aliases set on litellm model alias map
if specific_deployment is True:
return model, self._get_deployment_by_litellm_model(model=model)
elif self.has_model_id(model):
elif model not in self.model_names and self.has_model_id(model):
deployment: Final = self.get_deployment(model_id=model)
if deployment is not None:
deployment_model: Final = deployment.litellm_params.model
@ -13475,7 +13504,7 @@ class Router:
async def async_pre_routing_hook(
self,
model: str,
request_kwargs: dict,
request_kwargs: dict[str, object],
messages: list[dict[str, Any]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
@ -13523,6 +13552,18 @@ class Router:
)
return None
from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
await authorize_member_auto_router_inference(
deployment=self._selected_strategy_marker_deployment(
model=registered_model_name,
strategy_tags=selected_strategy.tags,
request_kwargs=request_kwargs,
),
request_kwargs=request_kwargs,
llm_router=self,
)
from litellm.proxy.guardrails.auto_router_compression import (
messages_for_routing,
model_hop_compression_armed,
@ -13622,25 +13663,34 @@ class Router:
return pre_routing_hook_response
def _selected_strategy_marker_deployment(
self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object]
) -> DeploymentTypedDict | None:
markers: Final = tuple(
deployment
for deployment in self.deployments_for_request(model, request_kwargs)
if "model" in deployment["litellm_params"]
and str(deployment["litellm_params"]["model"]).startswith(AUTO_ROUTER_MODEL_PREFIX)
)
tag_matched: Final = tuple(
deployment
for deployment in markers
if (tuple(deployment["litellm_params"]["tags"] or ()) if "tags" in deployment["litellm_params"] else ())
== strategy_tags
)
return tag_matched[0] if tag_matched else (markers[0] if markers else None)
def _forwardable_alias_marker_params(
self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object]
) -> tuple[tuple[str, object], ...]:
marker_params: Final = tuple(
litellm_params
for deployment in self.deployments_for_request(model, request_kwargs)
if str((litellm_params := deployment["litellm_params"]).get("model", "")).startswith(
AUTO_ROUTER_MODEL_PREFIX
)
marker: Final = self._selected_strategy_marker_deployment(
model=model, strategy_tags=strategy_tags, request_kwargs=request_kwargs
)
tag_matched: Final = tuple(
params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags
)
selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None)
if selected is None:
if marker is None:
return ()
return tuple(
(key, value)
for key, value in selected.items()
for key, value in marker["litellm_params"].items()
if key not in _ALIAS_PARAMS_NEVER_FORWARDED
and key not in CustomPricingLiteLLMParams.model_fields
and value is not None

119
litellm/router_strategy/complexity_router/README.md
View file

@ -68,6 +68,117 @@ still resolve to a deployment in `model_list`; this configuration does not creat
- abc
```
### Capability forecasting
Set `classifier_type: capability` to use
[NVIDIA NeMo Switchyard's packaged capability classifier](https://github.com/NVIDIA-NeMo/Switchyard/blob/main/crates/libsy/src/prompts/capability-classifier/prompt.md).
The classifier forecasts the probability that an efficient model completes
the whole task, identifies the capability-card boundary that applies, and leaves the
route choice to a deterministic threshold policy
```yaml
model_list:
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
classifier_type: capability
classifier_llm_config:
model: classifier-model
capability_classifier_config:
efficient_tier: SIMPLE
capable_tier: REASONING
base_threshold: 0.5
threshold_step: 0.1
tiers:
SIMPLE:
- efficient-model-a
- efficient-model-b
REASONING: capable-model
```
The structured classifier verdict contains `crux`, `primary_rule`,
`capability_boundary`, and `p_solve`. The policy computes the required solve
probability as follows
- `supported`: `base_threshold`
- `uncertain` or `unmatched`: `base_threshold + threshold_step`
- `unsupported`: `base_threshold + 2 * threshold_step`
The efficient tier is selected when `p_solve` is greater than or equal to the
adjusted threshold. Otherwise the capable tier is selected. A malformed,
inconsistent, empty, or unavailable verdict always fails closed to the capable
tier. `base_threshold` is required, `threshold_step` defaults to `0`, and their
maximum adjusted threshold must not exceed `1`
The classifier receives the packaged Switchyard system prompt, the opening user
task, and the latest user follow-up when present. Caller system messages,
assistant turns, and intermediate tool results are not sent. The classifier call
uses strict JSON Schema output and the existing classifier timeout, circuit
breaker, attribution, redaction, reasoning-effort, and optional vision settings
`efficient_tier` and `capable_tier` name built-in complexity tiers with configured
model pools. The forecast still makes one binary quality decision, while the
ordinary tier pool may contain multiple equivalent deployments. Session affinity,
keyword overrides, plan-mode floors, modality checks, and other post-classification
complexity-router controls continue to apply
Routing decisions record the adjusted threshold and the complete valid forecast:
`classifier_p_solve`, `classifier_capability_boundary`, `classifier_primary_rule`,
and `classifier_crux`. Prompt redaction removes `classifier_crux` while retaining
the derived fields needed to audit the decision
#### Calibrating solve probabilities
Supply a fitted monotone logit calibration under `capability_classifier_config`
to transform the forecast before applying the threshold. Calibration is opt-in;
without it the router uses the raw probability. Fit coefficients on benchmark
outcomes from separate training repositories, select thresholds on a validation
split, and report quality and cost on an untouched evaluation split
```yaml
capability_classifier_config:
efficient_tier: SIMPLE
capable_tier: REASONING
base_threshold: 0.66
threshold_step: 0
max_output_tokens: 512
response_format: json_object
calibration:
version: your-benchmark-artifact-v1
slope: 1.0
intercept: 0.0
```
The example coefficients are an identity mapping, not a trained calibration.
The mapping is `sigmoid(slope * logit(clip(p_solve, 1e-6, 1-1e-6)) + intercept)`.
The slope must be nonnegative, so calibration cannot improve ranking. It can
make probabilities more accurate and thresholds easier to interpret. The version
is recorded for auditing; the router does not check whether an artifact matches
the judge, capability card, efficient solver, or agent harness. Operators must
keep those aligned and refit when they change
Logs retain `classifier_p_solve` and add `classifier_calibrated_p_solve` and
`classifier_calibration_version`. `classifier_threshold` is compared to the
calibrated probability. Invalid verdicts still route to the capable tier
`response_format` defaults to `json_schema`. For endpoints that support JSON
objects but not strict schemas, `json_object` appends the same schema to the
unchanged capability prompt and retains strict local validation. Set
`classifier_llm_config.timeout_ms` to cover the measured judge latency; a local
judge may need longer than the default 3000 ms. `max_output_tokens` still defaults
to 4096; 512 is an explicit benchmark setting for a short, non-reasoning judge
For a controlled whole-task benchmark, use `adaptive: false`,
`session_affinity: true`, and a unique session ID for every task and policy arm.
Disable keyword, plan-mode, housekeeping, and other optional overrides when
measuring only the capability policy. When adaptive selection is enabled, it
cannot select below the capability decision, including a capable-tier fallback
Configure capability forecasting through YAML or the model-management API.
The dashboard preserves its classifier and calibration on an untouched save;
it does not provide a capability-card editor
### Heuristic v2
Set `classifier_type: heuristic_v2` to classify with the bundled calibrated
@ -529,3 +640,11 @@ Technical code keywords are detected case-insensitively and include:
| Best For | Cost optimization | Intent routing |
Use `complexity_router` when you want to optimize costs by routing simple queries to cheaper models. Use `auto_router` when you need semantic intent matching (e.g., routing "customer support" queries to a specialized model).
## Experimental LLM V2 classifier
LLM V2 combines task demands, available verification, and model capability in one judge call. It forecasts whole-task success for an efficient and a capable solver. The router compares their probabilities against an explicitly configured quality allowance and selects the capable solver when classification fails
This classifier is intended for evaluation. Its probabilities are raw forecasts unless matching per-model calibration is supplied, and an estimated quality allowance is not a measured quality guarantee. It requires two model groups, profiles for both solvers, and a description of their harness and budget. Adaptive selection is disabled for this mode so it cannot override the forecast. Existing user-turn classification can reuse a decision until the user changes the task
V2 reads all human task messages and follow-ups, without the complexity classifier's prior-turn truncation or assistant summaries. Long task histories can therefore increase judge cost or exceed its context window, which falls back to the capable solver. Profiles must describe every deployment behind their model group and calibration must match the prompt, solver settings, and harness being evaluated

4
litellm/router_strategy/complexity_router/__init__.py
View file

@ -16,6 +16,8 @@ from litellm.router_strategy.complexity_router.complexity_router import (
from litellm.router_strategy.complexity_router.config import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
DEFAULT_COMPLEXITY_CONFIG,
CapabilityCalibrationConfig,
CapabilityClassifierConfig,
ClassificationRubric,
ComplexityRouterConfig,
ComplexityTier,
@ -28,6 +30,8 @@ from litellm.router_strategy.complexity_router.config import (
__all__ = [
"DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE",
"DEFAULT_COMPLEXITY_CONFIG",
"CapabilityCalibrationConfig",
"CapabilityClassifierConfig",
"ClassificationRubric",
"ComplexityRouter",
"ComplexityRouterConfig",

216
litellm/router_strategy/complexity_router/capability_classifier.py Normal file
View file

@ -0,0 +1,216 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Capability forecast contract and routing policy adapted from NVIDIA NeMo Switchyard."""
import json
from collections.abc import Mapping
from sys import float_info
from types import MappingProxyType
from typing import Final, Literal, NamedTuple, TypeAlias
from pydantic import BaseModel, ConfigDict, Field, StrictFloat, TypeAdapter, model_validator
CapabilityBoundary: TypeAlias = Literal["supported", "uncertain", "unsupported", "unmatched"]
CapabilityRule: TypeAlias = Literal[
"SUP-1",
"SUP-2",
"SUP-3",
"SUP-4",
"SUP-5",
"UNC-1",
"UNC-2",
"LIM-1",
"LIM-2",
"none",
]
CAPABILITY_CLASSIFIER_SYSTEM_PROMPT: Final = """You are a task-level probability forecaster for a model router. You receive the
task's opening instruction and, when present, its latest user follow-up, plus
the qualitative capability card below.
Forecast one binary event:
SUCCESS means that the efficient agent completes the whole task correctly on
one fresh run under the actual harness, tools, and budget, as judged by the
final verifier. FAILURE means any other outcome. The two outcomes are
exhaustive.
Use only evidence in the instruction and the capability card. Do not assume
hidden repository state, unmentioned tools, validators, documentation, access,
or future work habits. Do not invent empirical counts, success rates, or base
rates. The capability card is qualitative evidence, not a measured prior.
# Assessment procedure
1. State the crux: the hardest material requirement for whole-task success.
2. Select the one capability rule that best describes the crux. Use
primary_rule=none and capability_boundary=unmatched when no rule applies.
Rule ids are opaque labels. Do not infer a boundary from an id's spelling.
3. Privately identify the strongest instruction-visible reasons for SUCCESS
and FAILURE, then imagine the most likely concrete failure.
4. Privately consider material unknowns. Missing information should limit
extreme estimates, but it is not evidence that p_solve must equal 0.50.
5. Estimate p_solve last. It is the probability of whole-task SUCCESS, not
confidence in this assessment, a route recommendation, or a cost judgment.
Interpret probabilities as natural frequencies. If p_solve is 0.70 for 100
comparable fresh runs, about 70 should succeed and 30 should fail. Use the full
range when justified. Reserve 0.00 and 1.00 for outcomes that are logically
impossible or certain under the visible contract. Supported does not mean 1.00,
and unsupported does not mean 0.00. The downstream routing threshold is not
part of this forecast.
# Efficient-agent capability card
The route verbs in this source card are inherited qualitative descriptions.
They do not ask you to output a route and do not assign a fixed probability to
any boundary.
- SUP-1 [supported]: Route to the Efficient model when the task provides a complete output contract and a deterministic local validator that covers the material requirements.
- SUP-2 [supported]: Route to the Efficient model when all required inputs are available, the target environment can be inspected, and correctness can be verified end-to-end without inaccessible external state.
- SUP-3 [supported]: Route to the Efficient model when mathematical behavior, interfaces, shapes, data types, tolerances, and performance requirements are explicit and exercised by a representative harness.
- SUP-4 [supported]: Route to the Efficient model when the required mechanism is identified, the relevant search space is bounded, and the success condition is executable. Do not infer this rule merely from the task's technical domain.
- SUP-5 [supported]: Route to the Efficient model when reconstruction or behavioral reproduction is constrained by an executable reference, parser, format specification, or checker strong enough to distinguish correct from merely plausible output.
- UNC-1 [uncertain]: Treat the route as uncertain when multiple reasonable interpretations of preprocessing, representation, indexing, naming, or output placement would produce different results and neither the instructions nor a validator resolve the choice.
- UNC-2 [uncertain]: Treat the route as uncertain when success requires finding every relevant item across heterogeneous inputs or environment state, but the task does not define the search boundary or provide a completeness check.
- LIM-1 [unsupported]: Prefer the Capable model when correctness depends primarily on extracting precise information from noisy visual, temporal, or rendered media and no machine-checkable extraction or replay mechanism is available.
- LIM-2 [unsupported]: Prefer the Capable model when success depends on reproducing undocumented reference behavior, hidden intermediate state, or an unknown configuration, and small deviations fail despite satisfying the visible specification.
# Output
Return exactly one JSON object matching the response schema supplied with the
request. Do not include markdown or commentary.
p_solve must be between 0.00 and 1.00. p_fail is exactly 1.00 - p_solve and
must not be emitted separately. Do not output recommended_route, confidence,
abstain, counts, task totals, empirical rates, or any other field."""
_BOUNDARY_STEPS: Final = MappingProxyType(
{
"supported": 0,
"uncertain": 1,
"unmatched": 1,
"unsupported": 2,
}
)
_RULE_BOUNDARIES: Final = MappingProxyType(
{
"SUP-1": "supported",
"SUP-2": "supported",
"SUP-3": "supported",
"SUP-4": "supported",
"SUP-5": "supported",
"UNC-1": "uncertain",
"UNC-2": "uncertain",
"LIM-1": "unsupported",
"LIM-2": "unsupported",
"none": "unmatched",
}
)
class CapabilityClassifierVerdict(BaseModel):
"""Strict structured verdict returned by the capability forecaster."""
model_config = ConfigDict(extra="forbid", frozen=True)
crux: str = Field(min_length=1)
primary_rule: CapabilityRule
capability_boundary: CapabilityBoundary
p_solve: StrictFloat = Field(ge=0.0, le=1.0)
@model_validator(mode="after")
def _validate_rule_boundary_pair(self) -> "CapabilityClassifierVerdict":
if not self.crux.strip():
raise ValueError("crux must contain non-whitespace text")
expected: Final = _RULE_BOUNDARIES[self.primary_rule]
if self.capability_boundary != expected:
raise ValueError(
f"primary_rule {self.primary_rule!r} requires capability_boundary {expected!r}, "
f"got {self.capability_boundary!r}"
)
return self
def routing_threshold(self, base_threshold: float, threshold_step: float) -> float:
"""Required efficient-model solve probability for this boundary."""
return base_threshold + _BOUNDARY_STEPS[self.capability_boundary] * threshold_step
def meets_routing_threshold(self, threshold: float) -> bool:
"""Inclusive comparison with Switchyard's one-epsilon rounding guard."""
return self.p_solve >= threshold or abs(threshold - self.p_solve) <= float_info.epsilon
class CapabilityClassifierForecast(NamedTuple):
verdict: CapabilityClassifierVerdict
threshold: float
p_solve: float
calibration_version: str | None
def meets_routing_threshold(self) -> bool:
return self.p_solve >= self.threshold or abs(self.threshold - self.p_solve) <= float_info.epsilon
_CAPABILITY_CLASSIFIER_RESPONSE_FORMAT_JSON: Final = """{
"type": "json_schema",
"json_schema": {
"name": "CapabilityClassifierDecision",
"strict": true,
"schema": {
"type": "object",
"additionalProperties": false,
"required": ["crux", "primary_rule", "capability_boundary", "p_solve"],
"properties": {
"crux": {"type": "string", "minLength": 1},
"primary_rule": {
"type": "string",
"enum": ["SUP-1", "SUP-2", "SUP-3", "SUP-4", "SUP-5", "UNC-1", "UNC-2", "LIM-1", "LIM-2", "none"]
},
"capability_boundary": {
"type": "string",
"enum": ["supported", "uncertain", "unsupported", "unmatched"]
},
"p_solve": {"type": "number", "minimum": 0.0, "maximum": 1.0}
}
}
}
}"""
_RESPONSE_FORMAT_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def capability_classifier_response_format(
mode: Literal["json_schema", "json_object"] = "json_schema",
) -> Mapping[str, object]:
"""Fresh copy of Switchyard's packaged strict JSON Schema wrapper."""
return (
_RESPONSE_FORMAT_ADAPTER.validate_json('{"type": "json_object"}')
if mode == "json_object"
else _RESPONSE_FORMAT_ADAPTER.validate_json(_CAPABILITY_CLASSIFIER_RESPONSE_FORMAT_JSON)
)
def capability_classifier_system_prompt(mode: Literal["json_schema", "json_object"]) -> str:
if mode == "json_schema":
return CAPABILITY_CLASSIFIER_SYSTEM_PROMPT
wrapper: Final = _RESPONSE_FORMAT_ADAPTER.validate_python(capability_classifier_response_format()["json_schema"])
return (
CAPABILITY_CLASSIFIER_SYSTEM_PROMPT
+ "\n\nReturn exactly one JSON object matching this JSON Schema:\n"
+ json.dumps(wrapper["schema"], indent=2, sort_keys=True)
)
def unwrap_classifier_json(content: str) -> str:
"""Remove the optional Markdown fence without repairing or weakening verdict JSON."""
text: Final = content.strip()
if not text.startswith("```"):
return text
unfenced: Final = text.removeprefix("```").removeprefix("json").lstrip("\n\r")
return unfenced.removesuffix("```").strip()
def parse_capability_classifier_verdict(content: str) -> CapabilityClassifierVerdict:
"""Parse raw JSON or the fenced JSON shape tolerated by Switchyard."""
return CapabilityClassifierVerdict.model_validate_json(unwrap_classifier_json(content))

Some files were not shown because too many files have changed in this diff Show more