Merge origin/main into litellm_openai_error_type_from_status

This commit is contained in:
mateo-berri 2026-10-29 20:03:20 -07:00
commit cfec5a12e4
225 changed files with 10020 additions and 1434 deletions

View file

@ -51,7 +51,7 @@ jobs:
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
env:
UV_PYTHON: "3.12"
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5

View file

@ -630,6 +630,12 @@ jobs:
- name: check_migrations_lock_timeout
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
- name: check_no_none_string_error_fields
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
- name: test_check_no_none_string_error_fields
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py

View file

@ -1003,9 +1003,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@ -1451,9 +1451,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi

View file

@ -153,7 +153,7 @@ lint-install:
# the same set CI's ':(glob)litellm/**/*.py' selects.
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@base_ref=$$($(RESOLVE_BASE)) && \
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \

View file

@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.80"
version = "0.1.81"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.80"
version = "0.1.81"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

View file

@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
"/test",
"/debug/memory/summary",
"/api/event_logging/batch",
"/eu_jev/v1/systemone",
}
)

File diff suppressed because one or more lines are too long

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.111"
version = "0.4.112"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.111"
version = "0.4.112"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

View file

@ -1783,6 +1783,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.eu_jev.decisions.transformation import (
EuJevDecisionsConfig as EuJevDecisionsConfig,
)
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)

View file

@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureDecisionsConfig",
"AzureAIDecisionsConfig",
@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),

View file

@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.secret_redaction import (
REDACTED,
python_redact_string,
python_redact_structured_value,
redact_internal_details,
redact_structured_value,
)
from litellm.litellm_core_utils.secret_redaction import (
redact_string as redact_secret_string,
@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
return redact_internal_details(value)
_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
_CLIENT_ITEMS: Final = TypeAdapter(list[object])
def _redacted_client_field(key: str | None, value: object) -> object:
if isinstance(value, str):
return redact_internal_details(redact_structured_value(key, value))
if isinstance(value, dict):
return {
child_key: _redacted_client_field(child_key, child)
for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
}
if isinstance(value, (list, tuple)):
return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
if not isinstance(value, (int, float)):
return value
text: Final = str(value)
return value if redact_structured_value(key, text) == text else REDACTED
def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
if not _ENABLE_SECRET_REDACTION:
return fields
return _redacted_client_field(None, dict(fields))
def _substituted_color_message(record: logging.LogRecord) -> str | None:
"""Render a record's ``color_message`` against its args, or None if absent.

View file

@ -8,10 +8,9 @@ The following caching mechanisms are supported:
2. **RedisSemanticCache**
3. **QdrantSemanticCache**
4. **InMemoryCache**
5. **DiskCache**
6. **S3Cache**
7. **AzureBlobCache**
8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
5. **S3Cache**
6. **AzureBlobCache**
7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
## Folder Structure
@ -20,7 +19,6 @@ litellm/caching/
├── base_cache.py
├── caching.py
├── caching_handler.py
├── disk_cache.py
├── dual_cache.py
├── in_memory_cache.py
├── qdrant_semantic_cache.py

View file

@ -1,6 +1,5 @@
from .azure_blob_cache import AzureBlobCache
from .caching import Cache, LiteLLMCacheType
from .disk_cache import DiskCache
from .dual_cache import DualCache
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache

View file

@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
from .azure_blob_cache import AzureBlobCache
from .base_cache import BaseCache
from .disk_cache import DiskCache
from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache
@ -153,7 +152,7 @@ class Cache:
Initializes the cache based on the given type.
Args:
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
# Redis Cache Args
host (str, optional): The host address for the Redis cache. Required if type is "redis".
@ -174,9 +173,6 @@ class Cache:
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
# Disk Cache Args
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
# S3 Cache Args
s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
@ -306,8 +302,11 @@ class Cache:
account_url=azure_account_url,
container=azure_blob_container,
)
elif type == LiteLLMCacheType.DISK:
self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
else:
supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
if disk_cache_dir is not None:
verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
if "cache" not in litellm.input_callback:
litellm.input_callback.append("cache")
if "cache" not in litellm.success_callback:
@ -1044,7 +1043,7 @@ def enable_cache(
Enable cache with the specified configuration.
Args:
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
host (Optional[str]): The host address of the cache server. Defaults to None.
port (Optional[str]): The port number of the cache server. Defaults to None.
password (Optional[str]): The password for the cache server. Defaults to None.
@ -1056,7 +1055,7 @@ def enable_cache(
None
Raises:
None
ValueError: If no cache is set yet and the cache type is unsupported.
"""
print_verbose("LiteLLM: Enabling Cache")
if "cache" not in litellm.input_callback:
@ -1091,7 +1090,7 @@ def update_cache(
Update the cache for LiteLLM.
Args:
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
host (Optional[str]): The host of the cache. Defaults to None.
port (Optional[str]): The port of the cache. Defaults to None.
password (Optional[str]): The password for the cache. Defaults to None.

View file

@ -9,7 +9,7 @@ This file is a wrapper around caching.py
This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
In each method it will call the appropriate method from caching.py
"""

View file

@ -1,92 +0,0 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
try:
import diskcache as dc
except ModuleNotFoundError as e:
raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
# if users don't provider one, use the default litellm cache
if disk_cache_dir is None:
self.disk_cache = dc.Cache(".litellm_cache")
else:
self.disk_cache = dc.Cache(disk_cache_dir)
def set_cache(self, key, value, **kwargs):
if "ttl" in kwargs:
self.disk_cache.set(key, value, expire=kwargs["ttl"])
else:
self.disk_cache.set(key, value)
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
async def async_set_cache_pipeline(self, cache_list, **kwargs):
for cache_key, cache_value in cache_list:
if "ttl" in kwargs:
self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
else:
self.set_cache(key=cache_key, value=cache_value)
def get_cache(self, key, **kwargs):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response
return None
def batch_get_cache(self, keys: list, **kwargs):
return_val: Final = []
for k in keys:
val = self.get_cache(key=k, **kwargs)
return_val.append(val)
return return_val
def increment_cache(self, key, value: int, **kwargs) -> int:
with self.disk_cache.transact():
cached_value: Final = self.get_cache(key=key)
init_value: Final = cached_value if isinstance(cached_value, int) else 0
new_value: Final = init_value + value
self.set_cache(key, new_value, **kwargs)
return new_value
async def async_get_cache(self, key, **kwargs):
return self.get_cache(key=key, **kwargs)
async def async_batch_get_cache(self, keys: list, **kwargs):
return_val: Final = []
for k in keys:
val = self.get_cache(key=k, **kwargs)
return_val.append(val)
return return_val
async def async_increment(self, key, value: int, **kwargs) -> int:
return self.increment_cache(key=key, value=value, **kwargs)
def flush_cache(self):
self.disk_cache.clear()
async def disconnect(self):
pass
def delete_cache(self, key):
self.disk_cache.pop(key)

View file

@ -1887,7 +1887,10 @@ def completion_cost(
video_resolution=video_resolution,
)
elif call_type in _SPEECH_CALL_TYPES:
prompt_characters = litellm.utils.count_characters(text=prompt)
prompt_characters = _speech_billed_characters(
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
prompt,
)
elif call_type in _TRANSCRIPTION_CALL_TYPES:
# Check _hidden_params first (duration stored there to
# avoid polluting the response body), then fall back to
@ -2366,6 +2369,11 @@ def response_cost_calculator(
raise e
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
def _video_input_image_cost(
model: str,
custom_llm_provider: str | None,

View file

@ -53,7 +53,7 @@ from litellm.main import (
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import ImageGenerationRequestQuality
from litellm.types.router import GenericLiteLLMParams
from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
from litellm.types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
LlmProviders,
@ -1010,6 +1010,7 @@ def image_edit(
optional_params=dict(image_edit_request_params),
litellm_params={
**image_edit_request_params,
**litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
"litellm_call_id": litellm_call_id,
"model_info": model_info,
"vertex_location": litellm_params.vertex_location,

View file

@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
"output_cost_per_image_token",
)
def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool:
return pricing is not None and (
"input_cost_per_pixel" in pricing
or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing)
)
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
# request in the cost-calc path, so the f-strings are built once here instead
# of being rebuilt for every model_info key on every call. Longest-first so a
@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+")
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
value: Final = _requested_image_param(optional_params, "size")
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
def get_web_search_requests(server_tool_use: object) -> int | None:
@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol):
def _model_priced_image_cost_calculator(
custom_llm_provider: str | None, model: str
custom_llm_provider: str | None, model: str, pricing: ModelInfo | None
) -> _ModelPricedImageCostCalculator | None:
match custom_llm_provider:
case litellm.LlmProviders.RECRAFT.value:
@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator(
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
case litellm.LlmProviders.DASHSCOPE.value:
if _deployment_prices_images_without_an_output_price(pricing):
return None
from litellm.llms.dashscope.image_generation.cost_calculator import (
cost_calculator as dashscope_image_cost_calculator,
)
return dashscope_image_cost_calculator
case _:
return None
@ -2178,7 +2194,7 @@ class CostCalculatorUtils:
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
pricing: Final = deployment_pricing(model_info)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
if model_priced_calculator is not None:
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)

View file

@ -0,0 +1,45 @@
"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set.
Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it.
"""
from collections.abc import Sequence
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing_extensions import assert_never
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType(
{effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}
)
def nearest_declared_reasoning_effort(
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
) -> str:
"""Rounding up takes the weakest declared level at least as strong as the request, else the
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
a model that cannot turn it off says so itself. A level outside the strength order is likewise
returned as is for upstream to judge."""
ranked: Final = sorted(
(effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK),
key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort],
)
if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked:
return requested
strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested]
match rounding:
case "up":
return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1])
case "down":
return next(
(effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0]
)
case _:
assert_never(rounding)

View file

@ -0,0 +1,27 @@
from collections.abc import Iterable
from typing import Final
from litellm.types.utils import ModelResponseStream, StreamingChoices
def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
if choice.finish_reason is not None:
return False
if getattr(choice, "logprobs", None) is not None:
return False
delta: Final = choice.delta if hasattr(choice, "delta") else None
if delta is None:
return True
return _all_values_none(delta.model_dump().values())
def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)
def is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
if chunk.provider_specific_fields is not None:
return False
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])

View file

@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
if expected_cls is None:
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content]
if missing:
raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}")
return expected_cls
@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[
def _format_object_parameters(parameters, indent):
properties: Final = parameters.get("properties")
if not properties:
if not isinstance(properties, Mapping) or not properties:
return ""
required_params: Final = parameters.get("required", [])
required: Final = parameters.get("required")
required_params: Final = tuple(required) if isinstance(required, list) else ()
lines: Final = []
for key, props in properties.items():
description = props.get("description")
description = props.get("description") if isinstance(props, Mapping) else None
if description:
lines.append(f"// {description}")
question = "?"
@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent):
def _format_type(props, indent):
if not isinstance(props, Mapping):
return "any"
type: Final = props.get("type")
enum: Final = props.get("enum")
if type == "string":
if "enum" in props:
return " | ".join([f'"{item}"' for item in props["enum"]])
if isinstance(enum, list):
return " | ".join([f'"{item}"' for item in enum])
return "string"
elif type == "array":
# items is required, OpenAI throws an error if it's missing
return f"{_format_type(props['items'], indent)}[]"
return f"{_format_type(props.get('items'), indent)}[]"
elif type == "object":
return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}"
elif type in ["integer", "number"]:
if "enum" in props:
return " | ".join([f'"{item}"' for item in props["enum"]])
if isinstance(enum, list):
return " | ".join([f'"{item}"' for item in enum])
return "number"
elif type == "boolean":
return "boolean"

View file

@ -6,6 +6,7 @@ import copy
import json
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object)
return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None
@dataclass(frozen=True, slots=True)
class ToolsWithoutStrict:
tools: tuple[Mapping[str, object], ...]
dropped_count: int
def tools_without_strict(tools: object) -> ToolsWithoutStrict | None:
try:
validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools)
except ValidationError:
return None
dropped_count: Final = sum(tool.get("strict") is True for tool in validated)
if dropped_count == 0:
return None
return ToolsWithoutStrict(
tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated),
dropped_count=dropped_count,
)
def warn_claude_strict_tools_dropped(dropped_count: int) -> None:
litellm.verbose_logger.warning(
"Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had "
"strict true) and sent the request again because drop_params is set.",
dropped_count,
)
def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
"""
Return a new message list with thinking / redacted_thinking content blocks removed

View file

@ -160,6 +160,7 @@ async def prepare_context_managed_request(
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.messages.context_management.editors.compact import (
apply_client_compaction_block_history,
slice_around_compaction_block,
)
# Skip the client-history pre-processing when a ``compact_20260112``
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
if polyfill_will_run:
history_result: PolyfillResult | None = None
working_messages: _AnthropicMessages = messages
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
working_system: _AnthropicSystem = system
else:
history_result = apply_client_compaction_block_history(

View file

@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import AppliedEdit
from ..convertible_blocks import drop_non_string_tool_result_texts
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
from .editors import (
CheckedEdit,
@ -103,7 +104,7 @@ async def apply_context_management(
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
``drop_invalid_edits``.
``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it.
The dispatcher is async so async editors (``compact_20260112``) can
``await`` the configured summarization model. Sync editors run in a
@ -119,7 +120,7 @@ async def apply_context_management(
raise first_error
well_formed_edits: Final = tuple(check.edit for check in checked)
current_messages = messages
current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits))
current_system = system
aggregated_applied: Final[list[AppliedEdit]] = []
aggregated_compaction_block = None

View file

@ -39,6 +39,21 @@ if TYPE_CHECKING:
T = TypeVar("T")
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
_TINY_RESULT_BYTES: Final = 46
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
{
"text",
"image_url",
"image",
"document",
"file",
"tool_use",
"tool_result",
"thinking",
"redacted_thinking",
"tool_reference",
"search_result",
}
)
ToolNames = Literal["all"] | frozenset[str]
@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel):
class _ClearToolUsesFields(BaseModel):
model_config = ConfigDict(frozen=True)
trigger: Mapping[str, object] | None = None
keep: Mapping[str, object] | None = None
clear_at_least: _ClearAtLeast | None = None
exclude_tools: tuple[str, ...] | None = None
clear_tool_inputs: bool | tuple[str, ...] | None = None
_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently
# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
_API_MESSAGES: Final = MappingProxyType(
{
"tuple_type": "Input should be a valid array",
"dict_type": "Input should be an object",
"model_type": "Input does not match the expected shape.",
}
)
@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]:
def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int:
"""Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it."""
before: Final = [result, use] if clears_input else [result]
after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)]
before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,)
after: Final[Sequence[Mapping[str, object]]] = (
(_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),)
)
counts: Final = tuple(
litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after)
_count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after)
)
return max(counts[0] - counts[1], 0)
@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
return tuple(candidates[:batch_end])
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
nested: Final = _parse(_BLOCKS, block.get("content"))
if nested is None:
return block
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
blocks: Final = _parse(_BLOCKS, message.get("content"))
if blocks is None:
return message
return {
**message,
"content": [
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
],
}
def _count_tokens(
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
) -> int:
return litellm.token_counter(
model=model,
messages=[_countable_message(message) for message in messages],
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
def _trigger_met(
trigger: dict[str, object],
model: str,
@ -218,11 +267,7 @@ def _trigger_met(
if not isinstance(threshold, int):
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
message_tokens: Final = litellm.token_counter(
model=model,
messages=messages,
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
message_tokens: Final = _count_tokens(model, messages, tools)
if message_tokens > threshold:
return True, message_tokens
current_tokens: Final = message_tokens + count_system_tokens(model, system)
@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919(
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
)
tokens_before: Final = (
counted_tokens
if counted_tokens is not None
else litellm.token_counter(
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
)
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools)
kept_after, _ = history_after_client_compaction(final_messages, system)
tokens_after: Final = litellm.token_counter(
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
tokens_after: Final = _count_tokens(model, kept_after, tools)
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
return final_messages, {

View file

@ -137,7 +137,7 @@ def _find_latest_compaction_index(
return None, None
def _slice_around_compaction_block(
def slice_around_compaction_block(
messages: Sequence[_MsgT],
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
"""Apply Anthropic's "drop everything before the compaction block" rule.
@ -194,7 +194,7 @@ def history_after_client_compaction(
system: str | Sequence[Mapping[str, object]] | None,
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
sliced, block = _slice_around_compaction_block(messages)
sliced, block = slice_around_compaction_block(messages)
if block is None:
return list(messages), system
tail: Final = _strip_compaction_blocks(sliced)
@ -615,7 +615,7 @@ def apply_client_compaction_block_history(
tail is forwarded unchanged (with compaction blocks stripped) so recent
turns the summary does not cover are preserved.
"""
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
if prior_compaction_block is None:
return None
@ -686,7 +686,7 @@ async def apply_compact_20260112(
# opt-in gate below so that even when summarization is disabled we still
# strip Anthropic-only ``compaction`` blocks from messages going to
# non-Anthropic backends (which would reject them).
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
augmented_system: str | list[dict[str, object]] | None = system
if isinstance(prior_summary_text, str) and prior_summary_text:

View file

@ -10,11 +10,15 @@ from typing import Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for_wire,
@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
AnthropicOutputConfig,
AnthropicOutputSchema,
ContextManagementResponse,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import request_safeguards_evaluator
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
from ..convertible_blocks import reject_malformed_tool_result_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_SYSTEM_ITEMS: Final = TypeAdapter(list[object])
_SPEC: Final = TypeAdapter(dict[str, object])
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _split_context_management(
spec: AnthropicContextManagementSpec,
kwargs: Mapping[str, object],
) -> _SplitEdits:
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
unsent, as it was before the polyfill ran here.
"""
if not isinstance(spec, dict):
return None, ()
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
return spec, ()
edits: Final = normalize_spec(spec) or []
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
return ({"edits": native} if native else None), tuple(
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
)
def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None:
if system is None or isinstance(system, str):
return system
try:
items: Final = _SYSTEM_ITEMS.validate_python(system)
except ValidationError:
return None
return tuple(block for block in map(as_str_mapping, items) if block is not None)
async def _polyfilled(
*,
model: str,
messages: AnthropicRequestMessages,
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
system: object,
edits: Sequence[Mapping[str, object]],
kwargs: Mapping[str, object],
) -> PolyfillResult | None:
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
if not edits:
return None
reject_malformed_tool_result_text(messages, model, kwargs)
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
blocks: Final = _system_blocks(system)
try:
return await apply_context_management(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks),
context_management_spec={"edits": [dict(edit) for edit in edits]},
drop_invalid_edits=drop_invalid_edits,
)
except AnthropicContextManagementError:
raise
except Exception as e:
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
raise AnthropicContextManagementError(
status_code=500, message=f"context_management polyfill failed: {e}"
) from e
async def count_view_messages(
*,
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]]:
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
if context_management is None or not isinstance(context_management.spec, dict):
return messages
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
result: Final = await _polyfilled(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=tools,
system=system,
edits=edits,
kwargs={"drop_params": context_management.drop_invalid_edits},
)
return messages if result is None else result.messages
def _with_applied_edits(
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
) -> AnthropicMessagesResponse:
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
if not applied_edits:
return response
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format: AnthropicOutputSchema | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
safeguards_evaluator=safeguards_evaluator,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
)
_record_bridged_response(responses_kwargs, result)
return served
@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
)
# Sync path
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
run_async_function(
_polyfilled,
model=model,
messages=messages,
tools=tools,
system=system,
edits=polyfilled_edits,
kwargs=kwargs,
)
if polyfilled_edits
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)

View file

@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import anyio
@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import (
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
)
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.base import LiteLLMBaseModel
@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper:
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
applied_edits: Sequence[AppliedEdit] | None = None,
) -> None:
self.responses_stream = responses_stream
self._applied_edits: Final = tuple(applied_edits or ())
self.model = model
self.litellm_logging_obj = litellm_logging_obj
self.safeguards_evaluator: Final = safeguards_evaluator
@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper:
"type": "message_delta",
"delta": message_delta_payload,
"usage": dict(anthropic_usage),
**(
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
if self._applied_edits
else {}
),
}
)
self._chunk_queue.append({"type": "message_stop"})

View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC):
anthropic_request_errors_on_thinking_binding_mismatch,
is_anthropic_invalid_thinking_block_error,
is_anthropic_thinking_binding_mismatch_error,
is_claude_strict_tools_rejection,
tools_without_strict,
validated_object_mapping,
)
if e.response.status_code != 400:
return False
error_text: Final[str] = e.response.text
body: Final = validated_object_mapping(request_body)
drops_rejected_strict_tools: Final = (
drops_unsupported_params(litellm_params.get("drop_params"))
and is_claude_strict_tools_rejection(e.response.status_code, error_text)
and body is not None
and tools_without_strict(body.get("tools")) is not None
)
if drops_rejected_strict_tools:
return True
caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch(
request_body
) and is_anthropic_thinking_binding_mismatch_error(error_text)
@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC):
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_block_error,
is_claude_strict_tools_rejection,
strip_thinking_blocks_from_anthropic_messages_request_dict,
tools_without_strict,
warn_claude_strict_tools_dropped,
)
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
without_strict: Final = (
tools_without_strict(request_data.get("tools"))
if is_claude_strict_tools_rejection(e.response.status_code, e.response.text)
else None
)
if without_strict is not None:
warn_claude_strict_tools_dropped(without_strict.dropped_count)
request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict
return request_data
def native_output_fallback_litellm_params(

View file

@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC):
Transform provider response to standard format
"""
async def async_transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> "HttpxBinaryResponseContent":
return self.transform_text_to_speech_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
)
def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException:
from ..chat.transformation import BaseLLMException

View file

@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine
from functools import partial
from typing import Final, TypeAlias
from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
is_bedrock_strict_tools_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
)
@ -34,7 +34,7 @@ def _native_output_retry(
def _strict_tools_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:

View file

@ -366,21 +366,10 @@ class NativeOutputFallback:
litellm_params: Mapping[str, object]
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
def warn_bedrock_strict_tools_dropped(model: str) -> None:
verbose_logger.warning(
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "

View file

@ -2467,8 +2467,7 @@ class BaseLLMHTTPHandler:
)
else:
verbose_logger.debug(
"Anthropic /v1/messages: invalid thinking signature; "
"stripping thinking blocks and retrying (attempt %s/%s).",
"Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).",
len(recovered_errors) + 2,
max_attempts,
)
@ -12850,7 +12849,7 @@ class BaseLLMHTTPHandler:
provider_config=text_to_speech_provider_config,
)
speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response(
speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response(
model=model,
raw_response=response,
logging_obj=logging_obj,

View file

@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic"
DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1"
DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1"
DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks"
DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH
DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com"
_STANDARD_DASHSCOPE_HOSTS: Final = frozenset(
@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool:
return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/")
def dashscope_api_root(api_base: str) -> str:
endpoints: Final = resolve_dashscope_endpoints(api_base)
if endpoints is not None:
return endpoints.origin
trimmed: Final = api_base.rstrip("/")
if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH):
return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH)
root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH)
return root if separator else trimmed
def is_aliyun_host(host: str | None) -> bool:
return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com"))
def resolve_dashscope_openai_api_base(
api_base: str | None,
env_var: str = "DASHSCOPE_API_BASE",
@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None:
if api_base is None:
return None
parsed: Final = urlparse(api_base)
host: Final = parsed.hostname or ""
on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com")
if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
return None
return f"{parsed.scheme}://{parsed.netloc}"

View file

@ -0,0 +1,160 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from typing_extensions import override
import litellm
from litellm.images.utils import image_to_data_url
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.dashscope.common_utils import (
DASHSCOPE_CHAT_COMPATIBLE_PATH,
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
dashscope_api_root,
is_dashscope_native_endpoint_url,
resolve_dashscope_endpoints,
)
from litellm.llms.dashscope.image_generation.transformation import (
DEFAULT_API_BASE,
OPENAI_TO_DASHSCOPE_SIZE,
DashScopeImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
MAX_INPUT_IMAGES: Final = 3
DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed")
_IMAGES: Final = TypeAdapter(tuple[object, ...])
def _dashscope_size(size: object) -> object:
if not isinstance(size, str):
return size
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
def _deployment_edit_url(api_base: str) -> str:
derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and (
resolve_dashscope_endpoints(api_base) is not None
or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH)
)
return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base
class DashScopeImageEditConfig(BaseImageEditConfig):
def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None:
super().__init__()
self._generation_config: Final = generation_config or DashScopeImageGenerationConfig()
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
return list(DASHSCOPE_IMAGE_EDIT_PARAMS)
@override
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: mirrors override contract
return {
key: _dashscope_size(value) if key == "size" else value
for key, value in image_edit_optional_params.items()
if key in DASHSCOPE_IMAGE_EDIT_PARAMS
}
@override
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors override contract
model: str,
api_key: str | None = None,
litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
return self._generation_config.validate_environment(
headers=dict(headers),
model=model,
messages=[],
optional_params={},
litellm_params=dict(litellm_params or {}),
api_key=api_key,
api_base=api_base,
)
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
if api_base is not None:
return _deployment_edit_url(api_base)
env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE")
derived: Final = (
f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None
)
return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE
@override
def use_multipart_form_data(self) -> bool:
return False
@override
def transform_image_edit_request(
self,
model: str,
prompt: str | None,
image: object,
image_edit_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
headers: Mapping[str, str],
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
images: Final = tuple(candidate for candidate in candidates if candidate is not None)
if len(images) > MAX_INPUT_IMAGES:
raise litellm.BadRequestError(
message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}",
model=model,
llm_provider="dashscope",
)
prompt_content: Final = ({"text": prompt},) if prompt is not None else ()
return {
"model": model,
"input": {
"messages": [
{
"role": "user",
"content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content],
}
]
},
"parameters": dict(image_edit_optional_request_params),
}, ()
@override
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> ImageResponse:
return self._generation_config.transform_image_generation_response(
model=model,
raw_response=raw_response,
model_response=ImageResponse(),
logging_obj=logging_obj,
request_data={},
optional_params={},
litellm_params={},
encoding=None,
)

View file

@ -0,0 +1,19 @@
from typing import Final
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
from litellm.types.utils import ImageResponse, ModelInfo
def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
prices: Final = resolve_image_model_info(
model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info
)
output_cost_per_image: Final = prices.get("output_cost_per_image")
if output_cost_per_image is None:
raise ValueError(f"No output_cost_per_image found for dashscope/{model}")
input_images: Final = image_response.hidden_params.get("billable_input_image_count")
input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * (
input_images if isinstance(input_images, int) else 0
)
return output_cost_per_image * len(image_response.data or ()) + input_image_cost

View file

@ -23,7 +23,7 @@ Response format:
}
"""
from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object)
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = {
"256x256": "256*256",
"512x512": "512*512",
"1024x1024": "1024*1024",
@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
}
def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]:
contents: Final = _JSON_OBJECTS.validate_python(message.get("content", []))
return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url)
def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]:
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
for choice in choices:
yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {})))
def _dashscope_size(size: object) -> object:
if not isinstance(size, str):
return size
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
"""
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
non_default_params: dict[str, object],
optional_params: dict[str, object],
model: str,
drop_params: bool,
) -> dict:
) -> dict[str, object]:
supported_params: Final = self.get_supported_openai_params(model)
mapped: Final[dict] = {}
for k, v in non_default_params.items():
if k in optional_params:
continue
if k not in supported_params:
continue
if k == "size":
# Convert "WxH" → "W*H"
mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
else:
mapped[k] = v
return mapped
requested: Final = {
k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params
}
return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()}
def _resolve_api_key(self, api_key: str | None) -> str:
resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY")
@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
stream: bool | None = None,
) -> str:
image_env_api_base: Final = self._image_api_base_env()
@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
def validate_environment(
self,
headers: dict,
headers: dict[str, str],
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
) -> dict[str, str]:
headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}"
headers["Content-Type"] = "application/json"
return headers
@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
request_data: dict[str, object],
optional_params: dict[str, object],
litellm_params: dict[str, object],
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
for choice in choices:
message = _JSON_OBJECT.validate_python(choice.get("message", {}))
content_list = _JSON_OBJECTS.validate_python(message.get("content", []))
for content_item in content_list:
image_url = content_item.get("image")
if image_url:
model_response.data.append(ImageObject(url=image_url))
return model_response
images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))]
usage: Final = response_object.get("usage")
usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {}
width: Final = usage_fields.get("output_width", usage_fields.get("width"))
height: Final = usage_fields.get("output_height", usage_fields.get("height"))
image_response: Final = model_response.model_copy(
update={
"data": images,
**({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}),
}
)
input_image_count: Final = usage_fields.get("input_image_count")
if isinstance(input_image_count, int) and input_image_count > 0:
image_response.hidden_params = {
**model_response.hidden_params,
"billable_input_image_count": input_image_count,
}
return image_response

View file

@ -0,0 +1,297 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from itertools import chain
from typing import TYPE_CHECKING, Final
from urllib.parse import urlparse
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import override
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
)
from litellm.llms.dashscope.common_utils import (
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
DashScopeError,
dashscope_api_root,
is_aliyun_host,
missing_dashscope_family_key_message,
resolve_dashscope_family_api_key,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.utils import Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com"
_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-"
_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None])
@dataclass(frozen=True, slots=True)
class _TextToSpeechFamily:
path: str
response_formats: frozenset[str]
sends_format: bool
takes_instructions: bool
MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily(
path=DASHSCOPE_MULTIMODAL_GENERATION_PATH,
response_formats=frozenset({"wav"}),
sends_format=False,
takes_instructions=False,
)
SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily(
path="/api/v1/services/audio/tts/SpeechSynthesizer",
response_formats=frozenset({"mp3", "wav", "opus", "pcm"}),
sends_format=True,
takes_instructions=True,
)
class _Audio(BaseModel):
model_config = ConfigDict(frozen=True)
url: str
class _Output(BaseModel):
model_config = ConfigDict(frozen=True)
audio: _Audio
class _Usage(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: int | None = None
output_tokens: int | None = None
characters: int | None = None
class _SpeechResponse(BaseModel):
model_config = ConfigDict(frozen=True)
output: _Output | None = None
usage: _Usage | None = None
code: str | None = None
message: str | None = None
def text_to_speech_family(model: str) -> _TextToSpeechFamily:
if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX):
return SPEECH_SYNTHESIZER_FAMILY
return MULTIMODAL_GENERATION_FAMILY
def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None:
match key:
case "response_format" if value in family.response_formats:
return {"format": value} if family.sends_format else {}
case "instructions" if family.takes_instructions:
return {"instruction": value}
case _:
return None
def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it",
model=model,
llm_provider="dashscope",
)
def _speech_response(raw_response: httpx.Response) -> _SpeechResponse:
try:
return _SpeechResponse.model_validate_json(raw_response.content)
except ValidationError as e:
raise DashScopeError(
status_code=502,
message=f"Unexpected DashScope text-to-speech response: {e}",
headers=raw_response.headers,
) from e
def _https_on_aliyun(url: str) -> str:
parsed: Final = urlparse(url)
return (
parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url
)
def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]:
parsed: Final = _speech_response(raw_response)
if parsed.output is None:
raise DashScopeError(
status_code=max(raw_response.status_code, 400),
message=parsed.message or parsed.code or raw_response.text,
headers=raw_response.headers,
)
return _https_on_aliyun(parsed.output.audio.url), parsed.usage
def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None:
extension: Final = raw_response.request.extensions.get("timeout")
if extension is None:
return None
phases: Final = _TIMEOUT_PHASES.validate_python(extension)
return httpx.Timeout(
connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool")
)
def _token_usage(usage: _Usage | None) -> Usage | None:
if usage is None or usage.input_tokens is None or usage.output_tokens is None:
return None
return Usage(
prompt_tokens=usage.input_tokens,
completion_tokens=usage.output_tokens,
total_tokens=usage.input_tokens + usage.output_tokens,
)
def _refused_audio_url(error: SSRFError) -> DashScopeError:
return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}")
def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent:
if audio_response.status_code != 200:
raise DashScopeError(
status_code=502,
message=f"DashScope audio download failed with HTTP {audio_response.status_code}",
headers=audio_response.headers,
)
binary: Final = HttpxBinaryResponseContent(audio_response)
binary.usage = _token_usage(usage)
if usage is not None and usage.characters is not None:
binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters}
return binary
def _shared_async_client() -> AsyncHTTPHandler:
return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE)
class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig):
def __init__(
self,
sync_client: Callable[[], HTTPHandler] = get_httpx_client,
async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client,
) -> None:
super().__init__()
self._sync_client: Final = sync_client
self._async_client: Final = async_client
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
family: Final = text_to_speech_family(model)
return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())]
@override
def map_openai_params(
self,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract
drop_params: bool = False,
kwargs: Mapping[str, object] | None = None,
) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract
family: Final = text_to_speech_family(model)
fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items())
unsupported: Final = (
*(f"{key}={value!r}" for key, value, field in fields if field is None),
*(("voice as an object",) if isinstance(voice, dict) else ()),
)
if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))):
raise _unsupported(model, unsupported)
supported: Final = tuple(field for _, _, field in fields if field is not None)
return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported))
@override
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors override contract
model: str,
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key)
if resolved_api_key is None:
raise ValueError(missing_dashscope_family_key_message("dashscope"))
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE")
root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE
return f"{root}{text_to_speech_family(model).path}"
@override
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: str | None,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
headers: dict[str, str], # mutable-ok: mirrors override contract
) -> TextToSpeechRequestData:
voice_field: Final = {"voice": voice} if voice is not None else {}
return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}}
@override
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> HttpxBinaryResponseContent:
audio_url, usage = _audio_url(raw_response)
try:
audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response))
except SSRFError as e:
raise _refused_audio_url(e) from e
return _binary_audio(audio_response, usage)
@override
async def async_transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> HttpxBinaryResponseContent:
audio_url, usage = _audio_url(raw_response)
try:
audio_response: Final = await async_safe_get(
self._async_client(), audio_url, timeout=_request_timeout(raw_response)
)
except SSRFError as e:
raise _refused_audio_url(e) from e
return _binary_audio(audio_response, usage)
@override
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract
) -> BaseLLMException:
return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers))

View file

View file

@ -0,0 +1,34 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
_EU_JEV_PREFIX: Final = "eu_jev/"
def eu_jev_registry_model(requested_model: str | None) -> str:
release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None
if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES:
return EU_JEV_LATEST_MODEL
return release
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
requested_model: Final = decision_body.get("model")
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
hook_model: Final = hook_body.get("model")
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
if hook_model == eu_jev_scoped_model(client_body):
client_model: Final = client_body.get("model")
if client_model is None:
return MappingProxyType(without_model)
if isinstance(client_model, str):
return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)})
return MappingProxyType({**without_model, "model": client_model})
if isinstance(hook_model, str):
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
return hook_body

View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class EuJevDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("EU_JEV_API_KEY",)
api_base_env = ("EU_JEV_API_BASE",)
def get_default_api_base(self) -> str | None:
return "https://eu-jev.com"

View file

@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod
This config enables pass-through behavior to the proxy's /v1/responses endpoint.
"""
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
def _openai_sdk_error_body(
error_message: str,
) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict
try:
decoded: Final = _ERROR_BODY.validate_json(error_message)
except ValidationError:
return None
nested: Final = decoded.get("error")
return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded
class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
return join_api_base(api_base, "responses")
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract
) -> BaseLLMException:
return BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
body=_openai_sdk_error_body(error_message),
)
def supports_native_websocket(self) -> bool:
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
return False

View file

@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
)
async def _sent_messages(
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
messages_format: TokenCountMessagesFormat,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
match messages_format:
case "openai":
return messages
case "anthropic":
try:
return await count_view_messages(
model=model,
messages=messages,
tools=tools,
system=system,
context_management=context_management,
)
except AnthropicContextManagementError as error:
return error
case _:
assert_never(messages_format)
class OpenAITokenCounter(BaseTokenCounter):
"""Token counter implementation for OpenAI provider using the Responses API."""
@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter):
rejected_before_provider_call=True,
)
sent_messages: Final = await _sent_messages(
model_to_use, messages, tools, system, messages_format, context_management
)
if isinstance(sent_messages, AnthropicContextManagementError):
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="openai_api",
error=True,
error_message=sent_messages.message,
status_code=sent_messages.status_code,
rejected_before_provider_call=True,
)
resolved: Final = _count_credential(deployment, request_model or model_to_use)
if isinstance(resolved, OpenAIError):
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
input_items, instructions, counted_tools, text = self._responses_request_parts(
model_to_use=model_to_use,
messages=messages,
messages=sent_messages,
tools=tools,
system=system,
messages_format=messages_format,

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
)
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
from litellm.llms.laya.common_utils import laya_response_model
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler:
response_model: Final = (
laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model
)
logged_model: Final = response_model or request_model or "unknown"
registry_request_model: Final = (
eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model
)
logged_model: Final = response_model or registry_request_model or "unknown"
model_name: Final = f"{custom_llm_provider}/{logged_model}"
usage: Final = response.usage or _TypeSafeUsage()
input_tokens: Final = usage.input_tokens
output_tokens: Final = usage.output_tokens
candidate_models: Final = (
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
if custom_llm_provider == "eu_jev"
else (response_model, registry_request_model)
)
candidate_model_keys: Final = tuple(
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (

View file

@ -7975,6 +7975,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
model=model,
custom_llm_provider=custom_llm_provider,
stream_options=kwargs.get("stream_options"),
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
and kwargs["_litellm_strip_stream_usage"] is True,
)
else:
## OpenAI / Azure Text Completion Returns here
@ -8824,8 +8826,9 @@ def speech(
extra_headers: Final = kwargs.get("extra_headers", None)
model_info: Final = kwargs.get("model_info", None)
shared_session: Final = kwargs.get("shared_session", None)
caller_api_base: Final = api_base or base_url
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url
model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base
)
kwargs.pop("tags", [])
@ -8880,13 +8883,13 @@ def speech(
custom_llm_provider=custom_llm_provider,
)
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
if custom_llm_provider == "edenai":
litellm_params_dict["api_base"] = api_base
if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None:
litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base
response = base_llm_http_handler.text_to_speech_handler(
model=model,
input=input,
voice=voice if isinstance(voice, str) else None,
text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(),
text_to_speech_provider_config=text_to_speech_provider_config,
text_to_speech_optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,

View file

@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,
}
)

View file

@ -18346,25 +18346,31 @@
"dashscope/qwen-image-2.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.035,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-2.0-pro": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.075,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.03,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0-pro": {
@ -18372,7 +18378,54 @@
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen3-tts-flash": {
"input_cost_per_character": 1e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-flash": {
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 1.87e-06,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-plus": {
"input_cost_per_character": 3e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-plus": {
"input_cost_per_character": 2e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-flash": {
"input_cost_per_character": 1.5e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/wan3.0-video": {
@ -72330,6 +72383,28 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"eu_jev/eu-jeff-1.0.0": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

View file

@ -1148,9 +1148,9 @@
"text_completion": true,
"embeddings": false,
"image_generations": true,
"image_edits": false,
"image_edits": true,
"audio_transcriptions": false,
"audio_speech": false,
"audio_speech": true,
"rerank": false,
"ocr": false,
"search": false,
@ -2241,6 +2241,32 @@
"generateContent": false
}
},
"eu_jev": {
"display_name": "eu/jev (`eu_jev`)",
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": true,
"a2a": false,
"generateContent": false
}
},
"laya": {
"display_name": "Laya (`laya`)",
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",

View file

@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/tinyfish/",
"/transcribe",
"/typesafe/",
"/eu_jev/",
"/laya/",
"/bespoke/",
"/openrouter/",

View file

@ -23170,6 +23170,39 @@
"title": "CachedTokensDetails",
"type": "object"
},
"ChoiceAnswer": {
"additionalProperties": true,
"properties": {
"choice": {
"title": "Choice",
"type": "string"
},
"confidence": {
"title": "Confidence",
"type": "number"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"choice",
"confidence",
"probabilities"
],
"title": "ChoiceAnswer",
"type": "object"
},
"Click": {
"additionalProperties": true,
"description": "A click action.",
@ -23934,6 +23967,112 @@
"title": "ErrorResponse",
"type": "object"
},
"EuJevDecisionResponse": {
"properties": {
"answers": {
"additionalProperties": {
"discriminator": {
"mapping": {
"choice": "#/components/schemas/ChoiceAnswer",
"noul": "#/components/schemas/NoulAnswer",
"score": "#/components/schemas/ScoreAnswer"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/components/schemas/ChoiceAnswer"
},
{
"$ref": "#/components/schemas/NoulAnswer"
},
{
"$ref": "#/components/schemas/ScoreAnswer"
}
]
},
"description": "One answer per question, under the same names",
"title": "Answers",
"type": "object"
},
"meta": {
"$ref": "#/components/schemas/EuJevMeta"
},
"model": {
"description": "The release that answered, such as eu-jeff-1.0.0",
"title": "Model",
"type": "string"
},
"usage": {
"$ref": "#/components/schemas/EuJevUsage"
}
},
"required": [
"model",
"answers",
"usage",
"meta"
],
"title": "EuJevDecisionResponse",
"type": "object"
},
"EuJevMeta": {
"properties": {
"cost_eur": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "The exact charge in euros, on authenticated requests",
"title": "Cost Eur"
},
"latency_ms": {
"description": "Server-side time for token quoting, credit reservation and inference",
"title": "Latency Ms",
"type": "integer"
},
"mode": {
"const": "live",
"title": "Mode",
"type": "string"
},
"request_id": {
"title": "Request Id",
"type": "string"
}
},
"required": [
"request_id",
"mode",
"latency_ms"
],
"title": "EuJevMeta",
"type": "object"
},
"EuJevUsage": {
"properties": {
"input_tokens": {
"minimum": 0.0,
"title": "Input Tokens",
"type": "integer"
},
"output_tokens": {
"minimum": 0.0,
"title": "Output Tokens",
"type": "integer"
}
},
"required": [
"input_tokens",
"output_tokens"
],
"title": "EuJevUsage",
"type": "object"
},
"FileSearchTool": {
"additionalProperties": true,
"description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).",
@ -25594,6 +25733,26 @@
"title": "NamespaceTool",
"type": "object"
},
"NoulAnswer": {
"additionalProperties": true,
"properties": {
"noul": {
"title": "Noul",
"type": "number"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"noul"
],
"title": "NoulAnswer",
"type": "object"
},
"OperationCreateFile": {
"additionalProperties": true,
"description": "Instruction describing how to create a file via the apply_patch tool.",
@ -28866,6 +29025,59 @@
"title": "Result",
"type": "object"
},
"ScoreAnswer": {
"additionalProperties": true,
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
},
"legend": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
]
},
"title": "Legend",
"type": "object"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"score": {
"title": "Score",
"type": "number"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"score",
"confidence",
"legend",
"probabilities"
],
"title": "ScoreAnswer",
"type": "object"
},
"Screenshot": {
"additionalProperties": true,
"description": "A screenshot action.",
@ -31758,6 +31970,315 @@
]
}
},
"/eu_jev/v1/systemone": {
"post": {
"description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)",
"operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post",
"requestBody": {
"content": {
"application/json": {
"example": {
"model": "eu-jeff-latest",
"questions": {
"frustration": {
"criteria": [
"calm",
"mildly annoyed",
"angry"
],
"instructions": "How frustrated is the customer?",
"type": "score"
},
"is_refund_request": {
"instructions": "Is the customer asking for a refund?",
"type": "noul"
},
"team": {
"criteria": {
"billing": "Payments, invoices, and refunds",
"support": "Technical issues and bugs"
},
"instructions": "Which team should handle this?",
"type": "choice"
}
},
"state": "I was charged twice. Can I get a refund?"
},
"schema": {
"properties": {
"model": {
"default": "eu-jeff-latest",
"description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it",
"title": "Model",
"type": "string"
},
"questions": {
"additionalProperties": {
"discriminator": {
"propertyName": "type"
},
"oneOf": [
{
"properties": {
"criteria": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"description": "Two to 64 options, option name to description. A null description uses the option name",
"maxProperties": 64,
"minProperties": 2,
"title": "Criteria",
"type": "object"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevChoiceQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"anyOf": [
{
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
]
},
"propertyNames": {
"enum": [
"true",
"false"
]
},
"type": "object"
},
{
"type": "null"
}
],
"description": "Optional descriptions of what true and false mean",
"title": "Criteria"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions"
],
"title": "EuJevNoulQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"description": "Two to 10 level descriptions, ordered from low to high",
"items": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"maxItems": 10,
"minItems": 2,
"title": "Criteria",
"type": "array"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevScoreQuestion",
"type": "object"
}
]
},
"description": "One to eight questions, keyed by names of up to 80 characters",
"maxProperties": 8,
"minProperties": 1,
"propertyNames": {
"maxLength": 80,
"minLength": 1
},
"title": "Questions",
"type": "object"
},
"state": {
"anyOf": [
{
"maxLength": 100000,
"minLength": 1,
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
],
"description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array",
"title": "State"
}
},
"required": [
"state",
"questions"
],
"title": "EuJevDecisionRequest",
"type": "object"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/EuJevDecisionResponse"
}
}
},
"description": "Successful Response"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Eu Jev Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/fal_ai/{endpoint}": {
"delete": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",

View file

@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor
)
from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes
@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
message=str(e.detail),
internal_message=e.internal_message,
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=e.status_code,
)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
)
if isinstance(e, ProxyException):
@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
return ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_401_UNAUTHORIZED,
)

View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import (
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
from litellm.proxy._types import *
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str):
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"]
```
"""
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.proxy_server import general_settings, premium_user
try:
@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
if RouteChecks.is_builtin_eu_jev_decision_route(route=route):
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
if requested_model is not None and not isinstance(requested_model, str):
raise HTTPException(status_code=400, detail="eu_jev model must be a string")
return [f"eu_jev/{eu_jev_registry_model(requested_model)}"]
if route == "/cost/predict-cache":
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
return _dedupe_model_candidates(prediction_models)

View file

@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException:
f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}"
),
type=ProxyErrorTypes.auth_provider_unavailable,
param="None",
param=None,
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)

View file

@ -12,6 +12,11 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
class OAuth2TokenValidationError(ValueError):
"""The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it
inactive, or could not be reached."""
class Oauth2Handler:
"""
Handles OAuth2 token validation.
@ -207,7 +212,7 @@ class Oauth2Handler:
)
except httpx.HTTPStatusError as e:
# This will catch any 4xx or 5xx errors
raise ValueError(f"Oauth 2.0 Token validation failed: {e}")
raise OAuth2TokenValidationError(f"Oauth 2.0 Token validation failed: {e}")
except Exception as e:
# This will catch any other errors (like network issues)
raise ValueError(f"An error occurred during token validation: {e}")
raise OAuth2TokenValidationError(f"An error occurred during token validation: {e}")

View file

@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend")
_LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp"))
_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone"
_AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS))
# Writes that only touch the caller's own credential row (BYOK connections and
@ -301,7 +302,7 @@ class RouteChecks:
f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}"
),
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -503,7 +504,19 @@ class RouteChecks:
for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value:
if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"):
return True
return False
return RouteChecks.is_builtin_eu_jev_decision_route(route=route)
@staticmethod
def is_builtin_eu_jev_decision_route(route: str) -> bool:
if route.rstrip("/") != _EU_JEV_DECISION_ROUTE:
return False
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack
InitPassThroughEndpointHelpers,
)
return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
route=_EU_JEV_DECISION_ROUTE
)
@staticmethod
def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool:

View file

@ -1160,6 +1160,24 @@ def _matches_routing_override(token_claims: dict, override: "JWTRoutingOverride"
)
def oauth2_introspection_applies(
api_key: str, route: str, *, enable_oauth2_auth: bool, enable_jwt_auth: bool, jwt_handler: JWTHandler
) -> bool:
"""Whether LiteLLM validates ``api_key`` on ``route`` by OAuth2 token introspection, on LLM API, info and
lens routes only: a non-JWT token when OAuth2 auth is on globally, or a JWT matched by a routing override."""
is_jwt: Final = enable_jwt_auth and jwt_handler.is_jwt(token=api_key)
# Routing uses unverified JWT claims only to choose auth path.
# Final authentication is enforced by the selected validator.
route_jwt_to_oauth2: Final = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler)
covered_route: Final = (
RouteChecks.is_llm_api_route(route=route)
or RouteChecks.is_info_route(route=route)
or route == "/lens"
or route.startswith("/lens/")
)
return covered_route and ((enable_oauth2_auth and not is_jwt) or route_jwt_to_oauth2)
def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool:
routing_overrides: Final = jwt_handler.litellm_jwtauth.routing_overrides
if not routing_overrides:
@ -2278,23 +2296,13 @@ async def user_api_key_auth_builder(
enable_oauth2_auth: Final = general_settings.get("enable_oauth2_auth", False) is True
enable_jwt_auth: Final = general_settings.get("enable_jwt_auth", False) is True
is_jwt = jwt_handler.is_jwt(token=api_key) if enable_jwt_auth else False
# Routing uses unverified JWT claims only to choose auth path.
# Final authentication is enforced by the selected validator.
route_jwt_to_oauth2 = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler)
# OAuth2 applies for:
# 1) when global OAuth2 auth is enabled on LLM + info routes
# 2) JWT tokens that explicitly match routing_overrides on LLM + info routes
route_is_lens: Final = route == "/lens" or route.startswith("/lens/")
should_apply_override_oauth2: Final = route_jwt_to_oauth2 and (
RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens
)
should_apply_global_oauth2: Final = enable_oauth2_auth and (
RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens
)
if (should_apply_global_oauth2 and not is_jwt) or should_apply_override_oauth2:
if oauth2_introspection_applies(
api_key,
route,
enable_oauth2_auth=enable_oauth2_auth,
enable_jwt_auth=enable_jwt_auth,
jwt_handler=jwt_handler,
):
from litellm.proxy.proxy_server import premium_user
if premium_user is not True:
@ -2578,7 +2586,7 @@ async def user_api_key_auth_builder(
raise ProxyException(
message="Authentication Error, No api key passed in.",
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
elif api_key == "":
@ -2586,7 +2594,7 @@ async def user_api_key_auth_builder(
raise ProxyException(
message="Authentication Error, Malformed API Key passed in. Ensure Key has `Bearer ` prefix.",
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -2827,7 +2835,7 @@ async def user_api_key_auth_builder(
raise ProxyException(
message=f"Authentication Error, Tried to access route={route}, which is only for MASTER KEY",
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -2960,7 +2968,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
raise ProxyException(
message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.",
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
await enforce_key_and_fallback_model_access(
@ -3011,7 +3019,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
"Keys owned by this user cannot be used."
),
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -3317,7 +3325,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
raise ProxyException(
message="Authentication Error, Invalid proxy server token passed",
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
if valid_token_dict is not None:

View file

@ -492,7 +492,7 @@ lite codex exec "summarize the repo"
Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug, plus whichever reasoning levels its model group lists on `/model_group/info` the installed Codex can parse. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. The entry also turns on pi's session header (`x-session-id`, sent by pi 0.80.7 and later), so every chat turn of one pi session shares one session id in the logs and sticks to one deployment when session affinity is on. A custom `pi --session-id` must be at least 8 letters, digits, `_`, or `-` for the proxy to read it. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.

View file

@ -11,7 +11,7 @@ from dataclasses import dataclass
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, TypeAlias, TypeVar
from typing import Final, Literal, TypeAlias, TypeVar
from urllib.parse import urlsplit
import click
@ -41,6 +41,7 @@ from .pi import (
LITELLM_PROXY_API_KEY_ENV,
PI_PROVIDER_NAME,
ListingFailure,
ModelLimits,
PiSyncError,
fetch_model_ids,
fetch_model_limits,
@ -611,9 +612,27 @@ class _OpenCodeLimit(LiteLLMBaseModel):
output: int
class _OpenCodeVariant(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
reasoningEffort: str
class _OpenCodeReasoningOption(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["effort"] = "effort"
values: tuple[str, ...]
class _OpenCodeModel(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
name: str
limit: _OpenCodeLimit | None = None
reasoning: Literal[True] | None = None
reasoning_options: tuple[_OpenCodeReasoningOption, ...] | None = None
variants: Mapping[str, _OpenCodeVariant] | None = None
class _OpenCodeProviderOptions(LiteLLMBaseModel):
@ -632,15 +651,43 @@ class _OpenCodeConfig(LiteLLMBaseModel):
provider: Mapping[str, _OpenCodeProvider]
def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel:
if model.max_input_tokens is None or model.max_output_tokens is None:
return _OpenCodeModel(name=model.id)
_NO_GROUPS: Final[Mapping[str, ModelLimits]] = MappingProxyType({})
def _reasoning_efforts(model_id: str, groups: Mapping[str, ModelLimits]) -> tuple[str, ...] | None:
group: Final = groups.get(model_id)
return group.supported_reasoning_efforts if group is not None else None
def _opencode_model_entry(model: ListedModel, groups: Mapping[str, ModelLimits]) -> _OpenCodeModel:
"""One OpenCode model entry, with the model group's reasoning efforts as its variants when known.
OpenCode 1.x builds its own variants for a reasoning model and replaces them only through
`reasoning_options`. 2.x ignores `reasoning` and `reasoning_options` and builds its own variants
unless the entry lists some, so the efforts go in all three, and a model with none lists an
empty `variants`."""
limit: Final = (
_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens)
if model.max_input_tokens is not None and model.max_output_tokens is not None
else None
)
efforts: Final = _reasoning_efforts(model.id, groups)
if efforts is None:
return _OpenCodeModel(name=model.id, limit=limit)
if not efforts:
return _OpenCodeModel(name=model.id, limit=limit, variants=MappingProxyType({}))
return _OpenCodeModel(
name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens)
name=model.id,
limit=limit,
reasoning=True,
reasoning_options=(_OpenCodeReasoningOption(values=efforts),),
variants=MappingProxyType({effort: _OpenCodeVariant(reasoningEffort=effort) for effort in efforts}),
)
def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str:
def opencode_provider_config(
base_url: str, models: Sequence[ListedModel], groups: Mapping[str, ModelLimits] = _NO_GROUPS
) -> str:
"""OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`.
One model entry per chat-capable /v1/models row (mode chat, responses, or
@ -648,6 +695,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so
it never lands in the config text. OpenCode merges this inline config over
the user's own files, leaving unrelated keys and providers untouched.
`groups` is /model_group/info by model group, where the reasoning efforts come from.
"""
chat_models: Final = _chat_models(models)
provider: Final = _OpenCodeProvider(
@ -657,7 +705,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
baseURL=base_url.rstrip("/") + "/v1",
apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}",
),
models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}),
models=MappingProxyType({m.id: _opencode_model_entry(m, groups) for m in chat_models}),
)
config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider}))
return config.model_dump_json(exclude_none=True)
@ -672,7 +720,8 @@ def opencode_model_sync_env(
) -> Mapping[str, str] | ModelSyncSkipped:
"""Env addition that hands OpenCode the proxy's model list, or why it was skipped.
Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT.
Fetches /v1/models with the key, and /model_group/info for each model's reasoning
efforts, and packs them into OPENCODE_CONFIG_CONTENT.
An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a
failed fetch is reported rather than raised: OpenCode still launches on the
plain OPENAI_* env, just without a synced model list.
@ -682,16 +731,20 @@ def opencode_model_sync_env(
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
if isinstance(listing, ModelSyncSkipped):
return listing
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
groups: Final = fetch_model_limits(base_url, api_key, get=get)
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing, groups)})
def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow:
def _codex_row(
model: ListedModel, upstream: ModelUpstream | None, groups: Mapping[str, ModelLimits]
) -> CodexCatalogRow:
return CodexCatalogRow(
id=model.id,
mode=model.mode,
max_input_tokens=model.max_input_tokens,
upstream_model=upstream.model if upstream is not None else None,
reasoning_effort=upstream.reasoning_effort if upstream is not None else None,
supported_reasoning_efforts=_reasoning_efforts(model.id, groups),
)
@ -700,6 +753,8 @@ def codex_model_catalog(
stock: Sequence[CodexStockModel],
instructions: str,
upstreams: Mapping[str, ModelUpstream],
groups: Mapping[str, ModelLimits] = _NO_GROUPS,
client_version: str | None = None,
) -> str | None:
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
@ -714,17 +769,19 @@ def codex_model_catalog(
deployments pin, and a window no larger than its listed max_input_tokens.
A model Codex does not know gets the fallback
entry, with the same base instructions Codex itself uses so the agent never
runs without a system prompt. Apart from aliases, the catalog is the one
`/v1/models?client_version=...` serves, built from the installed Codex's own
entries instead of the proxy's vendored ones.
runs without a system prompt, and offers the reasoning efforts `groups` gives its model group
that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is
the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries
instead of the proxy's vendored ones.
"""
rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models)
rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models)
body: Final = codex_models_response_json(
rows,
stock=MappingProxyType({m.slug: m for m in stock}),
instructions=instructions,
byte_limit=None,
alias_defaults="deployment",
client_version=client_version,
)
return body.json if body.listed else None
@ -771,7 +828,7 @@ def _codex_debug_models(
encoding="utf-8",
timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
)
except (OSError, subprocess.TimeoutExpired) as e:
except (OSError, subprocess.TimeoutExpired, UnicodeError) as e:
return ModelSyncSkipped(f"`{name} debug models` failed: {e}")
if completed.returncode == 0:
return completed.stdout
@ -793,6 +850,25 @@ def _stock_codex_models(
return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
def _codex_version(
binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
) -> str | None:
"""The version `codex --version` prints (`codex-cli 0.165.0`), or None when it cannot be read."""
try:
completed: Final = run(
_windows_command(binary, (binary, "--version")),
env=dict(env),
stdin=subprocess.DEVNULL,
capture_output=True,
encoding="utf-8",
timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
)
except (OSError, subprocess.TimeoutExpired, UnicodeError):
return None
words: Final = completed.stdout.split()
return words[-1] if completed.returncode == 0 and words else None
def codex_model_sync_args(
base_env: Mapping[str, str],
base_url: str,
@ -811,7 +887,8 @@ def codex_model_sync_args(
atomically replaced on every launch. The Codex at `binary` first lists its
own models, so the ones the proxy serves keep that Codex's entries, an alias
included when every unpaused deployment /v1/model/info lists for it routes
to the same OpenAI model and Codex knows that model, and then
to the same OpenAI model and Codex knows that model, and its version, so a
model it does not know offers only reasoning efforts it can parse, and then
reads the file back once before it is handed over. The key never lands in
the file. A failed fetch, read, listing, write or read-back is reported
rather than raised: Codex still launches with its built-in catalog and takes
@ -834,7 +911,9 @@ def codex_model_sync_args(
if isinstance(stock, ModelSyncSkipped):
return stock
upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get)
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams)
groups: Final = fetch_model_limits(base_url, api_key, get=get)
version: Final = _codex_version(binary, base_env, run=run)
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams, groups, version)
if catalog is None:
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
try:

View file

@ -50,8 +50,11 @@ class PiSyncError:
@dataclass(frozen=True, slots=True)
class ModelLimits:
"""What /model_group/info says about one model group; `supported_reasoning_efforts` is None when unknown."""
context_window: int | None
max_tokens: int | None
supported_reasoning_efforts: tuple[str, ...] | None = None
_NonEmptyString = Annotated[str, StringConstraints(min_length=1)]
@ -76,9 +79,12 @@ class _ModelList(LiteLLMBaseModel):
class _ModelGroup(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
model_group: str
max_input_tokens: float | None = None
max_output_tokens: float | None = None
supported_reasoning_efforts: tuple[str, ...] | None = None
class _ModelGroupList(LiteLLMBaseModel):
@ -137,8 +143,8 @@ def fetch_model_limits(
*,
get: Callable[..., requests.Response] = requests.get,
) -> Mapping[str, ModelLimits]:
"""Best effort: pi falls back to its own defaults for models without limits,
so an unavailable /model_group/info must not block the launch."""
"""Best effort: pi, OpenCode and Codex fall back to their own defaults for models without
limits or reasoning efforts, so an unavailable /model_group/info must not block the launch."""
url: Final = base_url.rstrip("/") + "/model_group/info"
try:
resp: Final = get(
@ -156,6 +162,7 @@ def fetch_model_limits(
group.model_group: ModelLimits(
context_window=int(group.max_input_tokens) if group.max_input_tokens else None,
max_tokens=int(group.max_output_tokens) if group.max_output_tokens else None,
supported_reasoning_efforts=group.supported_reasoning_efforts,
)
for group in listing.data
}
@ -168,6 +175,33 @@ def models_json_path(env: Mapping[str, str]) -> Path:
return root / "models.json"
_PI_LADDER_LEVELS: Final = ("minimal", "low", "medium", "high")
_PI_OPT_IN_LEVELS: Final = ("xhigh", "max")
_REASONING_OFF_EFFORT: Final = "none"
def _thinking_level_map(efforts: tuple[str, ...]) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized
"""Each level pi offers maps to a value: pi drops a level mapped to null and shows xhigh and max only when set."""
return {
"off": _REASONING_OFF_EFFORT if _REASONING_OFF_EFFORT in efforts else None,
**{level: level if level in efforts else None for level in _PI_LADDER_LEVELS},
**{level: level for level in _PI_OPT_IN_LEVELS if level in efforts},
}
def _reasoning_fields(limit: ModelLimits | None) -> Mapping[str, JsonValue]:
efforts: Final = limit.supported_reasoning_efforts if limit else None
if not efforts:
return MappingProxyType({})
return MappingProxyType(
{
"reasoning": True,
"thinkingLevelMap": _thinking_level_map(efforts),
"compat": {"supportsDeveloperRole": False},
}
)
def _model_entry(
model_id: str, limits: Mapping[str, ModelLimits]
) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized
@ -178,7 +212,7 @@ def _model_entry(
output: Final[dict[str, JsonValue]] = ( # mutable-ok: JSON field
{"maxTokens": limit.max_tokens} if limit and limit.max_tokens else {}
)
return {"id": model_id, **context, **output}
return {"id": model_id, **context, **output, **_reasoning_fields(limit)}
def provider_block(

View file

@ -114,6 +114,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
error_status_code,
openai_error_param,
openai_error_type,
relayed_provider_specific_fields,
)
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING_BYTES,
@ -588,6 +589,7 @@ def model_deployments_support_stream_options(
_STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD)
_STREAM_USAGE_TRACKED_ROUTES: Final = frozenset({"acompletion", "atext_completion"})
def stream_usage_tracking_updates(
@ -613,7 +615,7 @@ def stream_usage_tracking_updates(
return scrub
if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data:
return {**scrub, "stream_options": {"include_usage": False}}
if always_include is False or route_type != "acompletion":
if always_include is False or route_type not in _STREAM_USAGE_TRACKED_ROUTES:
return scrub
if isinstance(stream_options, dict) and stream_options.get("include_usage") is True:
return scrub
@ -4461,7 +4463,8 @@ class ProxyBaseLLMRequestProcessing:
param=openai_error_param(e),
openai_code=getattr(e, "code", None),
code=_code,
provider_specific_fields=getattr(e, "provider_specific_fields", None),
provider_specific_fields=getattr(e, "provider_specific_fields", None)
or relayed_provider_specific_fields(e),
headers=safe_headers,
)

View file

@ -7,7 +7,8 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo
lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry
Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
upstream.
upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning
levels, trimmed to the ones the requesting Codex release can parse.
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
@ -26,13 +27,16 @@ from pathlib import Path
from types import MappingProxyType, NoneType
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError
from packaging.version import Version
from pydantic import ConfigDict, Field, StringConstraints, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort
from litellm.types.llms.base_model import LiteLLMBaseModel
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RouterRegistry
from litellm.types.proxy.model_listing import ModelInfoResponse
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
@ -43,6 +47,10 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
_TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
_BODY_PREFIX: Final = '{"models":['
_BODY_SUFFIX: Final = "]}"
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
_CODEX_UNSET_EFFORT: Final = "none"
class CodexServiceTier(LiteLLMBaseModel):
@ -67,6 +75,15 @@ class _ConfiguredTier(LiteLLMBaseModel):
_CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...])
class CodexFallbackReasoningLevel(LiteLLMBaseModel):
"""A `ReasoningEffortPreset` as Codex reads it: picking it sends `effort` as the request's reasoning effort."""
model_config = ConfigDict(frozen=True)
effort: str
description: str
class CodexTruncationPolicy(LiteLLMBaseModel):
mode: Literal["bytes"] = "bytes"
limit: int = 10_000
@ -82,10 +99,13 @@ class CodexFallbackModel(LiteLLMBaseModel):
same as `codex -m` did.
"""
model_config = ConfigDict(frozen=True)
slug: str
display_name: str
description: NoneType = None
supported_reasoning_levels: tuple[()] = ()
default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None)
supported_reasoning_levels: tuple[CodexFallbackReasoningLevel, ...] = ()
shell_type: Literal["unified_exec"] = "unified_exec"
visibility: Literal["list"] = "list"
supported_in_api: Literal[True] = True
@ -153,7 +173,8 @@ class CodexCatalogRow:
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
value per deployment behind the id, None where a deployment sets none."""
value per deployment behind the id, None where a deployment sets none. `supported_reasoning_efforts`
is the model group's, None when unknown."""
id: str
mode: str | None = None
@ -162,6 +183,7 @@ class CodexCatalogRow:
reasoning_effort: str | None = None
display_name: str | None = None
service_tiers: tuple[object, ...] = ()
supported_reasoning_efforts: tuple[str, ...] | None = None
search_tool_support: tuple[bool | None, ...] = ()
@ -261,8 +283,40 @@ def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None:
return True if per_deployment and None not in per_deployment else None
def _codex_can_drive(row: CodexCatalogRow) -> bool:
return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
def _codex_can_drive(model_id: str, mode: str | None) -> bool:
return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES)
def _parses_custom_efforts(client_version: str | None) -> bool:
try:
return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS
except ValueError:
return False
def codex_reasoning_levels(
efforts: Sequence[str], client_version: str | None
) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]:
"""The reasoning levels a fallback entry offers for a model group's efforts, and its default level.
Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for
a version that does not parse, only the efforts every release knows are offered. The default is
set whenever a level is offered, because Codex reads a missing one as `none`: `none` itself when the
group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the
nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium,
else the weakest declared level, else the first one."""
parses_any: Final = _parses_custom_efforts(client_version)
offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES)
if not offered:
return (), None
nearest: Final = nearest_declared_reasoning_effort(_PREFERRED_DEFAULT_EFFORT, offered, "down")
levels: Final = tuple(
CodexFallbackReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream")
for effort in offered
)
if _CODEX_UNSET_EFFORT in offered:
return levels, _CODEX_UNSET_EFFORT
return levels, nearest if nearest in offered else offered[0]
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
@ -309,6 +363,7 @@ def _entry(
served: frozenset[str],
instructions: str,
alias_defaults: CodexAliasDefaults,
client_version: str | None,
) -> CodexCatalogEntry:
stock_model: Final = _stock_for(row, stock)
stock_tiers: Final = MappingProxyType(
@ -317,9 +372,12 @@ def _entry(
configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
search_tool: Final = offered_search_tool(row.search_tool_support)
if stock_model is None:
levels, default_level = codex_reasoning_levels(row.supported_reasoning_efforts or (), client_version)
return CodexFallbackModel(
slug=row.id,
display_name=row.display_name if row.display_name is not None else row.id,
default_reasoning_level=default_level,
supported_reasoning_levels=levels,
priority=index,
service_tiers=configured if configured is not None else (),
context_window=row.max_input_tokens,
@ -406,6 +464,7 @@ def codex_models_response_json(
instructions: str | None = None,
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
alias_defaults: CodexAliasDefaults = "codex",
client_version: str | None = None,
) -> CodexCatalogBody:
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
@ -417,14 +476,15 @@ def codex_models_response_json(
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
them, as Codex would use them for the model's own name.
them, as Codex would use them for the model's own name. `client_version` is the requesting Codex's version, which decides the reasoning
levels a fallback entry may offer; None counts as a version too old for custom levels.
"""
known: Final = bundled_codex_models() if stock is None else stock
prompt: Final = codex_base_instructions() if instructions is None else instructions
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
drivable: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode))
served: Final = frozenset(row.id for row in drivable)
entries: Final = tuple(
_entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable)
_entry(index, row, known, served, prompt, alias_defaults, client_version) for index, row in enumerate(drivable)
)
serialized: Final = tuple(_serialized(entry) for entry in entries)
kept: Final = _kept_positions(entries, serialized, byte_limit)
@ -435,6 +495,15 @@ def codex_models_response_json(
)
def _group_reasoning_efforts(
row: ModelInfoResponse, lookup_id: str, registry: RouterRegistry | None, team_id: str | None
) -> tuple[str, ...] | None:
if registry is None or not _codex_can_drive(row["id"], row.get("mode")):
return None
group: Final = registry.group(lookup_id, include_hidden=True, team_id=team_id)
return group.supported_reasoning_efforts if group is not None else None
def _catalog_row(
row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None
) -> CodexCatalogRow:
@ -446,6 +515,7 @@ def _catalog_row(
upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None,
display_name=registry.group(label_id, field="display_name") if registry is not None else None,
service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (),
supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id),
search_tool_support=(
registry.resolve(lookup_id, kind="search_tool_support", team_id=team_id) if registry is not None else ()
),
@ -463,7 +533,8 @@ def codex_catalog_rows(
team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias`
reading its target's deployments under its own name; `team_id` is the requesting key's team, so the
upstream model that picks Codex's stock entry and the tiers are read only off the deployments its
requests can route to. Display names are read under `label_entries`' ids when given."""
requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the
entry. Display names are read under `label_entries`' ids when given."""
lookup_ids: Final = MappingProxyType(dict(entries))
label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries))
return tuple(
@ -480,9 +551,13 @@ def codex_model_list_body(
llm_router: Router | None,
team_id: str | None = None,
label_entries: Sequence[tuple[str, str]] | None = None,
*,
client_version: str | None = None,
) -> str:
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out."""
body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries))
body: Final = codex_models_response_json(
codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version
)
if body.left_out:
verbose_proxy_logger.warning(
"Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, "

View file

@ -8,7 +8,9 @@ from types import MappingProxyType
from typing import Final
from fastapi import HTTPException, status
from pydantic import TypeAdapter
from litellm._logging import redact_internal_details_from_client_fields
from litellm.constants import STRINGIFIED_NONE
from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
@ -28,6 +30,8 @@ _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, sta
_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600)
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
_FAILOVER_REFUSALS: Final = (
@ -92,6 +96,18 @@ def openai_error_param(exc: object) -> str | None:
return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
def relayed_provider_specific_fields(
exc: object,
) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict
body: Final = attribute_of(exc, "body")
if not isinstance(body, dict):
return None
fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields")
if not isinstance(fields, dict):
return None
return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields)))
def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException:
match exc:
case ProxyException():
@ -146,14 +162,43 @@ def _is_failover_refusal(exc: Exception) -> bool:
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
def answered_status_code(message: str, status_code: int) -> int:
return int(proxy_exception_status_code(message, str(status_code)))
def route_failure_proxy_exception(
exc: Exception,
message: str,
default_status_code: int,
headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers
openai_code: str | None = None,
) -> ProxyException:
raised_status: Final = error_status_code(exc, default_status_code)
answered_status: Final = answered_status_code(message, raised_status)
return ProxyException(
message=message,
type=_route_failure_error_type(exc, raised_status, answered_status),
param=openai_error_param(exc),
headers=headers,
openai_code=openai_code,
code=answered_status,
)
def _route_failure_error_type(exc: Exception, raised_status: int, answered_status: int) -> str:
if answered_status != raised_status and answered_status in _AUTH_FAILURE_STATUS_CODES:
return ProxyErrorTypes.auth_error.value
return openai_error_type(exc, answered_status)
def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
answered_status: Final = answered_status_code(message, status_code)
return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status))
def status_typed_proxy_exception(exc: HTTPException) -> ProxyException:
message: Final = str(exc.detail)
answered_status: Final = int(proxy_exception_status_code(message, str(exc.status_code)))
answered_status: Final = answered_status_code(message, exc.status_code)
return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status))

View file

@ -435,7 +435,7 @@ class PrismaDBExceptionHandler:
return ProxyException(
message=PrismaDBExceptionHandler.database_unavailable_message(e),
type=ProxyErrorTypes.no_db_connection,
param="None",
param=None,
code=HTTPStatus.SERVICE_UNAVAILABLE.value,
)

View file

@ -664,4 +664,7 @@ class GenericGuardrailAPI(CustomGuardrail):
GuardrailEventHooks.pre_call,
GuardrailEventHooks.post_call,
GuardrailEventHooks.during_call,
GuardrailEventHooks.pre_mcp_call,
GuardrailEventHooks.during_mcp_call,
GuardrailEventHooks.post_mcp_call,
]

View file

@ -2505,6 +2505,7 @@ def refresh_proxy_server_request_body_snapshot(
"litellm_logging_obj",
VECTOR_STORE_CALLER_METADATA_KWARG,
STREAM_USAGE_INJECTED_FIELD,
"_litellm_strip_stream_usage",
}
)
| _TRANSPORT_ONLY_CREDENTIAL_KEYS

View file

@ -75,7 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
coordination_redis_cache,
publish_config_change,
)
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for
from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
@ -4959,7 +4959,7 @@ async def info_key_fn(
raise ProxyException(
message="Internal server error from /key/info. Details are in the proxy server logs.",
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -8156,7 +8156,7 @@ async def key_health(
raise ProxyException(
message=f"Key health check failed: {e}",
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)

View file

@ -46,6 +46,7 @@ from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy
encrypt_callback_vars,
is_sensitive_callback_key,
)
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports
_get_validated_callback_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
convert_key_logging_metadata_to_callback,
@ -463,7 +464,7 @@ async def add_team_callbacks(
raise ProxyException(
message="Internal Server Error, " + str(e),
type=ProxyErrorTypes.internal_server_error.value,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -606,7 +607,7 @@ async def delete_team_callback(
raise ProxyException(
message="Internal Server Error, " + str(e),
type=ProxyErrorTypes.internal_server_error.value,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
else:
@ -747,7 +748,7 @@ async def disable_team_logging(
raise ProxyException(
message="Internal Server Error, " + str(e),
type=ProxyErrorTypes.internal_server_error.value,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -838,7 +839,7 @@ async def get_team_callbacks(
raise ProxyException(
message=getattr(e, "detail", f"Internal Server Error({e})"),
type=ProxyErrorTypes.internal_server_error.value,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -846,6 +847,6 @@ async def get_team_callbacks(
raise ProxyException(
message="Internal Server Error, " + str(e),
type=ProxyErrorTypes.internal_server_error.value,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)

View file

@ -68,8 +68,11 @@ module materially harder to understand.
behind a single generic branch unless tests prove every mode still behaves
correctly.
- Be especially careful with legacy `delegate_auth_to_upstream: true`. `auth_type: oauth2`
with `delegate_auth_to_upstream: true` is deprecated: LiteLLM admission is required
for matching MCP routes. Use `auth_type: oauth_delegate` for client-forwarded OAuth.
with `delegate_auth_to_upstream: true` is deprecated but still supported. A bare
`Authorization` that is a LiteLLM credential is validated and never forwarded. Any other
bearer is the upstream token: it is forwarded, and the caller is admitted as the JWT or
custom-auth user it also proves when that user is granted the server, else anonymously.
New servers should use `auth_type: true_passthrough` or `auth_type: oauth_delegate`.
OAuth discovery endpoints stay public so clients can start the RFC 9728 flow.
- Keep database-backed fields in sync across migrations, typed models under
`litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this

View file

@ -2,14 +2,15 @@ import asyncio
import functools
import re
import secrets
from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
import jwt
from fastapi import HTTPException
from pydantic import SecretStr, TypeAdapter
from starlette.datastructures import Headers
@ -45,9 +46,11 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth
from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.oauth2_check import OAuth2TokenValidationError
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports
_get_bearer_token_or_received_api_key, # pyright: ignore[reportPrivateUsage] # shared x-litellm-api-key parser lives with user_api_key_auth
_run_centralized_common_checks, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
oauth2_introspection_applies,
run_centralized_common_checks,
user_api_key_auth,
)
@ -100,6 +103,9 @@ if TYPE_CHECKING:
_EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({})
OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
_PROXY_SETTINGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_CAUSE_CHAIN_MAX_DEPTH: Final = 16
_IdentityCheck: TypeAlias = Literal["oauth2", "oauth2_proxy", "jwt"]
_CACHED_VALUE: Final = TypeAdapter(object)
@ -233,6 +239,136 @@ def _is_legacy_delegate_cold_start(mcp_servers: list[str] | None, client_ip: str
return True
def _cause_chain(exc: BaseException) -> Iterator[BaseException]:
link: BaseException | None = exc # rebind-ok: advances one link of the cause chain per iteration
for _ in range(_CAUSE_CHAIN_MAX_DEPTH):
if link is None:
return
yield link
link = link.__cause__ or link.__context__
def _caused_by(exc: BaseException, error_type: type[BaseException]) -> bool:
return any(isinstance(link, error_type) for link in _cause_chain(exc))
def _signed_with(token: str, secret: str) -> bool:
"""True when ``token`` is a JWT whose HS256 signature verifies with ``secret``, whatever its claims say."""
try:
jwt.decode(
token,
secret,
algorithms=["HS256"],
options={"verify_exp": False, "verify_nbf": False, "verify_iat": False, "verify_aud": False},
)
except jwt.PyJWTError:
return False
return True
async def _bearer_is_litellm_credential(token: str) -> bool:
"""True when ``token`` is, or may be, a credential LiteLLM issued: the master key or a JWT it signed
(dashboard sessions, BYOK sessions), a sealed gateway bearer, or a virtual key the key table holds (a
lookup fault counts as held). Only ``sk-`` tokens are looked up, because no other token can be a
virtual key."""
from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module
is_sealed_gateway_bearer,
virtual_key_may_exist,
)
from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module
LITELLM_VIRTUAL_KEY_PREFIX,
)
from litellm.proxy.proxy_server import master_key # noqa: PLC0415 # startup owns the master key
if master_key and (secrets.compare_digest(token.encode(), master_key.encode()) or _signed_with(token, master_key)):
return True
if is_sealed_gateway_bearer(token):
return True
return token.startswith(LITELLM_VIRTUAL_KEY_PREFIX) and await virtual_key_may_exist(token)
def _identity_check(token: str, route: str, settings: Mapping[str, object]) -> _IdentityCheck | None:
"""The identity check LiteLLM's own auth would run on ``token`` for ``route``, in its order: OAuth2
introspection, then trusted reverse-proxy identity headers, then a JWT whose issuer may be LiteLLM's."""
from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle
from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module
jwt_issuer_may_be_litellm,
)
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
jwt_handler,
premium_user,
)
enable_jwt_auth: Final = settings.get("enable_jwt_auth") is True
if oauth2_introspection_applies(
token,
route,
enable_oauth2_auth=settings.get("enable_oauth2_auth") is True,
enable_jwt_auth=enable_jwt_auth,
jwt_handler=jwt_handler,
):
return "oauth2" if premium_user is True else None
if settings.get("enable_oauth2_proxy_auth") is True:
return "oauth2_proxy"
if enable_jwt_auth and premium_user is True and JWTHandler.is_jwt(token=token) and jwt_issuer_may_be_litellm(token):
return "jwt"
return None
async def _verify_identity(check: Literal["oauth2_proxy", "jwt"], token: str, request: Request) -> None:
from litellm.proxy.auth.oauth2_proxy_hook import ( # noqa: PLC0415 # proxy import cycle
handle_oauth2_proxy_request,
)
from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration
match check:
case "oauth2_proxy":
await handle_oauth2_proxy_request(request=request)
case "jwt":
await jwt_handler.auth_jwt(token=token) # pyright: ignore[reportUnknownMemberType] # returns a bare dict; only whether it raises matters
case _: # pragma: no cover - exhaustiveness guard, unreachable while the Literal holds
assert_never(check) # pragma: no cover
async def _upstream_bearer_identity(token: str, litellm_api_key: str, request: Request) -> UserAPIKeyAuth | None:
"""The LiteLLM user an upstream bearer also proves, or ``None`` when it proves none.
Custom auth decides alone, so any rejection is ``None``. Otherwise the identity check LiteLLM would run
must vouch for the caller. Reverse-proxy headers and a JWT are verified first, and OAuth2 introspection
runs once inside the full check, where a provider that does not vouch is ``None``. Past that, only a
user LiteLLM does not know is ``None``; every other refusal (budget, blocked team, disallowed route)
propagates."""
from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 # proxy import cycle
from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module
custom_auth_configured,
)
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict
)
if custom_auth_configured():
try:
return await user_api_key_auth(api_key=litellm_api_key, request=request)
except (HTTPException, ProxyException):
return None
check: Final = _identity_check(
token, get_request_route(request), _PROXY_SETTINGS_ADAPTER.validate_python(general_settings)
)
if check is None:
return None
if check != "oauth2":
try:
await _verify_identity(check, token, request)
except Exception: # noqa: BLE001 # any verification failure means LiteLLM does not vouch for this bearer
return None
try:
return await user_api_key_auth(api_key=litellm_api_key, request=request)
except (HTTPException, ProxyException, OAuth2TokenValidationError) as exc:
if _caused_by(exc, UserNotFoundError) or _caused_by(exc, OAuth2TokenValidationError):
return None
raise
def _is_litellm_auth_admission_error(exc: Exception) -> bool:
if isinstance(exc, HTTPException):
return exc.status_code == 401
@ -704,6 +840,19 @@ class MCPRequestHandler:
route=request_route,
mcp_servers=mcp_servers,
)
elif oauth2_headers and (
legacy_delegate_targets := MCPRequestHandler._legacy_delegate_targets(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
):
validated_user_api_key_auth = await MCPRequestHandler._admit_legacy_delegate_bearer( # rebind-ok: one assignment per admission arm, like its siblings
authorization_value=oauth2_headers["Authorization"],
litellm_api_key=litellm_api_key,
targets=legacy_delegate_targets,
request=request,
)
elif oauth2_headers:
# Authorization on a non-delegated server: the bearer must be a real
# LiteLLM credential, so a failed validation is a genuine 401/403 and
@ -741,10 +890,14 @@ class MCPRequestHandler:
custom_key_header_name: Final = OPTIONAL_STRING_ADAPTER.validate_python(
general_settings.get("litellm_key_header_name")
)
admitted_credential: Final = MCPRequestHandler.caller_admission_credential(
headers,
validated_user_api_key_auth,
custom_key_header_name=custom_key_header_name,
admitted_credential: Final = (
None
if validated_user_api_key_auth.mcp_forwards_caller_authorization
else MCPRequestHandler.caller_admission_credential(
headers,
validated_user_api_key_auth,
custom_key_header_name=custom_key_header_name,
)
)
(
oauth2_headers,
@ -934,6 +1087,53 @@ class MCPRequestHandler:
return False
return True
@staticmethod
def _legacy_delegate_targets(
path: str, mcp_servers: Sequence[str] | None, client_ip: str | None
) -> tuple[MCPServer, ...]:
"""Every server the request targets when ALL of them are legacy ``delegate_auth_to_upstream``
servers, else ``()``. Names resolve the way downstream routing resolves them, so a permissive
``x-mcp-servers`` header cannot open the gate for a stricter path target."""
from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module
global_mcp_server_manager,
is_legacy_upstream_delegate,
)
servers: Final = tuple(
global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip)
for name in MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers)
)
delegates: Final = tuple(
server for server in servers if server is not None and is_legacy_upstream_delegate(server)
)
return delegates if servers and len(delegates) == len(servers) else ()
@staticmethod
async def _admit_legacy_delegate_bearer(
authorization_value: str,
litellm_api_key: str,
targets: tuple[MCPServer, ...],
request: Request,
) -> UserAPIKeyAuth:
"""Admit a bare ``Authorization`` bearer sent to legacy ``delegate_auth_to_upstream`` servers.
A LiteLLM credential runs normal admission, and its errors propagate. Any other bearer is the
upstream token and is marked for forwarding: a user it also proves who is granted every target is
admitted as that user, and every other caller is admitted anonymously."""
from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module
global_mcp_server_manager,
)
token: Final = _normalize_caller_admission_credential(authorization_value)
if await _bearer_is_litellm_credential(token):
return await user_api_key_auth(api_key=litellm_api_key, request=request)
identity: Final = await _upstream_bearer_identity(token, litellm_api_key, request)
if identity is not None:
allowed: Final = frozenset(await global_mcp_server_manager.get_allowed_mcp_servers(identity))
if all(server.server_id in allowed for server in targets):
return identity.model_copy(update={"mcp_forwards_caller_authorization": True})
return UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True})
@staticmethod
def _single_dcr_bridge_delegate_target(
path: str, mcp_servers: list[str] | None, client_ip: str | None
@ -1553,7 +1753,7 @@ class MCPRequestHandler:
return expiry >= datetime.now(timezone.utc)
@staticmethod
def _resolve_target_server_names(path: str, mcp_servers_header: list[str] | None) -> list[str]:
def _resolve_target_server_names(path: str, mcp_servers_header: Sequence[str] | None) -> Sequence[str]:
"""
Resolve the target MCP server names exactly as downstream routing
does (``server.py::extract_mcp_auth_context``).

View file

@ -61,7 +61,6 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool:
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
jwt_handler,
master_key,
user_custom_auth,
)
if "x-litellm-api-key" in request.headers:
@ -71,10 +70,26 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool:
return "authorization" in request.headers
if token.startswith("sk-") or (master_key and secrets.compare_digest(token.encode(), master_key.encode())):
return True
if user_custom_auth is not None or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled:
if custom_auth_configured() or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled:
return True
if not JWTHandler.is_jwt(token):
return await _opaque_bearer_is_gateway_credential(token)
return is_sealed_gateway_bearer(token) or await virtual_key_may_exist(token)
return jwt_issuer_may_be_litellm(token)
def custom_auth_configured() -> bool:
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
user_custom_auth,
)
return user_custom_auth is not None
def jwt_issuer_may_be_litellm(token: str) -> bool:
"""False only when the JWT names an issuer LiteLLM's JWT auth is not configured to accept."""
from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle
from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration
claims: Final = JWTHandler.get_unverified_claims(token)
issuer: Final = claims.get("iss") if claims is not None else None
global_issuer: Final = os.getenv("JWT_ISSUER")
@ -86,25 +101,35 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool:
)
async def _opaque_bearer_is_gateway_credential(token: str) -> bool:
from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle
def is_sealed_gateway_bearer(token: str) -> bool:
"""True for a bearer only the gateway issues: a bridge or refresh envelope, a v2 GCM blob, or a
dashboard UI token. A UI token that cannot be read still counts, so a lookup fault never reads as foreign."""
from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken # noqa: PLC0415 # proxy import cycle
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle
from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle
from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # envelope imports bridge types
is_envelope,
is_refresh_envelope,
)
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies
prisma_client,
user_api_key_cache,
)
if is_envelope(token) or is_refresh_envelope(token) or token.startswith(V2_GCM_PREFIX):
return True
try:
if ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None:
return True
return ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None
except Exception as exc: # noqa: BLE001 # an identity lookup fault must not permit cookie fallback
verbose_logger.debug("OAuth bearer ownership could not be checked (%s)", type(exc).__name__)
return True
async def virtual_key_may_exist(token: str) -> bool:
"""False only when the key table answers that ``token`` is not a virtual key; a lookup fault counts as held."""
from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle
from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies
prisma_client,
user_api_key_cache,
)
try:
await IdentityStore(prisma_client, user_api_key_cache).resolve(hashed_token=hash_token(token))
except KeyNotFoundError:
return False

View file

@ -1196,8 +1196,11 @@ def should_strip_caller_authorization(
pass-through cold-start case (RFC 9728) the bearer in
``Authorization`` is the upstream OAuth token and must be
forwarded, so we keep it.
- **Delegated OAuth servers**: admission always runs and there is no
anonymous path, so the caller's separate ``Authorization`` is
- **Legacy delegate servers**: forward when MCP admission marked the
caller's ``Authorization`` as the upstream token
(``mcp_forwards_caller_authorization``); otherwise follow the
delegated OAuth rule below.
- **Delegated OAuth servers**: the caller's separate ``Authorization`` is
forwarded only when a distinct ``x-litellm-api-key`` carried
admission. Without that header the ``Authorization`` *was* the
admission credential — a virtual key, an IdP JWT, or an SSO / OIDC /
@ -1206,6 +1209,12 @@ def should_strip_caller_authorization(
- **Every other auth-enabled mode**: strip when ``Authorization`` is the
credential admission validated.
"""
if (
user_api_key_auth is not None
and user_api_key_auth.mcp_forwards_caller_authorization
and is_legacy_upstream_delegate(mcp_server)
):
return False
if _authorization_is_admission_jwt(raw_headers, user_api_key_auth):
return True
if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
@ -1557,6 +1566,18 @@ async def _materialize_auth_headers(auth: httpx2.Auth | None) -> dict[str, str]
return {header_name: header_value} if header_value else None
def is_legacy_upstream_delegate(server: MCPServer) -> bool:
"""True for an interactive ``auth_type=oauth2`` server with ``delegate_auth_to_upstream=true``, whose
caller ``Authorization`` is the upstream token. The flow is resolved, not read from the column, so an
unstamped M2M-shaped row stays excluded: its stored client credentials drive egress, and admitting it
anonymously would let any caller act as LiteLLM's service account."""
return (
server.auth_type == MCPAuth.oauth2
and server.delegate_auth_to_upstream is True
and MCPServerManager.effective_oauth2_flow(server) != "client_credentials"
)
def _consumes_caller_authorization(server: MCPServer) -> bool:
"""True when this server's egress forwards the caller's request-wide ``Authorization`` upstream:
the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated
@ -1726,18 +1747,15 @@ def _warn_overlapping_tool_policies(server: MCPServer) -> None:
def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None:
"""Direct legacy delegated OAuth configurations to the admitted replacement."""
if server.auth_type != MCPAuth.oauth2:
return
if getattr(server, "delegate_auth_to_upstream", False) is not True:
return
if server.has_client_credentials:
"""Point legacy delegated OAuth configurations at the client-forwarded replacements."""
if not is_legacy_upstream_delegate(server):
return
label: Final = get_server_prefix(server)
verbose_logger.warning(
"MCP server %r (id=%s, source=%s) uses deprecated auth_type=oauth2 with "
"delegate_auth_to_upstream=true. LiteLLM admission is now required; migrate to "
"auth_type=oauth_delegate for client-forwarded OAuth.",
"delegate_auth_to_upstream=true. It still works: a caller without a LiteLLM credential is "
"admitted anonymously and its upstream token is forwarded. Migrate to auth_type=true_passthrough "
"or auth_type=oauth_delegate.",
label,
server.server_id,
source,
@ -3766,12 +3784,12 @@ class MCPServerManager:
)
)
if is_anonymous:
passthrough_server_ids: Final = [
anonymous_server_ids: Final = [
server.server_id
for server in self.get_registry().values()
if server.auth_type == MCPAuth.true_passthrough
if server.auth_type == MCPAuth.true_passthrough or is_legacy_upstream_delegate(server)
]
combined_servers.update(passthrough_server_ids)
combined_servers.update(anonymous_server_ids)
restrict_allow_all: Final = (
resolved_general_settings.get("mcp_allow_all_keys_respects_mcp_scope", False)

View file

@ -588,6 +588,7 @@ if MCP_AVAILABLE:
from litellm.proxy.mcp_server.mcp_server_manager import (
MCPServerManager,
global_mcp_server_manager,
is_legacy_upstream_delegate,
)
######################################################
@ -2162,8 +2163,9 @@ if MCP_AVAILABLE:
oauth2_headers: Mapping[str, str] | None,
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
raw_headers: Mapping[str, str] | None,
requested_servers: Sequence[str] | None = None,
) -> None:
"""Probe pass-through upstream servers in parallel before the MCP session starts.
"""Probe pass-through and legacy delegate upstream servers in parallel before the MCP session starts.
Only servers the caller's key is already authorized to reach are probed —
the list is derived from _get_allowed_mcp_servers so that a user cannot
@ -2186,6 +2188,18 @@ if MCP_AVAILABLE:
mcp_servers=mcp_servers,
client_ip=client_ip,
)
# A legacy delegate is probed only when the single requested name resolves to it the way admission
# resolves it, so neither a multi-server connect nor an access group can be failed by one rejected
# token; its challenge echoes the requested name, matching the tokenless challenge.
requested_scope: Final = requested_servers if requested_servers is not None else mcp_servers
single_requested_name: Final = (
requested_scope[0] if requested_scope is not None and len(requested_scope) == 1 else None
)
requested_server: Final = (
operations.global_mcp_server_manager.get_mcp_server_by_name(single_requested_name, client_ip=client_ip)
if single_requested_name is not None
else None
)
prepared_headers: Final = tuple(
(
srv,
@ -2201,6 +2215,11 @@ if MCP_AVAILABLE:
)
for srv in allowed_servers
if srv.is_oauth_passthrough
or (
requested_server is not None
and srv.server_id == requested_server.server_id
and is_legacy_upstream_delegate(srv)
)
)
probe_headers: Final = tuple(
(
@ -2213,7 +2232,7 @@ if MCP_AVAILABLE:
for srv, (auth, extra) in prepared_headers
)
probe_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = tuple(
(srv, auth_header, srv.name)
(srv, auth_header, srv.name if srv.is_oauth_passthrough else single_requested_name or srv.name)
for srv, headers in probe_headers
if (
auth_header := next(
@ -2311,6 +2330,7 @@ if MCP_AVAILABLE:
if context.mcp_server_auth_headers is not None
else None,
raw_headers=context.raw_headers,
requested_servers=context.mcp_servers,
)
return None

View file

@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import LiteLLMRoutes
from litellm.proxy.auth.route_checks import RouteChecks
class BillableCategory(str, Enum):
@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None:
suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None)
if suffix_match is not None:
return suffix_match
if RouteChecks.is_builtin_eu_jev_decision_route(route=path):
return path
# Deep passthrough paths only: the bare prefix itself is not an inference call.
return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None)

View file

@ -91,6 +91,7 @@ from litellm.proxy.common_request_processing import (
open_sse_before_first_byte,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
_safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
@ -119,6 +120,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
)
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
HttpPassThroughEndpointHelpers,
YieldsToConfiguredPassThroughRoute,
create_pass_through_route,
create_websocket_passthrough_route,
websocket_passthrough_request,
@ -132,6 +134,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
is_allowed_to_call_vector_store_endpoint,
)
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool
from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
@ -714,6 +717,85 @@ def typesafe_allowlist_model_names() -> tuple[str, ...]:
return ("typesafe/*", *jev_models_typesafe_serves)
_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body(
EuJevDecisionRequest,
{
"model": "eu-jeff-latest",
"state": "I was charged twice. Can I get a refund?",
"questions": {
"team": {
"type": "choice",
"instructions": "Which team should handle this?",
"criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"},
},
"is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
"frustration": {
"type": "score",
"instructions": "How frustrated is the customer?",
"criteria": ["calm", "mildly annoyed", "angry"],
},
},
},
)
eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute)
@eu_jev_router.post(
"/eu_jev/v1/systemone",
tags=["eu/jev Pass-through", "pass-through"],
responses={200: {"model": EuJevDecisionResponse}},
openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY},
)
async def eu_jev_proxy_route(
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> Response:
"""
Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.
`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)
and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and
`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.
[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)
"""
body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request))
if "custom_body" in body:
raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests")
if body.get("stream"):
raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests")
eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev")
if eu_jev_credentials is None:
raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy")
base_url: Final = httpx.URL(
(eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com")
.rstrip("/")
.removesuffix("/v1")
)
updated_url: Final = base_url.copy_with(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"),
)
endpoint_func: Final = create_pass_through_route(
endpoint="v1/systemone",
target=str(updated_url),
custom_headers={
"Authorization": f"Bearer {eu_jev_credentials.api_key}",
"Content-Type": "application/json",
},
custom_llm_provider="eu_jev",
is_streaming_request=False,
)
return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python(
await endpoint_func(request, fastapi_response, user_api_key_dict)
)
router.include_router(eu_jev_router)
@router.post(
"/laya/v1/systemone",
tags=["Laya Pass-through", "pass-through"],

View file

@ -26,9 +26,11 @@ from fastapi import (
status,
)
from fastapi.responses import StreamingResponse
from fastapi.routing import APIRoute
from pydantic import TypeAdapter
from starlette.datastructures import UploadFile as StarletteUploadFile
from starlette.routing import BaseRoute, Route
from starlette.routing import BaseRoute, Match, Route
from starlette.types import Scope
from starlette.websockets import WebSocketState
from typing_extensions import ReadOnly
from websockets.asyncio.client import connect
@ -73,6 +75,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body
from litellm.llms.oss_decision import validate_oss_request
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._lazy_features import lazy_owned_routes
@ -90,6 +93,7 @@ from litellm.proxy.auth.auth_utils import (
get_request_route,
request_dispatched_to_pass_through_endpoint,
)
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import (
ProxyBaseLLMRequestProcessing,
@ -433,9 +437,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
parsed_url: Final = urlparse(url)
if custom_llm_provider == "chatgpt":
return EndpointType.OPENAI
if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith(
"/v1/systemone"
):
if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix(
"/"
).endswith("/v1/systemone"):
return EndpointType.DECISIONS
if (
("generateContent") in url
@ -1137,6 +1141,16 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return headers
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | None) -> Mapping[str, str]:
if eu_jev_client_body is None:
return MappingProxyType({})
return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)})
async def pass_through_request(
request: Request,
target: str,
@ -1255,7 +1269,12 @@ async def pass_through_request(
strip_client_guardrail_information(client_body)
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
_strip_client_pricing_overrides(client_body)
_parsed_body = client_body
eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body)
scoped_client_body: Final = {
**client_body,
**_eu_jev_scoped_model_override(eu_jev_client_body),
}
_parsed_body = scoped_client_body
if custom_llm_provider in ("laya", "bespoke"):
decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body)
checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request)
@ -1339,6 +1358,12 @@ async def pass_through_request(
)
_ = validate_oss_request(custom_llm_provider, decision_body)
_parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body)
if eu_jev_client_body is not None:
_parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage
eu_jev_upstream_body(
TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body
)
)
resolved_timeout: Final = resolve_pass_through_request_timeout(timeout)
async_client_obj: Final = get_async_httpx_client(
llm_provider=httpxSpecialProvider.PassThroughEndpoint,
@ -2240,17 +2265,23 @@ def create_pass_through_route(
stream,
) = await _parse_request_data_by_content_type(request)
if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path):
if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or (
custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path)
):
raise HTTPException(
status_code=404,
detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.",
)
passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route(
route=path, method=request.method
serves_configured_route: Final = custom_llm_provider != "eu_jev"
passthrough_params: Final = (
InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method)
if serves_configured_route
else None
)
if (
passthrough_params is None
serves_configured_route
and passthrough_params is None
and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None
):
raise HTTPException(
@ -3392,6 +3423,37 @@ class InitPassThroughEndpointHelpers:
del _registered_pass_through_routes[key]
verbose_proxy_logger.debug("Removed pass-through route from registry: %s", key)
@staticmethod
def _llm_api_routes_of_route(route_info: Mapping[str, object]) -> tuple[str, ...]:
path: Final = route_info.get("path")
if not isinstance(path, str):
return ()
if route_info.get("type") == "subpath":
return (path, path.rstrip("/") + "/*")
return (path,)
@staticmethod
def llm_api_routes_of_endpoint(endpoint_id: str) -> frozenset[str]:
route_infos: Final = (
route_info
for route_info in _registered_pass_through_routes.values()
if route_info["endpoint_id"] == endpoint_id
)
return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos)))
@staticmethod
def auth_enforced_llm_api_routes() -> frozenset[str]:
route_infos: Final = (
route_info for route_info in _registered_pass_through_routes.values() if route_info.get("auth") is True
)
return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos)))
@staticmethod
def drop_llm_api_routes(routes: frozenset[str]) -> None:
openai_routes: Final = LiteLLMRoutes.openai_routes.value
for route in routes.intersection(openai_routes):
openai_routes.remove(route)
@staticmethod
def clear_all_pass_through_routes():
"""Clear all pass-through routes from the registry"""
@ -3436,6 +3498,8 @@ class InitPassThroughEndpointHelpers:
return True
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route):
return True
# Fast path: check if any registered route key contains this path
# Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}"
@ -3512,6 +3576,25 @@ class InitPassThroughEndpointHelpers:
return None
@staticmethod
def has_configured_route_ignoring_trailing_slash(route: str) -> bool:
bare_route: Final = route.rstrip("/")
return any(
InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None
for form in (bare_route, f"{bare_route}/")
)
class YieldsToConfiguredPassThroughRoute(APIRoute):
def matches(self, scope: Scope) -> tuple[Match, Scope]:
match, child_scope = super().matches(scope)
if match is Match.NONE:
return match, child_scope
configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
route=get_request_route(Request(scope))
)
return (Match.NONE, {}) if configured else (match, child_scope)
def _get_combined_pass_through_endpoints(
pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint],
@ -3971,9 +4054,7 @@ async def update_pass_through_endpoints(
# Update the list
pass_through_endpoint_data[endpoint_index] = endpoint_dict
# Remove old routes from registry before they get re-registered
InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id)
replaced_llm_api_routes: Final = InitPassThroughEndpointHelpers.llm_api_routes_of_endpoint(endpoint_id)
## Update db
updated_data: Final = ConfigFieldUpdate(
@ -4023,6 +4104,9 @@ async def update_pass_through_endpoints(
auth=updated_endpoint.auth,
timeout=updated_endpoint.timeout,
)
InitPassThroughEndpointHelpers.drop_llm_api_routes(
replaced_llm_api_routes - InitPassThroughEndpointHelpers.auth_enforced_llm_api_routes()
)
return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else [])

View file

@ -1,6 +1,7 @@
import json
import os
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from fastapi import HTTPException
@ -11,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
LiteLLM_ManagedVectorStore,
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.secret_managers.main import (
GOOGLE_OIDC_SECRET_PREFIX,
UnresolvableSecretReference,
@ -25,12 +27,23 @@ if TYPE_CHECKING:
from litellm.router import Router
@dataclass(frozen=True, slots=True)
class _PassThroughCredentials:
api_key: str
api_base: str | None
def _get_proxy_llm_router() -> "Router | None":
from litellm.proxy.proxy_server import llm_router
return llm_router
def _is_team_deployment(deployment: Mapping[str, object]) -> bool:
model_info: Final = deployment.get("model_info")
return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id"))
def _get_str_value(values: dict[str, object] | None, key: str) -> str | None:
value: Final = values.get(key) if values is not None else None
return value if isinstance(value, str) else None
@ -84,12 +97,13 @@ class PassthroughEndpointRouter:
custom_llm_provider: str,
region_name: str | None,
) -> str | None:
deployment_api_key: Final = self._get_deployment_api_key(
deployment_credentials: Final = self._get_deployment_credentials(
custom_llm_provider=custom_llm_provider,
region_name=region_name,
include_team_deployments=True,
)
if deployment_api_key is not None:
return _resolve_oidc_reference(deployment_api_key)
if deployment_credentials is not None:
return _resolve_oidc_reference(deployment_credentials.api_key)
verbose_router_logger.debug(
"No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider
)
@ -98,29 +112,56 @@ class PassthroughEndpointRouter:
)
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None:
deployment_credentials: Final = self._get_deployment_credentials(
custom_llm_provider=custom_llm_provider,
region_name=None,
include_team_deployments=False,
)
api_key: Final = _resolve_oidc_reference(
deployment_credentials.api_key
if deployment_credentials is not None
else get_secret_str(
self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider)
)
)
if not api_key:
return None
return _PassThroughCredentials(
api_key=api_key,
api_base=deployment_credentials.api_base if deployment_credentials is not None else None,
)
def has_credentials(self, custom_llm_provider: str) -> bool:
if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None:
if (
self._get_deployment_credentials(
custom_llm_provider=custom_llm_provider, region_name=None, include_team_deployments=True
)
is not None
):
return True
env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
custom_llm_provider=custom_llm_provider,
)
return bool(os.environ.get(env_variable_name))
def _get_deployment_api_key(
def _get_deployment_credentials(
self,
custom_llm_provider: str,
region_name: str | None,
) -> str | None:
include_team_deployments: bool,
) -> _PassThroughCredentials | None:
llm_router: Final = self.llm_router_getter()
if llm_router is None:
return None
deployments: Final = llm_router.registry().deployments() or ()
return next(
(
api_key
credentials
for deployment in deployments
if (
api_key := self._resolve_matching_deployment_api_key(
if (include_team_deployments or not _is_team_deployment(deployment))
and (
credentials := self._resolve_matching_deployment_credentials(
litellm_params=deployment["litellm_params"],
custom_llm_provider=custom_llm_provider,
region_name=region_name,
@ -131,12 +172,12 @@ class PassthroughEndpointRouter:
None,
)
def _resolve_matching_deployment_api_key(
def _resolve_matching_deployment_credentials(
self,
litellm_params: LiteLLMParamsTypedDict,
custom_llm_provider: str,
region_name: str | None,
) -> str | None:
) -> _PassThroughCredentials | None:
if litellm_params.get("use_in_pass_through") is not True:
return None
if self._get_deployment_provider(litellm_params) != custom_llm_provider:
@ -152,7 +193,8 @@ class PassthroughEndpointRouter:
)
if deployment_region != region_name:
return None
return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None
def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None:
model: Final = litellm_params.get("model")

View file

@ -375,7 +375,7 @@ class PassThroughEndpointLogging:
kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract
elif (
self.is_typesafe_route(custom_llm_provider)
or custom_llm_provider in ("laya", "bespoke")
or custom_llm_provider in ("eu_jev", "laya", "bespoke")
or self.is_openrouter_decisions_route(url_route, custom_llm_provider)
):
from .llm_provider_handlers.typesafe_passthrough_logging_handler import (

View file

@ -201,7 +201,6 @@ from litellm.types.utils import (
CountContextManagement,
ModelResponse,
ModelResponseStream,
StreamingChoices,
TextCompletionResponse,
TokenCountMessagesFormat,
TokenCountResponse,
@ -384,6 +383,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
SensitiveDataMasker,
mask_sensitive_keys,
)
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.llms.anthropic.messages.convertible_blocks import (
deployment_drops_params,
drop_non_string_tool_result_texts,
@ -529,9 +529,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
)
from litellm.proxy.common_utils.openai_error_payload import (
LITELLM_CALL_ID_HEADER,
attribute_of,
headers_with_litellm_call_id,
litellm_call_id_headers,
openai_error_param,
proxy_exception_for,
route_failure_proxy_exception,
with_litellm_call_id,
)
from litellm.proxy.common_utils.periodic_reload_schedule import (
@ -5190,7 +5193,7 @@ def _build_redis_usage_cache_from_environment() -> RedisCache | None:
Lets the proxy's coordination Redis (cross-pod tpm/rpm rate limits, spend
tracking, pod lock manager) run when the response-cache backend is not a
plain Redis KV cache (e.g. a semantic cache, disk, or s3).
plain Redis KV cache (e.g. a semantic cache or s3).
Returns None when the environment carries no connection target (host, url,
cluster nodes, or sentinel nodes).
@ -9898,11 +9901,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe
# Including it in the SSE response leaks internal details to clients.
error_msg: Final = str(e)
proxy_exception: Final = ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", 500),
proxy_exception: Final = route_failure_proxy_exception(
e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR
)
error_returned: Final = json.dumps({"error": proxy_exception.to_dict()})
yield f"data: {error_returned}\n\n"
@ -10214,29 +10214,6 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
def _is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
if chunk.provider_specific_fields is not None:
return False
return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or [])
def _is_empty_streaming_choice(choice: StreamingChoices) -> bool:
if choice.finish_reason is not None:
return False
if getattr(choice, "logprobs", None) is not None:
return False
delta: Final = choice.delta if hasattr(choice, "delta") else None
if delta is None:
return True
return _all_values_none(delta.model_dump().values())
def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)
async def _apply_streaming_chunk_hooks(
*,
chunk: Any,
@ -10616,7 +10593,7 @@ async def async_data_generator(
fallback_model_from_metadata=fallback_model_from_metadata,
)
if strip_stream_usage and _is_injected_stream_usage_artifact(chunk):
if strip_stream_usage and is_injected_stream_usage_artifact(chunk):
if pending_fallback_event:
yield _format_fallback_metadata_sse_event(
fallback_model=fallback_model_from_metadata,
@ -10735,11 +10712,10 @@ async def async_data_generator(
{
"error": sse_error_payload(client_error)[1]
if isinstance(client_error, HTTPException)
else ProxyException(
message=getattr(client_error, "message", str(client_error)),
type=getattr(client_error, "type", "None"),
param=getattr(client_error, "param", "None"),
code=getattr(client_error, "status_code", 500),
else route_failure_proxy_exception(
client_error,
str(attribute_of(client_error, "message", client_error)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
).to_dict()
},
default=str,
@ -10997,7 +10973,7 @@ class ProxyStartupEvent:
"""
Builds a standalone Redis cache from REDIS_* environment variables so
use_redis_transaction_buffer can run when the proxy cache backend is not
Redis (e.g. disk, s3).
Redis (e.g. s3).
Returns None when the buffer is disabled, or when no Redis host or url
is set in the environment.
@ -12360,8 +12336,10 @@ async def model_list(
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
provider's model catalog. When present, the response is Codex's own
catalog shape (`{"models": [...]}`) built from the same listing, with
each model's `model_info.service_tiers` as its service tiers; absent,
the OpenAI shape below is returned
each model's `model_info.service_tiers` as its service tiers and, for a
model that Codex release does not know, the reasoning efforts
`/model_group/info` reports that it can parse; absent, the OpenAI shape
below is returned
- include_metadata: Include additional metadata in the response with fallback information
- fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
Defaults to "general" when include_metadata=true
@ -12520,8 +12498,13 @@ async def model_list(
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
if client_version is not None:
return Response(
content=codex_model_list_body(
admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id
content=await asyncio.to_thread(
codex_model_list_body,
admin_listing,
admin_entries,
llm_router,
team_id or user_api_key_dict.team_id,
client_version=client_version,
),
media_type="application/json",
)
@ -12596,7 +12579,15 @@ async def model_list(
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
if client_version is not None:
return Response(
content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels),
content=await asyncio.to_thread(
codex_model_list_body,
listing,
entries,
llm_router,
team_id or user_api_key_dict.team_id,
labels,
client_version=client_version,
),
media_type="application/json",
)
@ -13104,14 +13095,12 @@ async def completion(
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)
@ -13332,22 +13321,18 @@ async def moderations(
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e)),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_400_BAD_REQUEST,
headers=litellm_call_id_headers(litellm_call_id),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
code=getattr(e, "status_code", 500),
)
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
)
async def _audio_speech_chunk_generator(
@ -13485,13 +13470,12 @@ async def audio_speech(
detail=e.detail,
headers=headers_with_litellm_call_id(e.headers, litellm_call_id),
)
raise ProxyException(
message=getattr(e, "message", f"{e}"),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)
@ -13631,23 +13615,19 @@ async def audio_transcriptions(
)
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "message", str(e.detail)),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e.detail)),
status.HTTP_400_BAD_REQUEST,
headers=litellm_call_id_headers(litellm_call_id),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
else:
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
code=getattr(e, "status_code", 500),
)
raise route_failure_proxy_exception(
e,
str(attribute_of(e, "message", e)),
status.HTTP_500_INTERNAL_SERVER_ERROR,
headers=litellm_call_id_headers(litellm_call_id),
openai_code=getattr(e, "code", None),
)
######################################################################
@ -17247,7 +17227,7 @@ async def model_streaming_metrics(
raise ProxyException(
message=CommonProxyErrors.db_not_connected_error.value,
type="internal_error",
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -17381,7 +17361,7 @@ async def model_metrics(
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
startTime = startTime or datetime.now() - timedelta(days=DAYS_IN_A_MONTH)
@ -17496,7 +17476,7 @@ async def model_metrics_slow_responses(
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if api_key is None or api_key == "undefined":
@ -17585,7 +17565,7 @@ async def model_metrics_exceptions(
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -18069,6 +18049,7 @@ def get_model_group_info(
registry.deployments(
resolve_model_group_alias(registry.aliases(), model) or model,
source="routed_group",
team_id=team_id,
)
),
}
@ -18798,7 +18779,7 @@ async def login_v2(request: Request):
raise ProxyException(
message=getattr(e, "detail", str(e)),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
else:
@ -18806,7 +18787,7 @@ async def login_v2(request: Request):
raise ProxyException(
message=error_msg,
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -18874,7 +18855,7 @@ async def login_v3(request: Request):
raise ProxyException(
message=getattr(e, "detail", str(e)),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
else:
@ -18882,7 +18863,7 @@ async def login_v3(request: Request):
raise ProxyException(
message=error_msg,
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -18947,7 +18928,7 @@ async def login_v3_exchange(request: Request):
raise ProxyException(
message=str(e),
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)

View file

@ -1553,6 +1553,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "EU_JEV",
"provider_display_name": "eu/jev",
"litellm_provider": "eu_jev",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://eu-jev.com",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "eu_jev/eu-jeff-latest"
},
{
"provider": "FalAI",
"provider_display_name": "Fal AI",

View file

@ -50,6 +50,7 @@ from litellm.proxy.auth.authorization_dependencies import (
LogTeamLookupDependency,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.spend_tracking.spend_capture_rate import (
ProviderBillingCredentialMissing,
@ -636,7 +637,7 @@ async def view_spend_tags(
raise ProxyException(
message=getattr(e, "detail", f"/spend/tags Error({e})"),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -644,7 +645,7 @@ async def view_spend_tags(
raise ProxyException(
message="/spend/tags Error" + str(e),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -2142,7 +2143,7 @@ async def global_get_all_tag_names():
raise ProxyException(
message=getattr(e, "detail", f"/spend/all_tag_names Error({e})"),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -2150,7 +2151,7 @@ async def global_get_all_tag_names():
raise ProxyException(
message="/spend/all_tag_names Error" + str(e),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -2224,7 +2225,7 @@ async def global_view_spend_tags(
raise ProxyException(
message=getattr(e, "detail", f"/spend/tags Error({error_str})"),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -2232,7 +2233,7 @@ async def global_view_spend_tags(
raise ProxyException(
message="/spend/tags Error" + error_str,
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -2447,9 +2448,9 @@ async def calculate_spend(request: SpendCalculateRequest):
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", str(e)),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
param=openai_error_param(e),
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
)
if isinstance(e, litellm.exceptions.ModelNotMappedError):
raise ProxyException(
@ -2461,9 +2462,9 @@ async def calculate_spend(request: SpendCalculateRequest):
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"),
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", 500),
type=openai_error_type(e, error_status_code(e, 500)),
param=openai_error_param(e),
code=error_status_code(e, 500),
)
@ -2633,7 +2634,7 @@ async def ui_view_spend_logs(
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -2700,7 +2701,7 @@ async def ui_view_spend_logs(
raise ProxyException(
message="Start date and end date are required",
type="bad_request",
param="None",
param=None,
code=status.HTTP_400_BAD_REQUEST,
)
formats: Final = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"]
@ -3681,7 +3682,7 @@ async def view_spend_logs(
raise ProxyException(
message=getattr(e, "detail", f"/spend/logs Error({e})"),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -3689,7 +3690,7 @@ async def view_spend_logs(
raise ProxyException(
message="/spend/logs Error" + str(e),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -3716,7 +3717,7 @@ async def global_spend_reset():
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -3747,7 +3748,7 @@ async def global_spend_refresh():
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
@ -3818,7 +3819,7 @@ async def global_spend_for_internal_user(
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
try:
@ -3879,7 +3880,7 @@ async def global_spend_logs(
raise ProxyException(
message="Prisma Client is not initialized",
type="internal_error",
param="None",
param=None,
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -3923,7 +3924,7 @@ async def global_spend_logs(
raise ProxyException(
message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -3931,7 +3932,7 @@ async def global_spend_logs(
raise ProxyException(
message="/global/spend/logs Error" + error_str,
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
@ -3971,7 +3972,7 @@ async def global_spend():
raise ProxyException(
message=getattr(e, "detail", f"/global/spend Error({error_str})"),
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
@ -3979,7 +3980,7 @@ async def global_spend():
raise ProxyException(
message="/global/spend Error" + error_str,
type="internal_error",
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)

View file

@ -1617,7 +1617,7 @@ class Router(DeprecatedRouterMembers):
self._core_deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self._core_deployment_latency_map = {}
### CACHING ###
cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache
cache_type: Literal["local", "redis", "redis-semantic", "s3"] = "local" # default to an in-memory cache
redis_cache = None
cache_config: Final[dict[str, Any]] = {}
@ -12856,7 +12856,10 @@ class Router(DeprecatedRouterMembers):
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
reasoning_efforts_initialized = False
reasoning_efforts_unknown = False
model_list: Final = self._core_get_model_list_of_routed_group(model_group)
serves_team_deployments: Final = team_id is not None and bool(
self._team_deployments_named(team_id, model_group)
)
model_list: Final = self._core_get_model_list_of_routed_group(model_group, team_id)
model_group_description: Final[str | None] = next(
(d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)),
None,
@ -12864,7 +12867,8 @@ class Router(DeprecatedRouterMembers):
for model in model_list:
is_match = False
if (
"model_name" in model
serves_team_deployments
or "model_name" in model
and model["model_name"] == model_group
or "model_name" in model
and self._core_pattern_router.route(model_group) is not None
@ -14213,17 +14217,22 @@ class Router(DeprecatedRouterMembers):
lambda name: self._core_wildcard_route_deployments(name, team_id),
)
def _core_get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]:
def _core_get_model_list_of_routed_group(
self, model_group: str, team_id: str | None = None
) -> list[DeploymentTypedDict]:
"""
The deployments a request the router has already resolved to model_group is
served from: the ones named model_group, the routing group of that name, or
the wildcard route matching it when neither exists.
served from: team_id's own deployments under that public name when the team has
any, else the ones named model_group, the routing group of that name, or the
wildcard route matching it when neither exists.
Unlike get_model_list, model_group's own model_group_alias entry is not
followed. The router resolves an alias exactly once, so a group reached as
an alias target is served by its own deployments, never by a second hop:
in the chain X -> T -> U a request to X is served from T's deployments.
"""
if team_id is not None and self._team_deployments_named(team_id, model_group):
return self._core_get_all_deployments(model_name=model_group, team_id=team_id)
own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns
list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group)
)

View file

@ -419,7 +419,7 @@ class RouterRegistry:
if source == "named":
return r._core_deployments_named(model_name)
if source == "routed_group":
return r._core_get_model_list_of_routed_group(model_name)
return r._core_get_model_list_of_routed_group(model_name, team_id)
if source == "wildcard":
return r._core_wildcard_route_deployments(model_name, team_id)
if source == "routable":

View file

@ -27,20 +27,22 @@ it above.
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias, get_args
from typing_extensions import assert_never
from typing import Final, TypeAlias, get_args
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params
from litellm.litellm_core_utils.reasoning_effort_strength import (
REASONING_EFFORT_STRENGTH_ORDER,
ReasoningEffortRounding,
nearest_declared_reasoning_effort,
)
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.openai import REASONING_EFFORT
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
RequestedReasoningEffort: TypeAlias = str | Mapping[str, object] | None
REASONING_EFFORT_ADVERTISEMENT_ORDER: Final[tuple[str, ...]] = get_args(REASONING_EFFORT)
@ -115,35 +117,6 @@ def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) -
return declared_reasoning_efforts(entry)
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
_STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)})
def nearest_declared_reasoning_effort(
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
) -> str:
"""Rounding up takes the weakest declared level at least as strong as the request, else the
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
a model that cannot turn it off says so itself. A level outside the strength order is likewise
returned as is for upstream to judge."""
ranked: Final = sorted(
(effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort]
)
if requested in ranked or requested not in _STRENGTH_RANK or not ranked:
return requested
strength: Final = _STRENGTH_RANK[requested]
match rounding:
case "up":
return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= strength), ranked[-1])
case "down":
return next((effort for effort in reversed(ranked) if _STRENGTH_RANK[effort] <= strength), ranked[0])
case _:
assert_never(rounding)
_ROUNDING_KEY: Final = "reasoning_effort_rounding"
_DEFAULT_EFFORT: Final = "default"
_EFFORT_KEY: Final = "effort"
@ -163,7 +136,7 @@ def deployment_reasoning_effort_levels(deployment_model_info: object) -> tuple[s
def deployment_rounds_reasoning_effort(deployment_model_info: object) -> bool:
declared: Final = deployment_reasoning_effort_levels(deployment_model_info)
return declared is not None and any(level in _STRENGTH_RANK for level in declared)
return declared is not None and any(level in REASONING_EFFORT_STRENGTH_ORDER for level in declared)
def with_adaptive_thinking_as_effort(

View file

@ -98,6 +98,8 @@ class Feature(str, Enum):
"""Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature."""
TOOL_CUSTOM_FIELD = "tool_custom_field"
"""Tools carrying a non-spec custom object, which Python drops or hoists for the provider."""
STRICT_TOOLS_RETRY = "strict_tools_retry"
"""Strict tools under drop_params, which Python sends again without strict if Claude rejects their limits."""
CLIENT_HEADERS = "client_headers"
"""Proxy-forwarded headers that Python merges into extra_headers."""
DEPLOYMENT_HOOK = "deployment_hook"
@ -179,6 +181,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.STREAM_COST_INJECTION,
Feature.THINKING_SIGNATURE_RETRY,
Feature.TOOL_CUSTOM_FIELD,
Feature.STRICT_TOOLS_RETRY,
}
RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {

View file

@ -5,6 +5,7 @@ from __future__ import annotations
from typing import Final
import litellm
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
@ -48,6 +49,13 @@ def tool_custom_field(request: NativeCall) -> bool:
return any("custom" in (optional_mapping(tool) or {}) for tool in tools)
def strict_tools_retry(request: NativeCall) -> bool:
tools: Final = optional_sequence(request.resolved.get("tools")) or ()
return drops_unsupported_params(request.resolved.get("drop_params")) and any(
(optional_mapping(tool) or {}).get("strict") is True for tool in tools
)
def stream_cost_injection(request: NativeCall) -> bool:
"""Python adds response cost to streaming usage when the global setting is truthy."""
return streaming(request) and bool(litellm.include_cost_in_streaming_usage)

View file

@ -1,6 +1,7 @@
from typing import Final
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
class TextCompletionStreamWrapper:
@ -10,11 +11,18 @@ class TextCompletionStreamWrapper:
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
usage_injected_by_proxy: bool = False,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
self.usage_injected_by_proxy: Final = usage_injected_by_proxy
self.include_usage: Final = (
not usage_injected_by_proxy
and stream_options is not None
and stream_options.get("include_usage", False) is True
)
def __iter__(self):
return self
@ -22,7 +30,7 @@ class TextCompletionStreamWrapper:
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
def convert_to_text_completion_object(self, chunk: ModelResponse | ModelResponseStream):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
@ -43,8 +51,7 @@ class TextCompletionStreamWrapper:
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
if self.include_usage:
response.usage = chunk.get("usage", None)
else:
response.usage = None
@ -53,6 +60,9 @@ class TextCompletionStreamWrapper:
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def _is_injected_usage_artifact(self, chunk: ModelResponseStream) -> bool:
return self.usage_injected_by_proxy and is_injected_stream_usage_artifact(chunk)
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
@ -60,8 +70,9 @@ class TextCompletionStreamWrapper:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
continue
return self.convert_to_text_completion_object(chunk=chunk)
raise StopIteration
except StopIteration:
raise StopIteration
@ -81,8 +92,9 @@ class TextCompletionStreamWrapper:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
continue
return self.convert_to_text_completion_object(chunk=chunk)
raise StopIteration
except StopIteration:
raise StopAsyncIteration

View file

@ -13,7 +13,6 @@ class LiteLLMCacheType(str, Enum):
REDIS_SEMANTIC = "redis-semantic"
VALKEY_SEMANTIC = "valkey-semantic"
S3 = "s3"
DISK = "disk"
QDRANT_SEMANTIC = "qdrant-semantic"
AZURE_BLOB = "azure-blob"
GCS = "gcs"

View file

@ -204,9 +204,11 @@ class MCPServer(LiteLLMBaseModel):
access_groups: list[str] | None = None
allow_all_keys: bool = False
available_on_public_internet: bool = True
# Legacy opt-in to upstream-delegated authentication for ``oauth2``
# servers. LiteLLM admission still applies; use ``oauth_delegate`` for the
# supported client-forwarded OAuth flow.
# Deprecated opt-in to upstream-delegated authentication for ``oauth2``
# servers. A bare ``Authorization`` that is not a LiteLLM credential is the
# upstream token and is forwarded; see
# ``MCPRequestHandler._admit_legacy_delegate_bearer``. New servers should use
# ``true_passthrough`` or ``oauth_delegate``.
#
# Honored only for ``auth_type == oauth2``; ignored for any other
# ``auth_type``. OAuth pass-through for non-oauth2 servers

View file

@ -0,0 +1,100 @@
from collections.abc import Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from pydantic import ConfigDict, Field
from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer
from litellm.types.llms.base import LiteLLMPydanticObjectBase
EuJevEntry: TypeAlias = Annotated[
DecisionsJSON | None,
Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"),
]
class EuJevChoiceQuestion(LiteLLMPydanticObjectBase):
type: Literal["choice"]
instructions: EuJevEntry
criteria: Annotated[
Mapping[str, EuJevEntry],
Field(
min_length=2,
max_length=64,
description="Two to 64 options, option name to description. A null description uses the option name",
),
]
model_config = ConfigDict(frozen=True)
class EuJevNoulQuestion(LiteLLMPydanticObjectBase):
type: Literal["noul"]
instructions: EuJevEntry
criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = (
None
)
model_config = ConfigDict(frozen=True)
class EuJevScoreQuestion(LiteLLMPydanticObjectBase):
type: Literal["score"]
instructions: EuJevEntry
criteria: Annotated[
Sequence[EuJevEntry],
Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"),
]
model_config = ConfigDict(frozen=True)
EuJevQuestion: TypeAlias = Annotated[
EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion,
Field(discriminator="type"),
]
class EuJevDecisionRequest(LiteLLMPydanticObjectBase):
model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = (
"eu-jeff-latest"
)
state: Annotated[
Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object],
Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"),
]
questions: Annotated[
Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion],
Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"),
]
model_config = ConfigDict(frozen=True)
EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")]
class EuJevUsage(LiteLLMPydanticObjectBase):
input_tokens: Annotated[int, Field(ge=0)]
output_tokens: Annotated[int, Field(ge=0)]
model_config = ConfigDict(frozen=True)
class EuJevMeta(LiteLLMPydanticObjectBase):
request_id: str
mode: Literal["live"]
latency_ms: Annotated[
int, Field(description="Server-side time for token quoting, credit reservation and inference")
]
cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None
model_config = ConfigDict(frozen=True)
class EuJevDecisionResponse(LiteLLMPydanticObjectBase):
model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")]
answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")]
usage: EuJevUsage
meta: EuJevMeta
model_config = ConfigDict(frozen=True)

View file

@ -169,6 +169,10 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
# above; a forged value could at most narrow, but the stripping keeps the field's provenance
# single-owner so its meaning stays trustworthy.
mcp_session_resource_server_ids: tuple[str, ...] | None = Field(default=None, exclude=True)
# Set only by MCP admission when a legacy delegate_auth_to_upstream caller's Authorization is the
# upstream token rather than a LiteLLM credential, so egress forwards it. Stripped from validated
# input like the markers above: a forged value would forward a LiteLLM credential upstream.
mcp_forwards_caller_authorization: bool = Field(default=False, exclude=True)
mcp_toolset_id: str | None = Field(default=None, exclude=True)
authenticated_by_custom_auth: bool = Field(default=False, exclude=True)
via_virtual_key: bool = Field(
@ -230,6 +234,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
values.pop("mcp_explicit_grants_only", None)
values.pop("mcp_source_team_rpm_limits", None)
values.pop("mcp_session_resource_server_ids", None)
values.pop("mcp_forwards_caller_authorization", None)
values.pop("mcp_toolset_id", None)
values.pop("via_virtual_key", None)
values.pop("authenticated_by_custom_auth", None)

View file

@ -4,9 +4,11 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
import functools
import re
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
@ -989,9 +991,10 @@ class AlertingConfig(LiteLLMBaseModel):
alerting_threshold: float | None = 300
@functools.cache
def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
"""Resolve a class's annotations, keeping each resolved annotation opaque."""
return get_type_hints(model_class)
return MappingProxyType(get_type_hints(model_class))
class ModelGroupInfo(LiteLLMBaseModel):

View file

@ -4489,6 +4489,7 @@ class LlmProviders(str, Enum):
STRANDS_DECIDER = "strands_decider"
LAYA = "laya"
BESPOKE = "bespoke"
EU_JEV = "eu_jev"
MISTRAL = "mistral"
MILVUS = "milvus"
GROQ = "groq"

View file

@ -9968,11 +9968,12 @@ class ProviderConfigManager:
return litellm.LayaDecisionsConfig()
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI):
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV):
configs: Final = {
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig,
LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig,
}
return configs[provider]()
return None
@ -10923,19 +10924,8 @@ class ProviderConfigManager:
return None
@staticmethod
def get_provider_image_edit_config(
model: str,
provider: LlmProviders,
) -> BaseImageEditConfig | None:
if LlmProviders.OPENAI == provider:
from litellm.llms.openai.image_edit import OpenAIImageEditConfig
return OpenAIImageEditConfig()
elif LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config
return get_hosted_vllm_image_edit_config(model=model)
elif LlmProviders.AZURE == provider:
def _get_model_independent_image_edit_config(provider: LlmProviders) -> BaseImageEditConfig | None:
if LlmProviders.AZURE == provider:
from litellm.llms.azure.image_edit.transformation import (
AzureImageEditConfig,
)
@ -10953,6 +10943,36 @@ class ProviderConfigManager:
)
return BlackForestLabsImageEditConfig()
elif LlmProviders.LITELLM_PROXY == provider:
from litellm.llms.litellm_proxy.image_edit.transformation import (
LiteLLMProxyImageEditConfig,
)
return LiteLLMProxyImageEditConfig()
elif LlmProviders.DASHSCOPE == provider:
from litellm.llms.dashscope.image_edit.transformation import (
DashScopeImageEditConfig,
)
return DashScopeImageEditConfig()
return None
@staticmethod
def get_provider_image_edit_config(
model: str,
provider: LlmProviders,
) -> BaseImageEditConfig | None:
model_independent_config: Final = ProviderConfigManager._get_model_independent_image_edit_config(provider)
if model_independent_config is not None:
return model_independent_config
if LlmProviders.OPENAI == provider:
from litellm.llms.openai.image_edit import OpenAIImageEditConfig
return OpenAIImageEditConfig()
elif LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config
return get_hosted_vllm_image_edit_config(model=model)
elif LlmProviders.FAL_AI == provider:
from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config
@ -10965,12 +10985,6 @@ class ProviderConfigManager:
from litellm.llms.gemini.image_edit import get_gemini_image_edit_config
return get_gemini_image_edit_config(model)
elif LlmProviders.LITELLM_PROXY == provider:
from litellm.llms.litellm_proxy.image_edit.transformation import (
LiteLLMProxyImageEditConfig,
)
return LiteLLMProxyImageEditConfig()
elif LlmProviders.VERTEX_AI == provider:
from litellm.llms.vertex_ai.image_edit import (
get_vertex_ai_image_edit_config,
@ -11192,6 +11206,12 @@ class ProviderConfigManager:
return AWSPollyTextToSpeechConfig()
elif litellm.LlmProviders.EDENAI == provider:
return litellm.EdenAITextToSpeechConfig()
elif litellm.LlmProviders.DASHSCOPE == provider:
from litellm.llms.dashscope.text_to_speech.transformation import (
DashScopeTextToSpeechConfig,
)
return DashScopeTextToSpeechConfig()
return None
@staticmethod

View file

@ -18346,25 +18346,31 @@
"dashscope/qwen-image-2.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.035,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-2.0-pro": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.075,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.03,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0-pro": {
@ -18372,7 +18378,54 @@
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen3-tts-flash": {
"input_cost_per_character": 1e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-flash": {
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 1.87e-06,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-plus": {
"input_cost_per_character": 3e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-plus": {
"input_cost_per_character": 2e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-flash": {
"input_cost_per_character": 1.5e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/wan3.0-video": {
@ -72330,6 +72383,28 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"eu_jev/eu-jeff-1.0.0": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,

View file

@ -78,11 +78,6 @@ id = "GHSA-r4j5-9xpp-3985"
ignoreUntil = 2026-11-27
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
[[IgnoredVulns]]
id = "GHSA-w8v5-vhqr-4h9v"
ignoreUntil = 2026-11-27
reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists"
[[IgnoredVulns]]
id = "GHSA-45c3-73f7-pv4m"
ignoreUntil = 2026-11-27

View file

@ -11,7 +11,7 @@ authors = [
]
dependencies = [
"fastuuid>=0.14.0,<1.0; python_version < '3.15'",
"filelock>=3.16.1,<4.0",
"filelock>=3.16.1,<5",
"httpx[http2]>=0.28.0,<1.0",
"openai>=2.20.0,<4.0.0",
"python-dateutil>=2.8.2,<3.0",

Some files were not shown because too many files have changed in this diff Show more