mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge origin/main into litellm_openai_error_type_from_status
This commit is contained in:
commit
cfec5a12e4
225 changed files with 10020 additions and 1434 deletions
2
.github/workflows/test-integration.yml
vendored
2
.github/workflows/test-integration.yml
vendored
|
|
@ -51,7 +51,7 @@ jobs:
|
|||
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
|
||||
env:
|
||||
UV_PYTHON: "3.12"
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"32","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
|
|
|
|||
6
.github/workflows/test-linting.yml
vendored
6
.github/workflows/test-linting.yml
vendored
|
|
@ -630,6 +630,12 @@ jobs:
|
|||
- name: check_migrations_lock_timeout
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
|
||||
|
||||
- name: check_no_none_string_error_fields
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
|
||||
|
||||
- name: test_check_no_none_string_error_fields
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
|
|
|
|||
8
.github/workflows/test-unit.yml
vendored
8
.github/workflows/test-unit.yml
vendored
|
|
@ -1003,9 +1003,9 @@ jobs:
|
|||
run: |
|
||||
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
||||
|
|
@ -1451,9 +1451,9 @@ jobs:
|
|||
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
|
||||
run: |
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
fi
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -153,7 +153,7 @@ lint-install:
|
|||
# the same set CI's ':(glob)litellm/**/*.py' selects.
|
||||
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
@base_ref=$$($(RESOLVE_BASE)) && \
|
||||
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
|
||||
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
|
||||
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
|
||||
if [ -z "$$files" ]; then \
|
||||
echo "No changed litellm Python files to format-check."; \
|
||||
|
|
|
|||
|
|
@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
|
|||
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|
||||
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
|
||||
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
|
||||
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
|
||||
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.80"
|
||||
version = "0.1.81"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.80"
|
||||
version = "0.1.81"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
|||
"/test",
|
||||
"/debug/memory/summary",
|
||||
"/api/event_logging/batch",
|
||||
"/eu_jev/v1/systemone",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.111"
|
||||
version = "0.4.112"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.111"
|
||||
version = "0.4.112"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -1783,6 +1783,9 @@ if TYPE_CHECKING:
|
|||
from .llms.bespoke.decisions.transformation import (
|
||||
BespokeDecisionsConfig as BespokeDecisionsConfig,
|
||||
)
|
||||
from .llms.eu_jev.decisions.transformation import (
|
||||
EuJevDecisionsConfig as EuJevDecisionsConfig,
|
||||
)
|
||||
from .llms.openai.decisions.transformation import (
|
||||
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"OllamaDecisionsConfig",
|
||||
"LayaDecisionsConfig",
|
||||
"BespokeDecisionsConfig",
|
||||
"EuJevDecisionsConfig",
|
||||
"OpenAIDecisionsConfig",
|
||||
"AzureDecisionsConfig",
|
||||
"AzureAIDecisionsConfig",
|
||||
|
|
@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
|
||||
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
|
||||
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
|
||||
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
|
||||
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
|
||||
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
|
||||
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
|
||||
|
|
|
|||
|
|
@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
|
|||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
REDACTED,
|
||||
python_redact_string,
|
||||
python_redact_structured_value,
|
||||
redact_internal_details,
|
||||
redact_structured_value,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
redact_string as redact_secret_string,
|
||||
|
|
@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
|
|||
return redact_internal_details(value)
|
||||
|
||||
|
||||
_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_CLIENT_ITEMS: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _redacted_client_field(key: str | None, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return redact_internal_details(redact_structured_value(key, value))
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
child_key: _redacted_client_field(child_key, child)
|
||||
for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
|
||||
}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
|
||||
if not isinstance(value, (int, float)):
|
||||
return value
|
||||
text: Final = str(value)
|
||||
return value if redact_structured_value(key, text) == text else REDACTED
|
||||
|
||||
|
||||
def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return fields
|
||||
return _redacted_client_field(None, dict(fields))
|
||||
|
||||
|
||||
def _substituted_color_message(record: logging.LogRecord) -> str | None:
|
||||
"""Render a record's ``color_message`` against its args, or None if absent.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,9 @@ The following caching mechanisms are supported:
|
|||
2. **RedisSemanticCache**
|
||||
3. **QdrantSemanticCache**
|
||||
4. **InMemoryCache**
|
||||
5. **DiskCache**
|
||||
6. **S3Cache**
|
||||
7. **AzureBlobCache**
|
||||
8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
|
||||
5. **S3Cache**
|
||||
6. **AzureBlobCache**
|
||||
7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
|
||||
|
||||
## Folder Structure
|
||||
|
||||
|
|
@ -20,7 +19,6 @@ litellm/caching/
|
|||
├── base_cache.py
|
||||
├── caching.py
|
||||
├── caching_handler.py
|
||||
├── disk_cache.py
|
||||
├── dual_cache.py
|
||||
├── in_memory_cache.py
|
||||
├── qdrant_semantic_cache.py
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from .azure_blob_cache import AzureBlobCache
|
||||
from .caching import Cache, LiteLLMCacheType
|
||||
from .disk_cache import DiskCache
|
||||
from .dual_cache import DualCache
|
||||
from .gcs_cache import GCSCache
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
|
|||
|
||||
from .azure_blob_cache import AzureBlobCache
|
||||
from .base_cache import BaseCache
|
||||
from .disk_cache import DiskCache
|
||||
from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching
|
||||
from .gcs_cache import GCSCache
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
|
@ -153,7 +152,7 @@ class Cache:
|
|||
Initializes the cache based on the given type.
|
||||
|
||||
Args:
|
||||
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
|
||||
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
|
||||
|
||||
# Redis Cache Args
|
||||
host (str, optional): The host address for the Redis cache. Required if type is "redis".
|
||||
|
|
@ -174,9 +173,6 @@ class Cache:
|
|||
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
|
||||
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
|
||||
|
||||
# Disk Cache Args
|
||||
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
|
||||
|
||||
# S3 Cache Args
|
||||
s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
|
||||
s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
|
||||
|
|
@ -306,8 +302,11 @@ class Cache:
|
|||
account_url=azure_account_url,
|
||||
container=azure_blob_container,
|
||||
)
|
||||
elif type == LiteLLMCacheType.DISK:
|
||||
self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
|
||||
else:
|
||||
supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
|
||||
raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
|
||||
if disk_cache_dir is not None:
|
||||
verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
|
||||
if "cache" not in litellm.input_callback:
|
||||
litellm.input_callback.append("cache")
|
||||
if "cache" not in litellm.success_callback:
|
||||
|
|
@ -1044,7 +1043,7 @@ def enable_cache(
|
|||
Enable cache with the specified configuration.
|
||||
|
||||
Args:
|
||||
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
|
||||
type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
|
||||
host (Optional[str]): The host address of the cache server. Defaults to None.
|
||||
port (Optional[str]): The port number of the cache server. Defaults to None.
|
||||
password (Optional[str]): The password for the cache server. Defaults to None.
|
||||
|
|
@ -1056,7 +1055,7 @@ def enable_cache(
|
|||
None
|
||||
|
||||
Raises:
|
||||
None
|
||||
ValueError: If no cache is set yet and the cache type is unsupported.
|
||||
"""
|
||||
print_verbose("LiteLLM: Enabling Cache")
|
||||
if "cache" not in litellm.input_callback:
|
||||
|
|
@ -1091,7 +1090,7 @@ def update_cache(
|
|||
Update the cache for LiteLLM.
|
||||
|
||||
Args:
|
||||
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
|
||||
type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
|
||||
host (Optional[str]): The host of the cache. Defaults to None.
|
||||
port (Optional[str]): The port of the cache. Defaults to None.
|
||||
password (Optional[str]): The password for the cache. Defaults to None.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ This file is a wrapper around caching.py
|
|||
|
||||
This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
|
||||
|
||||
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
|
||||
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
|
||||
|
||||
In each method it will call the appropriate method from caching.py
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,92 +0,0 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
||||
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class DiskCache(BaseCache):
|
||||
def __init__(self, disk_cache_dir: str | None = None):
|
||||
try:
|
||||
import diskcache as dc
|
||||
except ModuleNotFoundError as e:
|
||||
raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
|
||||
|
||||
# if users don't provider one, use the default litellm cache
|
||||
if disk_cache_dir is None:
|
||||
self.disk_cache = dc.Cache(".litellm_cache")
|
||||
else:
|
||||
self.disk_cache = dc.Cache(disk_cache_dir)
|
||||
|
||||
def set_cache(self, key, value, **kwargs):
|
||||
if "ttl" in kwargs:
|
||||
self.disk_cache.set(key, value, expire=kwargs["ttl"])
|
||||
else:
|
||||
self.disk_cache.set(key, value)
|
||||
|
||||
async def async_set_cache(self, key, value, **kwargs):
|
||||
self.set_cache(key=key, value=value, **kwargs)
|
||||
|
||||
async def async_set_cache_pipeline(self, cache_list, **kwargs):
|
||||
for cache_key, cache_value in cache_list:
|
||||
if "ttl" in kwargs:
|
||||
self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
|
||||
else:
|
||||
self.set_cache(key=cache_key, value=cache_value)
|
||||
|
||||
def get_cache(self, key, **kwargs):
|
||||
original_cached_response: Final = self.disk_cache.get(key)
|
||||
if original_cached_response:
|
||||
try:
|
||||
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
|
||||
except Exception:
|
||||
cached_response = original_cached_response
|
||||
return cached_response
|
||||
return None
|
||||
|
||||
def batch_get_cache(self, keys: list, **kwargs):
|
||||
return_val: Final = []
|
||||
for k in keys:
|
||||
val = self.get_cache(key=k, **kwargs)
|
||||
return_val.append(val)
|
||||
return return_val
|
||||
|
||||
def increment_cache(self, key, value: int, **kwargs) -> int:
|
||||
with self.disk_cache.transact():
|
||||
cached_value: Final = self.get_cache(key=key)
|
||||
init_value: Final = cached_value if isinstance(cached_value, int) else 0
|
||||
new_value: Final = init_value + value
|
||||
self.set_cache(key, new_value, **kwargs)
|
||||
return new_value
|
||||
|
||||
async def async_get_cache(self, key, **kwargs):
|
||||
return self.get_cache(key=key, **kwargs)
|
||||
|
||||
async def async_batch_get_cache(self, keys: list, **kwargs):
|
||||
return_val: Final = []
|
||||
for k in keys:
|
||||
val = self.get_cache(key=k, **kwargs)
|
||||
return_val.append(val)
|
||||
return return_val
|
||||
|
||||
async def async_increment(self, key, value: int, **kwargs) -> int:
|
||||
return self.increment_cache(key=key, value=value, **kwargs)
|
||||
|
||||
def flush_cache(self):
|
||||
self.disk_cache.clear()
|
||||
|
||||
async def disconnect(self):
|
||||
pass
|
||||
|
||||
def delete_cache(self, key):
|
||||
self.disk_cache.pop(key)
|
||||
|
|
@ -1887,7 +1887,10 @@ def completion_cost(
|
|||
video_resolution=video_resolution,
|
||||
)
|
||||
elif call_type in _SPEECH_CALL_TYPES:
|
||||
prompt_characters = litellm.utils.count_characters(text=prompt)
|
||||
prompt_characters = _speech_billed_characters(
|
||||
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
|
||||
prompt,
|
||||
)
|
||||
elif call_type in _TRANSCRIPTION_CALL_TYPES:
|
||||
# Check _hidden_params first (duration stored there to
|
||||
# avoid polluting the response body), then fall back to
|
||||
|
|
@ -2366,6 +2369,11 @@ def response_cost_calculator(
|
|||
raise e
|
||||
|
||||
|
||||
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
|
||||
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
|
||||
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
|
||||
|
||||
|
||||
def _video_input_image_cost(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ from litellm.main import (
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.llms.openai import ImageGenerationRequestQuality
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
LITELLM_IMAGE_VARIATION_PROVIDERS,
|
||||
LlmProviders,
|
||||
|
|
@ -1010,6 +1010,7 @@ def image_edit(
|
|||
optional_params=dict(image_edit_request_params),
|
||||
litellm_params={
|
||||
**image_edit_request_params,
|
||||
**litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": model_info,
|
||||
"vertex_location": litellm_params.vertex_location,
|
||||
|
|
|
|||
|
|
@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
|
|||
"output_cost_per_image_token",
|
||||
)
|
||||
|
||||
|
||||
def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool:
|
||||
return pricing is not None and (
|
||||
"input_cost_per_pixel" in pricing
|
||||
or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing)
|
||||
)
|
||||
|
||||
|
||||
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
|
||||
# request in the cost-calc path, so the f-strings are built once here instead
|
||||
# of being rebuilt for every model_info key on every call. Longest-first so a
|
||||
|
|
@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
|
||||
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+")
|
||||
|
||||
|
||||
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
|
||||
|
|
@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st
|
|||
|
||||
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
|
||||
value: Final = _requested_image_param(optional_params, "size")
|
||||
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
|
||||
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
|
||||
|
||||
|
||||
def get_web_search_requests(server_tool_use: object) -> int | None:
|
||||
|
|
@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol):
|
|||
|
||||
|
||||
def _model_priced_image_cost_calculator(
|
||||
custom_llm_provider: str | None, model: str
|
||||
custom_llm_provider: str | None, model: str, pricing: ModelInfo | None
|
||||
) -> _ModelPricedImageCostCalculator | None:
|
||||
match custom_llm_provider:
|
||||
case litellm.LlmProviders.RECRAFT.value:
|
||||
|
|
@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator(
|
|||
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
|
||||
|
||||
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
|
||||
case litellm.LlmProviders.DASHSCOPE.value:
|
||||
if _deployment_prices_images_without_an_output_price(pricing):
|
||||
return None
|
||||
from litellm.llms.dashscope.image_generation.cost_calculator import (
|
||||
cost_calculator as dashscope_image_cost_calculator,
|
||||
)
|
||||
|
||||
return dashscope_image_cost_calculator
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
|
@ -2178,7 +2194,7 @@ class CostCalculatorUtils:
|
|||
)
|
||||
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
|
||||
pricing: Final = deployment_pricing(model_info)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
|
||||
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
|
||||
|
||||
if model_priced_calculator is not None:
|
||||
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
|
||||
|
|
|
|||
45
litellm/litellm_core_utils/reasoning_effort_strength.py
Normal file
45
litellm/litellm_core_utils/reasoning_effort_strength.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set.
|
||||
|
||||
Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
|
||||
|
||||
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
|
||||
REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType(
|
||||
{effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}
|
||||
)
|
||||
|
||||
|
||||
def nearest_declared_reasoning_effort(
|
||||
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
|
||||
) -> str:
|
||||
"""Rounding up takes the weakest declared level at least as strong as the request, else the
|
||||
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
|
||||
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
|
||||
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
|
||||
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
|
||||
a model that cannot turn it off says so itself. A level outside the strength order is likewise
|
||||
returned as is for upstream to judge."""
|
||||
ranked: Final = sorted(
|
||||
(effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK),
|
||||
key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort],
|
||||
)
|
||||
if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked:
|
||||
return requested
|
||||
strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested]
|
||||
match rounding:
|
||||
case "up":
|
||||
return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1])
|
||||
case "down":
|
||||
return next(
|
||||
(effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0]
|
||||
)
|
||||
case _:
|
||||
assert_never(rounding)
|
||||
27
litellm/litellm_core_utils/stream_usage_chunks.py
Normal file
27
litellm/litellm_core_utils/stream_usage_chunks.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
from collections.abc import Iterable
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.utils import ModelResponseStream, StreamingChoices
|
||||
|
||||
|
||||
def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
|
||||
if choice.finish_reason is not None:
|
||||
return False
|
||||
if getattr(choice, "logprobs", None) is not None:
|
||||
return False
|
||||
delta: Final = choice.delta if hasattr(choice, "delta") else None
|
||||
if delta is None:
|
||||
return True
|
||||
return _all_values_none(delta.model_dump().values())
|
||||
|
||||
|
||||
def _all_values_none(values: Iterable[object]) -> bool:
|
||||
return all(value is None for value in values)
|
||||
|
||||
|
||||
def is_injected_stream_usage_artifact(chunk: object) -> bool:
|
||||
if not isinstance(chunk, ModelResponseStream):
|
||||
return False
|
||||
if chunk.provider_specific_fields is not None:
|
||||
return False
|
||||
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])
|
||||
|
|
@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
|
|||
if expected_cls is None:
|
||||
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
|
||||
|
||||
missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content]
|
||||
if missing:
|
||||
raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}")
|
||||
|
||||
return expected_cls
|
||||
|
||||
|
||||
|
|
@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[
|
|||
|
||||
def _format_object_parameters(parameters, indent):
|
||||
properties: Final = parameters.get("properties")
|
||||
if not properties:
|
||||
if not isinstance(properties, Mapping) or not properties:
|
||||
return ""
|
||||
required_params: Final = parameters.get("required", [])
|
||||
required: Final = parameters.get("required")
|
||||
required_params: Final = tuple(required) if isinstance(required, list) else ()
|
||||
lines: Final = []
|
||||
for key, props in properties.items():
|
||||
description = props.get("description")
|
||||
description = props.get("description") if isinstance(props, Mapping) else None
|
||||
if description:
|
||||
lines.append(f"// {description}")
|
||||
question = "?"
|
||||
|
|
@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent):
|
|||
|
||||
|
||||
def _format_type(props, indent):
|
||||
if not isinstance(props, Mapping):
|
||||
return "any"
|
||||
type: Final = props.get("type")
|
||||
enum: Final = props.get("enum")
|
||||
if type == "string":
|
||||
if "enum" in props:
|
||||
return " | ".join([f'"{item}"' for item in props["enum"]])
|
||||
if isinstance(enum, list):
|
||||
return " | ".join([f'"{item}"' for item in enum])
|
||||
return "string"
|
||||
elif type == "array":
|
||||
# items is required, OpenAI throws an error if it's missing
|
||||
return f"{_format_type(props['items'], indent)}[]"
|
||||
return f"{_format_type(props.get('items'), indent)}[]"
|
||||
elif type == "object":
|
||||
return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}"
|
||||
elif type in ["integer", "number"]:
|
||||
if "enum" in props:
|
||||
return " | ".join([f'"{item}"' for item in props["enum"]])
|
||||
if isinstance(enum, list):
|
||||
return " | ".join([f'"{item}"' for item in enum])
|
||||
return "number"
|
||||
elif type == "boolean":
|
||||
return "boolean"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import copy
|
|||
import json
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
|
||||
|
|
@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object)
|
|||
return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
|
||||
|
||||
|
||||
_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile(
|
||||
r"Too many strict tools \(\d+\)"
|
||||
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
|
||||
r"|[Cc]ompiled grammar (?:is too large|size)"
|
||||
r"|tools\.\d+\.custom(?:\.strict)?: "
|
||||
r"|Grammar compilation timed out"
|
||||
)
|
||||
_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool:
|
||||
return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ToolsWithoutStrict:
|
||||
tools: tuple[Mapping[str, object], ...]
|
||||
dropped_count: int
|
||||
|
||||
|
||||
def tools_without_strict(tools: object) -> ToolsWithoutStrict | None:
|
||||
try:
|
||||
validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools)
|
||||
except ValidationError:
|
||||
return None
|
||||
dropped_count: Final = sum(tool.get("strict") is True for tool in validated)
|
||||
if dropped_count == 0:
|
||||
return None
|
||||
return ToolsWithoutStrict(
|
||||
tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated),
|
||||
dropped_count=dropped_count,
|
||||
)
|
||||
|
||||
|
||||
def warn_claude_strict_tools_dropped(dropped_count: int) -> None:
|
||||
litellm.verbose_logger.warning(
|
||||
"Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had "
|
||||
"strict true) and sent the request again because drop_params is set.",
|
||||
dropped_count,
|
||||
)
|
||||
|
||||
|
||||
def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
|
||||
"""
|
||||
Return a new message list with thinking / redacted_thinking content blocks removed
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ async def prepare_context_managed_request(
|
|||
"""Apply client compaction history, then optional context_management polyfill."""
|
||||
from litellm.llms.anthropic.messages.context_management.editors.compact import (
|
||||
apply_client_compaction_block_history,
|
||||
slice_around_compaction_block,
|
||||
)
|
||||
|
||||
# Skip the client-history pre-processing when a ``compact_20260112``
|
||||
|
|
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
|
|||
|
||||
if polyfill_will_run:
|
||||
history_result: PolyfillResult | None = None
|
||||
working_messages: _AnthropicMessages = messages
|
||||
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
|
||||
working_system: _AnthropicSystem = system
|
||||
else:
|
||||
history_result = apply_client_compaction_block_history(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.types.llms.anthropic import AppliedEdit
|
||||
|
||||
from ..convertible_blocks import drop_non_string_tool_result_texts
|
||||
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
|
||||
from .editors import (
|
||||
CheckedEdit,
|
||||
|
|
@ -103,7 +104,7 @@ async def apply_context_management(
|
|||
|
||||
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
|
||||
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
|
||||
``drop_invalid_edits``.
|
||||
``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it.
|
||||
|
||||
The dispatcher is async so async editors (``compact_20260112``) can
|
||||
``await`` the configured summarization model. Sync editors run in a
|
||||
|
|
@ -119,7 +120,7 @@ async def apply_context_management(
|
|||
raise first_error
|
||||
well_formed_edits: Final = tuple(check.edit for check in checked)
|
||||
|
||||
current_messages = messages
|
||||
current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits))
|
||||
current_system = system
|
||||
aggregated_applied: Final[list[AppliedEdit]] = []
|
||||
aggregated_compaction_block = None
|
||||
|
|
|
|||
|
|
@ -39,6 +39,21 @@ if TYPE_CHECKING:
|
|||
T = TypeVar("T")
|
||||
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
|
||||
_TINY_RESULT_BYTES: Final = 46
|
||||
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
|
||||
{
|
||||
"text",
|
||||
"image_url",
|
||||
"image",
|
||||
"document",
|
||||
"file",
|
||||
"tool_use",
|
||||
"tool_result",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_reference",
|
||||
"search_result",
|
||||
}
|
||||
)
|
||||
ToolNames = Literal["all"] | frozenset[str]
|
||||
|
||||
|
||||
|
|
@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel):
|
|||
class _ClearToolUsesFields(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
trigger: Mapping[str, object] | None = None
|
||||
keep: Mapping[str, object] | None = None
|
||||
clear_at_least: _ClearAtLeast | None = None
|
||||
exclude_tools: tuple[str, ...] | None = None
|
||||
clear_tool_inputs: bool | tuple[str, ...] | None = None
|
||||
|
||||
|
||||
_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently
|
||||
# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
|
||||
_API_MESSAGES: Final = MappingProxyType(
|
||||
{
|
||||
"tuple_type": "Input should be a valid array",
|
||||
"dict_type": "Input should be an object",
|
||||
"model_type": "Input does not match the expected shape.",
|
||||
}
|
||||
)
|
||||
|
|
@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]:
|
|||
|
||||
def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int:
|
||||
"""Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it."""
|
||||
before: Final = [result, use] if clears_input else [result]
|
||||
after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)]
|
||||
before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,)
|
||||
after: Final[Sequence[Mapping[str, object]]] = (
|
||||
(_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),)
|
||||
)
|
||||
counts: Final = tuple(
|
||||
litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after)
|
||||
_count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after)
|
||||
)
|
||||
return max(counts[0] - counts[1], 0)
|
||||
|
||||
|
|
@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
|
|||
return tuple(candidates[:batch_end])
|
||||
|
||||
|
||||
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
nested: Final = _parse(_BLOCKS, block.get("content"))
|
||||
if nested is None:
|
||||
return block
|
||||
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
|
||||
|
||||
|
||||
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
blocks: Final = _parse(_BLOCKS, message.get("content"))
|
||||
if blocks is None:
|
||||
return message
|
||||
return {
|
||||
**message,
|
||||
"content": [
|
||||
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _count_tokens(
|
||||
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
|
||||
) -> int:
|
||||
return litellm.token_counter(
|
||||
model=model,
|
||||
messages=[_countable_message(message) for message in messages],
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
|
||||
|
||||
def _trigger_met(
|
||||
trigger: dict[str, object],
|
||||
model: str,
|
||||
|
|
@ -218,11 +267,7 @@ def _trigger_met(
|
|||
|
||||
if not isinstance(threshold, int):
|
||||
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
|
||||
message_tokens: Final = litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
message_tokens: Final = _count_tokens(model, messages, tools)
|
||||
if message_tokens > threshold:
|
||||
return True, message_tokens
|
||||
current_tokens: Final = message_tokens + count_system_tokens(model, system)
|
||||
|
|
@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919(
|
|||
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
|
||||
)
|
||||
|
||||
tokens_before: Final = (
|
||||
counted_tokens
|
||||
if counted_tokens is not None
|
||||
else litellm.token_counter(
|
||||
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
)
|
||||
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools)
|
||||
kept_after, _ = history_after_client_compaction(final_messages, system)
|
||||
tokens_after: Final = litellm.token_counter(
|
||||
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
tokens_after: Final = _count_tokens(model, kept_after, tools)
|
||||
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
|
||||
|
||||
return final_messages, {
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ def _find_latest_compaction_index(
|
|||
return None, None
|
||||
|
||||
|
||||
def _slice_around_compaction_block(
|
||||
def slice_around_compaction_block(
|
||||
messages: Sequence[_MsgT],
|
||||
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
|
||||
"""Apply Anthropic's "drop everything before the compaction block" rule.
|
||||
|
|
@ -194,7 +194,7 @@ def history_after_client_compaction(
|
|||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
|
||||
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
sliced, block = slice_around_compaction_block(messages)
|
||||
if block is None:
|
||||
return list(messages), system
|
||||
tail: Final = _strip_compaction_blocks(sliced)
|
||||
|
|
@ -615,7 +615,7 @@ def apply_client_compaction_block_history(
|
|||
tail is forwarded unchanged (with compaction blocks stripped) so recent
|
||||
turns the summary does not cover are preserved.
|
||||
"""
|
||||
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
|
||||
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
|
||||
if prior_compaction_block is None:
|
||||
return None
|
||||
|
||||
|
|
@ -686,7 +686,7 @@ async def apply_compact_20260112(
|
|||
# opt-in gate below so that even when summarization is disabled we still
|
||||
# strip Anthropic-only ``compaction`` blocks from messages going to
|
||||
# non-Anthropic backends (which would reject them).
|
||||
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
|
||||
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
|
||||
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
|
||||
augmented_system: str | list[dict[str, object]] | None = system
|
||||
if isinstance(prior_summary_text, str) and prior_summary_text:
|
||||
|
|
|
|||
|
|
@ -10,11 +10,15 @@ from typing import Final, TypeAlias
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
supports_openai_prompt_cache_breakpoint,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.asyncify import (
|
||||
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
prompt_cache_breakpoint_for_wire,
|
||||
|
|
@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import (
|
|||
AnthropicMessagesRequest,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicOutputSchema,
|
||||
ContextManagementResponse,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import CountContextManagement
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
from ..adapters.handler import request_safeguards_evaluator
|
||||
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
|
||||
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
|
||||
from ..context_management.constants import COMPACT_EDIT_TYPE
|
||||
from ..context_management.dispatcher import normalize_spec
|
||||
from ..convertible_blocks import reject_malformed_tool_result_text
|
||||
from ..safeguards import with_safeguard_results
|
||||
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
|
||||
|
|
@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
|||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
|
||||
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
_SYSTEM_ITEMS: Final = TypeAdapter(list[object])
|
||||
_SPEC: Final = TypeAdapter(dict[str, object])
|
||||
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
|
||||
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
|
||||
|
||||
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
||||
|
||||
def _split_context_management(
|
||||
spec: AnthropicContextManagementSpec,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _SplitEdits:
|
||||
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
|
||||
|
||||
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
|
||||
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
|
||||
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
|
||||
unsent, as it was before the polyfill ran here.
|
||||
"""
|
||||
if not isinstance(spec, dict):
|
||||
return None, ()
|
||||
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
|
||||
return spec, ()
|
||||
edits: Final = normalize_spec(spec) or []
|
||||
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
|
||||
return ({"edits": native} if native else None), tuple(
|
||||
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
|
||||
)
|
||||
|
||||
|
||||
def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None:
|
||||
if system is None or isinstance(system, str):
|
||||
return system
|
||||
try:
|
||||
items: Final = _SYSTEM_ITEMS.validate_python(system)
|
||||
except ValidationError:
|
||||
return None
|
||||
return tuple(block for block in map(as_str_mapping, items) if block is not None)
|
||||
|
||||
|
||||
async def _polyfilled(
|
||||
*,
|
||||
model: str,
|
||||
messages: AnthropicRequestMessages,
|
||||
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
|
||||
system: object,
|
||||
edits: Sequence[Mapping[str, object]],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> PolyfillResult | None:
|
||||
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
|
||||
if not edits:
|
||||
return None
|
||||
reject_malformed_tool_result_text(messages, model, kwargs)
|
||||
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
|
||||
blocks: Final = _system_blocks(system)
|
||||
try:
|
||||
return await apply_context_management(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
|
||||
system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks),
|
||||
context_management_spec={"edits": [dict(edit) for edit in edits]},
|
||||
drop_invalid_edits=drop_invalid_edits,
|
||||
)
|
||||
except AnthropicContextManagementError:
|
||||
raise
|
||||
except Exception as e:
|
||||
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
|
||||
raise AnthropicContextManagementError(
|
||||
status_code=500, message=f"context_management polyfill failed: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
async def count_view_messages(
|
||||
*,
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
|
||||
if context_management is None or not isinstance(context_management.spec, dict):
|
||||
return messages
|
||||
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
|
||||
result: Final = await _polyfilled(
|
||||
model=model,
|
||||
messages=_BLOCK_LIST.validate_python(messages),
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=edits,
|
||||
kwargs={"drop_params": context_management.drop_invalid_edits},
|
||||
)
|
||||
return messages if result is None else result.messages
|
||||
|
||||
|
||||
def _with_applied_edits(
|
||||
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
|
||||
) -> AnthropicMessagesResponse:
|
||||
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
|
||||
if not applied_edits:
|
||||
return response
|
||||
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
|
||||
|
||||
|
||||
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
|
||||
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
|
||||
if logging_obj is not None:
|
||||
|
|
@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
model: str,
|
||||
context_management: dict[str, object] | None = None,
|
||||
context_management: AnthropicContextManagementSpec = None,
|
||||
metadata: dict[str, object] | None = None,
|
||||
output_config: AnthropicOutputConfig | None = None,
|
||||
stop_sequences: list[str] | None = None,
|
||||
|
|
@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
output_format: AnthropicOutputSchema | None = None,
|
||||
**kwargs: object,
|
||||
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
polyfill_result: Final = await _polyfilled(
|
||||
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
|
||||
)
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
safeguards_evaluator=safeguards_evaluator,
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
|
||||
served: Final = await with_safeguard_results(
|
||||
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
|
||||
)
|
||||
_record_bridged_response(responses_kwargs, result)
|
||||
return served
|
||||
|
||||
|
|
@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
max_tokens: int,
|
||||
messages: AnthropicRequestMessages,
|
||||
model: str,
|
||||
context_management: dict[str, object] | None = None,
|
||||
context_management: AnthropicContextManagementSpec = None,
|
||||
metadata: dict[str, object] | None = None,
|
||||
output_config: AnthropicOutputConfig | None = None,
|
||||
stop_sequences: list[str] | None = None,
|
||||
|
|
@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
)
|
||||
|
||||
# Sync path
|
||||
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
|
||||
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
|
||||
run_async_function(
|
||||
_polyfilled,
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
edits=polyfilled_edits,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
if polyfilled_edits
|
||||
else None
|
||||
)
|
||||
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
messages=polyfill_result.messages if polyfill_result is not None else messages,
|
||||
model=model,
|
||||
context_management=context_management,
|
||||
context_management=native_context_management,
|
||||
metadata=metadata,
|
||||
output_config=output_config,
|
||||
stop_sequences=stop_sequences,
|
||||
|
|
@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
|
|||
responses_stream=result,
|
||||
model=local_model_name(model, kwargs.get("custom_llm_provider")),
|
||||
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
|
||||
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
|
||||
)
|
||||
return wrapper.async_anthropic_sse_wrapper()
|
||||
|
||||
if not isinstance(result, ResponsesAPIResponse):
|
||||
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
|
||||
|
||||
return _ADAPTER.translate_response(result)
|
||||
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import anyio
|
||||
|
|
@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import (
|
|||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper:
|
|||
model: str,
|
||||
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
|
||||
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
|
||||
applied_edits: Sequence[AppliedEdit] | None = None,
|
||||
) -> None:
|
||||
self.responses_stream = responses_stream
|
||||
self._applied_edits: Final = tuple(applied_edits or ())
|
||||
self.model = model
|
||||
self.litellm_logging_obj = litellm_logging_obj
|
||||
self.safeguards_evaluator: Final = safeguards_evaluator
|
||||
|
|
@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper:
|
|||
"type": "message_delta",
|
||||
"delta": message_delta_payload,
|
||||
"usage": dict(anthropic_usage),
|
||||
**(
|
||||
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
|
||||
if self._applied_edits
|
||||
else {}
|
||||
),
|
||||
}
|
||||
)
|
||||
self._chunk_queue.append({"type": "message_stop"})
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
)
|
||||
|
|
@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
anthropic_request_errors_on_thinking_binding_mismatch,
|
||||
is_anthropic_invalid_thinking_block_error,
|
||||
is_anthropic_thinking_binding_mismatch_error,
|
||||
is_claude_strict_tools_rejection,
|
||||
tools_without_strict,
|
||||
validated_object_mapping,
|
||||
)
|
||||
|
||||
if e.response.status_code != 400:
|
||||
return False
|
||||
error_text: Final[str] = e.response.text
|
||||
body: Final = validated_object_mapping(request_body)
|
||||
drops_rejected_strict_tools: Final = (
|
||||
drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
and is_claude_strict_tools_rejection(e.response.status_code, error_text)
|
||||
and body is not None
|
||||
and tools_without_strict(body.get("tools")) is not None
|
||||
)
|
||||
if drops_rejected_strict_tools:
|
||||
return True
|
||||
caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch(
|
||||
request_body
|
||||
) and is_anthropic_thinking_binding_mismatch_error(error_text)
|
||||
|
|
@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC):
|
|||
"""
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
is_anthropic_invalid_thinking_block_error,
|
||||
is_claude_strict_tools_rejection,
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict,
|
||||
tools_without_strict,
|
||||
warn_claude_strict_tools_dropped,
|
||||
)
|
||||
|
||||
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
|
||||
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
|
||||
without_strict: Final = (
|
||||
tools_without_strict(request_data.get("tools"))
|
||||
if is_claude_strict_tools_rejection(e.response.status_code, e.response.text)
|
||||
else None
|
||||
)
|
||||
if without_strict is not None:
|
||||
warn_claude_strict_tools_dropped(without_strict.dropped_count)
|
||||
request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict
|
||||
return request_data
|
||||
|
||||
def native_output_fallback_litellm_params(
|
||||
|
|
|
|||
|
|
@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC):
|
|||
Transform provider response to standard format
|
||||
"""
|
||||
|
||||
async def async_transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
return self.transform_text_to_speech_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException:
|
||||
from ..chat.transformation import BaseLLMException
|
||||
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine
|
|||
from functools import partial
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
is_bedrock_native_output_rejection,
|
||||
is_bedrock_strict_tools_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
warn_bedrock_strict_tools_dropped,
|
||||
)
|
||||
|
|
@ -34,7 +34,7 @@ def _native_output_retry(
|
|||
def _strict_tools_retry(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
|
||||
if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
|
|
|
|||
|
|
@ -366,21 +366,10 @@ class NativeOutputFallback:
|
|||
litellm_params: Mapping[str, object]
|
||||
|
||||
|
||||
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
|
||||
r"Too many strict tools \(\d+\)"
|
||||
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
|
||||
r"|[Cc]ompiled grammar (?:is too large|size)"
|
||||
r"|tools\.\d+\.custom(?:\.strict)?: "
|
||||
r"|Grammar compilation timed out"
|
||||
)
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
|
||||
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
|
||||
|
||||
|
||||
def warn_bedrock_strict_tools_dropped(model: str) -> None:
|
||||
verbose_logger.warning(
|
||||
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
|
||||
|
|
|
|||
|
|
@ -2467,8 +2467,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"Anthropic /v1/messages: invalid thinking signature; "
|
||||
"stripping thinking blocks and retrying (attempt %s/%s).",
|
||||
"Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).",
|
||||
len(recovered_errors) + 2,
|
||||
max_attempts,
|
||||
)
|
||||
|
|
@ -12850,7 +12849,7 @@ class BaseLLMHTTPHandler:
|
|||
provider_config=text_to_speech_provider_config,
|
||||
)
|
||||
|
||||
speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response(
|
||||
speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response(
|
||||
model=model,
|
||||
raw_response=response,
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic"
|
|||
DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1"
|
||||
DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1"
|
||||
DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks"
|
||||
DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
|
||||
DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
|
||||
DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH
|
||||
DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com"
|
||||
|
||||
_STANDARD_DASHSCOPE_HOSTS: Final = frozenset(
|
||||
|
|
@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool:
|
|||
return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/")
|
||||
|
||||
|
||||
def dashscope_api_root(api_base: str) -> str:
|
||||
endpoints: Final = resolve_dashscope_endpoints(api_base)
|
||||
if endpoints is not None:
|
||||
return endpoints.origin
|
||||
trimmed: Final = api_base.rstrip("/")
|
||||
if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH):
|
||||
return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH)
|
||||
root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH)
|
||||
return root if separator else trimmed
|
||||
|
||||
|
||||
def is_aliyun_host(host: str | None) -> bool:
|
||||
return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com"))
|
||||
|
||||
|
||||
def resolve_dashscope_openai_api_base(
|
||||
api_base: str | None,
|
||||
env_var: str = "DASHSCOPE_API_BASE",
|
||||
|
|
@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None:
|
|||
if api_base is None:
|
||||
return None
|
||||
parsed: Final = urlparse(api_base)
|
||||
host: Final = parsed.hostname or ""
|
||||
on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com")
|
||||
if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
|
||||
if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
|
||||
return None
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
|
||||
|
|
|
|||
0
litellm/llms/dashscope/image_edit/__init__.py
Normal file
0
litellm/llms/dashscope/image_edit/__init__.py
Normal file
160
litellm/llms/dashscope/image_edit/transformation.py
Normal file
160
litellm/llms/dashscope/image_edit/transformation.py
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import override
|
||||
|
||||
import litellm
|
||||
from litellm.images.utils import image_to_data_url
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.dashscope.common_utils import (
|
||||
DASHSCOPE_CHAT_COMPATIBLE_PATH,
|
||||
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
|
||||
dashscope_api_root,
|
||||
is_dashscope_native_endpoint_url,
|
||||
resolve_dashscope_endpoints,
|
||||
)
|
||||
from litellm.llms.dashscope.image_generation.transformation import (
|
||||
DEFAULT_API_BASE,
|
||||
OPENAI_TO_DASHSCOPE_SIZE,
|
||||
DashScopeImageGenerationConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
MAX_INPUT_IMAGES: Final = 3
|
||||
DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed")
|
||||
|
||||
_IMAGES: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _dashscope_size(size: object) -> object:
|
||||
if not isinstance(size, str):
|
||||
return size
|
||||
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
|
||||
|
||||
|
||||
def _deployment_edit_url(api_base: str) -> str:
|
||||
derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and (
|
||||
resolve_dashscope_endpoints(api_base) is not None
|
||||
or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH)
|
||||
)
|
||||
return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base
|
||||
|
||||
|
||||
class DashScopeImageEditConfig(BaseImageEditConfig):
|
||||
def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None:
|
||||
super().__init__()
|
||||
self._generation_config: Final = generation_config or DashScopeImageGenerationConfig()
|
||||
|
||||
@override
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return list(DASHSCOPE_IMAGE_EDIT_PARAMS)
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
image_edit_optional_params: ImageEditOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: mirrors override contract
|
||||
return {
|
||||
key: _dashscope_size(value) if key == "size" else value
|
||||
for key, value in image_edit_optional_params.items()
|
||||
if key in DASHSCOPE_IMAGE_EDIT_PARAMS
|
||||
}
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: mirrors override contract
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
return self._generation_config.validate_environment(
|
||||
headers=dict(headers),
|
||||
model=model,
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params=dict(litellm_params or {}),
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
) -> str:
|
||||
if api_base is not None:
|
||||
return _deployment_edit_url(api_base)
|
||||
env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE")
|
||||
derived: Final = (
|
||||
f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None
|
||||
)
|
||||
return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE
|
||||
|
||||
@override
|
||||
def use_multipart_form_data(self) -> bool:
|
||||
return False
|
||||
|
||||
@override
|
||||
def transform_image_edit_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str | None,
|
||||
image: object,
|
||||
image_edit_optional_request_params: Mapping[str, object],
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: Mapping[str, str],
|
||||
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
|
||||
candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
|
||||
images: Final = tuple(candidate for candidate in candidates if candidate is not None)
|
||||
if len(images) > MAX_INPUT_IMAGES:
|
||||
raise litellm.BadRequestError(
|
||||
message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}",
|
||||
model=model,
|
||||
llm_provider="dashscope",
|
||||
)
|
||||
prompt_content: Final = ({"text": prompt},) if prompt is not None else ()
|
||||
return {
|
||||
"model": model,
|
||||
"input": {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content],
|
||||
}
|
||||
]
|
||||
},
|
||||
"parameters": dict(image_edit_optional_request_params),
|
||||
}, ()
|
||||
|
||||
@override
|
||||
def transform_image_edit_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> ImageResponse:
|
||||
return self._generation_config.transform_image_generation_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
request_data={},
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
19
litellm/llms/dashscope/image_generation/cost_calculator.py
Normal file
19
litellm/llms/dashscope/image_generation/cost_calculator.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
|
||||
from litellm.types.utils import ImageResponse, ModelInfo
|
||||
|
||||
|
||||
def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
|
||||
prices: Final = resolve_image_model_info(
|
||||
model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info
|
||||
)
|
||||
output_cost_per_image: Final = prices.get("output_cost_per_image")
|
||||
if output_cost_per_image is None:
|
||||
raise ValueError(f"No output_cost_per_image found for dashscope/{model}")
|
||||
input_images: Final = image_response.hidden_params.get("billable_input_image_count")
|
||||
input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * (
|
||||
input_images if isinstance(input_images, int) else 0
|
||||
)
|
||||
return output_cost_per_image * len(image_response.data or ()) + input_image_cost
|
||||
|
|
@ -23,7 +23,7 @@ Response format:
|
|||
}
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
|
||||
|
||||
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
|
||||
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
|
||||
OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = {
|
||||
"256x256": "256*256",
|
||||
"512x512": "512*512",
|
||||
"1024x1024": "1024*1024",
|
||||
|
|
@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
|
|||
}
|
||||
|
||||
|
||||
def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]:
|
||||
contents: Final = _JSON_OBJECTS.validate_python(message.get("content", []))
|
||||
return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url)
|
||||
|
||||
|
||||
def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]:
|
||||
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
|
||||
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
|
||||
for choice in choices:
|
||||
yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {})))
|
||||
|
||||
|
||||
def _dashscope_size(size: object) -> object:
|
||||
if not isinstance(size, str):
|
||||
return size
|
||||
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
|
||||
|
||||
|
||||
class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
|
||||
|
|
@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
non_default_params: dict[str, object],
|
||||
optional_params: dict[str, object],
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
) -> dict[str, object]:
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
mapped: Final[dict] = {}
|
||||
for k, v in non_default_params.items():
|
||||
if k in optional_params:
|
||||
continue
|
||||
if k not in supported_params:
|
||||
continue
|
||||
if k == "size":
|
||||
# Convert "WxH" → "W*H"
|
||||
mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
|
||||
else:
|
||||
mapped[k] = v
|
||||
return mapped
|
||||
requested: Final = {
|
||||
k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params
|
||||
}
|
||||
return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()}
|
||||
|
||||
def _resolve_api_key(self, api_key: str | None) -> str:
|
||||
resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY")
|
||||
|
|
@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
stream: bool | None = None,
|
||||
) -> str:
|
||||
image_env_api_base: Final = self._image_api_base_env()
|
||||
|
|
@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}"
|
||||
headers["Content-Type"] = "application/json"
|
||||
return headers
|
||||
|
|
@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
request_data: dict[str, object],
|
||||
optional_params: dict[str, object],
|
||||
litellm_params: dict[str, object],
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
|
|
@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
|
|||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
if not model_response.data:
|
||||
model_response.data = []
|
||||
|
||||
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
|
||||
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
|
||||
for choice in choices:
|
||||
message = _JSON_OBJECT.validate_python(choice.get("message", {}))
|
||||
content_list = _JSON_OBJECTS.validate_python(message.get("content", []))
|
||||
for content_item in content_list:
|
||||
image_url = content_item.get("image")
|
||||
if image_url:
|
||||
model_response.data.append(ImageObject(url=image_url))
|
||||
|
||||
return model_response
|
||||
images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))]
|
||||
usage: Final = response_object.get("usage")
|
||||
usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {}
|
||||
width: Final = usage_fields.get("output_width", usage_fields.get("width"))
|
||||
height: Final = usage_fields.get("output_height", usage_fields.get("height"))
|
||||
image_response: Final = model_response.model_copy(
|
||||
update={
|
||||
"data": images,
|
||||
**({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}),
|
||||
}
|
||||
)
|
||||
input_image_count: Final = usage_fields.get("input_image_count")
|
||||
if isinstance(input_image_count, int) and input_image_count > 0:
|
||||
image_response.hidden_params = {
|
||||
**model_response.hidden_params,
|
||||
"billable_input_image_count": input_image_count,
|
||||
}
|
||||
return image_response
|
||||
|
|
|
|||
0
litellm/llms/dashscope/text_to_speech/__init__.py
Normal file
0
litellm/llms/dashscope/text_to_speech/__init__.py
Normal file
297
litellm/llms/dashscope/text_to_speech/transformation.py
Normal file
297
litellm/llms/dashscope/text_to_speech/transformation.py
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import override
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
|
||||
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
|
||||
)
|
||||
from litellm.llms.dashscope.common_utils import (
|
||||
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
|
||||
DashScopeError,
|
||||
dashscope_api_root,
|
||||
is_aliyun_host,
|
||||
missing_dashscope_family_key_message,
|
||||
resolve_dashscope_family_api_key,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com"
|
||||
_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-"
|
||||
_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TextToSpeechFamily:
|
||||
path: str
|
||||
response_formats: frozenset[str]
|
||||
sends_format: bool
|
||||
takes_instructions: bool
|
||||
|
||||
|
||||
MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily(
|
||||
path=DASHSCOPE_MULTIMODAL_GENERATION_PATH,
|
||||
response_formats=frozenset({"wav"}),
|
||||
sends_format=False,
|
||||
takes_instructions=False,
|
||||
)
|
||||
SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily(
|
||||
path="/api/v1/services/audio/tts/SpeechSynthesizer",
|
||||
response_formats=frozenset({"mp3", "wav", "opus", "pcm"}),
|
||||
sends_format=True,
|
||||
takes_instructions=True,
|
||||
)
|
||||
|
||||
|
||||
class _Audio(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
url: str
|
||||
|
||||
|
||||
class _Output(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
audio: _Audio
|
||||
|
||||
|
||||
class _Usage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
characters: int | None = None
|
||||
|
||||
|
||||
class _SpeechResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
output: _Output | None = None
|
||||
usage: _Usage | None = None
|
||||
code: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
def text_to_speech_family(model: str) -> _TextToSpeechFamily:
|
||||
if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX):
|
||||
return SPEECH_SYNTHESIZER_FAMILY
|
||||
return MULTIMODAL_GENERATION_FAMILY
|
||||
|
||||
|
||||
def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None:
|
||||
match key:
|
||||
case "response_format" if value in family.response_formats:
|
||||
return {"format": value} if family.sends_format else {}
|
||||
case "instructions" if family.takes_instructions:
|
||||
return {"instruction": value}
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError:
|
||||
return UnsupportedParamsError(
|
||||
message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it",
|
||||
model=model,
|
||||
llm_provider="dashscope",
|
||||
)
|
||||
|
||||
|
||||
def _speech_response(raw_response: httpx.Response) -> _SpeechResponse:
|
||||
try:
|
||||
return _SpeechResponse.model_validate_json(raw_response.content)
|
||||
except ValidationError as e:
|
||||
raise DashScopeError(
|
||||
status_code=502,
|
||||
message=f"Unexpected DashScope text-to-speech response: {e}",
|
||||
headers=raw_response.headers,
|
||||
) from e
|
||||
|
||||
|
||||
def _https_on_aliyun(url: str) -> str:
|
||||
parsed: Final = urlparse(url)
|
||||
return (
|
||||
parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url
|
||||
)
|
||||
|
||||
|
||||
def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]:
|
||||
parsed: Final = _speech_response(raw_response)
|
||||
if parsed.output is None:
|
||||
raise DashScopeError(
|
||||
status_code=max(raw_response.status_code, 400),
|
||||
message=parsed.message or parsed.code or raw_response.text,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
return _https_on_aliyun(parsed.output.audio.url), parsed.usage
|
||||
|
||||
|
||||
def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None:
|
||||
extension: Final = raw_response.request.extensions.get("timeout")
|
||||
if extension is None:
|
||||
return None
|
||||
phases: Final = _TIMEOUT_PHASES.validate_python(extension)
|
||||
return httpx.Timeout(
|
||||
connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool")
|
||||
)
|
||||
|
||||
|
||||
def _token_usage(usage: _Usage | None) -> Usage | None:
|
||||
if usage is None or usage.input_tokens is None or usage.output_tokens is None:
|
||||
return None
|
||||
return Usage(
|
||||
prompt_tokens=usage.input_tokens,
|
||||
completion_tokens=usage.output_tokens,
|
||||
total_tokens=usage.input_tokens + usage.output_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _refused_audio_url(error: SSRFError) -> DashScopeError:
|
||||
return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}")
|
||||
|
||||
|
||||
def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent:
|
||||
if audio_response.status_code != 200:
|
||||
raise DashScopeError(
|
||||
status_code=502,
|
||||
message=f"DashScope audio download failed with HTTP {audio_response.status_code}",
|
||||
headers=audio_response.headers,
|
||||
)
|
||||
binary: Final = HttpxBinaryResponseContent(audio_response)
|
||||
binary.usage = _token_usage(usage)
|
||||
if usage is not None and usage.characters is not None:
|
||||
binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters}
|
||||
return binary
|
||||
|
||||
|
||||
def _shared_async_client() -> AsyncHTTPHandler:
|
||||
return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE)
|
||||
|
||||
|
||||
class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
def __init__(
|
||||
self,
|
||||
sync_client: Callable[[], HTTPHandler] = get_httpx_client,
|
||||
async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._sync_client: Final = sync_client
|
||||
self._async_client: Final = async_client
|
||||
|
||||
@override
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
|
||||
family: Final = text_to_speech_family(model)
|
||||
return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())]
|
||||
|
||||
@override
|
||||
def map_openai_params(
|
||||
self,
|
||||
model: str,
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract
|
||||
drop_params: bool = False,
|
||||
kwargs: Mapping[str, object] | None = None,
|
||||
) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract
|
||||
family: Final = text_to_speech_family(model)
|
||||
fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items())
|
||||
unsupported: Final = (
|
||||
*(f"{key}={value!r}" for key, value, field in fields if field is None),
|
||||
*(("voice as an object",) if isinstance(voice, dict) else ()),
|
||||
)
|
||||
if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))):
|
||||
raise _unsupported(model, unsupported)
|
||||
supported: Final = tuple(field for _, _, field in fields if field is not None)
|
||||
return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported))
|
||||
|
||||
@override
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: mirrors override contract
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: mirrors override contract
|
||||
resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key)
|
||||
if resolved_api_key is None:
|
||||
raise ValueError(missing_dashscope_family_key_message("dashscope"))
|
||||
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
|
||||
|
||||
@override
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
) -> str:
|
||||
configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE")
|
||||
root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE
|
||||
return f"{root}{text_to_speech_family(model).path}"
|
||||
|
||||
@override
|
||||
def transform_text_to_speech_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | None,
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
headers: dict[str, str], # mutable-ok: mirrors override contract
|
||||
) -> TextToSpeechRequestData:
|
||||
voice_field: Final = {"voice": voice} if voice is not None else {}
|
||||
return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}}
|
||||
|
||||
@override
|
||||
def transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> HttpxBinaryResponseContent:
|
||||
audio_url, usage = _audio_url(raw_response)
|
||||
try:
|
||||
audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response))
|
||||
except SSRFError as e:
|
||||
raise _refused_audio_url(e) from e
|
||||
return _binary_audio(audio_response, usage)
|
||||
|
||||
@override
|
||||
async def async_transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> HttpxBinaryResponseContent:
|
||||
audio_url, usage = _audio_url(raw_response)
|
||||
try:
|
||||
audio_response: Final = await async_safe_get(
|
||||
self._async_client(), audio_url, timeout=_request_timeout(raw_response)
|
||||
)
|
||||
except SSRFError as e:
|
||||
raise _refused_audio_url(e) from e
|
||||
return _binary_audio(audio_response, usage)
|
||||
|
||||
@override
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract
|
||||
) -> BaseLLMException:
|
||||
return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers))
|
||||
0
litellm/llms/eu_jev/__init__.py
Normal file
0
litellm/llms/eu_jev/__init__.py
Normal file
34
litellm/llms/eu_jev/common_utils.py
Normal file
34
litellm/llms/eu_jev/common_utils.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
|
||||
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
|
||||
_EU_JEV_PREFIX: Final = "eu_jev/"
|
||||
|
||||
|
||||
def eu_jev_registry_model(requested_model: str | None) -> str:
|
||||
release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None
|
||||
if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES:
|
||||
return EU_JEV_LATEST_MODEL
|
||||
return release
|
||||
|
||||
|
||||
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
|
||||
requested_model: Final = decision_body.get("model")
|
||||
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
|
||||
|
||||
|
||||
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
hook_model: Final = hook_body.get("model")
|
||||
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
|
||||
if hook_model == eu_jev_scoped_model(client_body):
|
||||
client_model: Final = client_body.get("model")
|
||||
if client_model is None:
|
||||
return MappingProxyType(without_model)
|
||||
if isinstance(client_model, str):
|
||||
return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)})
|
||||
return MappingProxyType({**without_model, "model": client_model})
|
||||
if isinstance(hook_model, str):
|
||||
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
|
||||
return hook_body
|
||||
9
litellm/llms/eu_jev/decisions/transformation.py
Normal file
9
litellm/llms/eu_jev/decisions/transformation.py
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
|
||||
|
||||
|
||||
class EuJevDecisionsConfig(BaseDecisionsConfig):
|
||||
api_key_env = ("EU_JEV_API_KEY",)
|
||||
api_base_env = ("EU_JEV_API_BASE",)
|
||||
|
||||
def get_default_api_base(self) -> str | None:
|
||||
return "https://eu-jev.com"
|
||||
|
|
@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod
|
|||
This config enables pass-through behavior to the proxy's /v1/responses endpoint.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _openai_sdk_error_body(
|
||||
error_message: str,
|
||||
) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict
|
||||
try:
|
||||
decoded: Final = _ERROR_BODY.validate_json(error_message)
|
||||
except ValidationError:
|
||||
return None
|
||||
nested: Final = decoded.get("error")
|
||||
return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded
|
||||
|
||||
|
||||
class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
||||
"""
|
||||
|
|
@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
return join_api_base(api_base, "responses")
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract
|
||||
) -> BaseLLMException:
|
||||
return BaseLLMException(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
body=_openai_sdk_error_body(error_message),
|
||||
)
|
||||
|
||||
def supports_native_websocket(self) -> bool:
|
||||
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy
|
||||
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
|
||||
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
|
||||
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
|
||||
LiteLLMAnthropicToResponsesAPIAdapter,
|
||||
)
|
||||
|
|
@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod
|
|||
)
|
||||
|
||||
|
||||
async def _sent_messages(
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
system: object,
|
||||
messages_format: TokenCountMessagesFormat,
|
||||
context_management: CountContextManagement | None,
|
||||
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
|
||||
match messages_format:
|
||||
case "openai":
|
||||
return messages
|
||||
case "anthropic":
|
||||
try:
|
||||
return await count_view_messages(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
context_management=context_management,
|
||||
)
|
||||
except AnthropicContextManagementError as error:
|
||||
return error
|
||||
case _:
|
||||
assert_never(messages_format)
|
||||
|
||||
|
||||
class OpenAITokenCounter(BaseTokenCounter):
|
||||
"""Token counter implementation for OpenAI provider using the Responses API."""
|
||||
|
||||
|
|
@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter):
|
|||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
sent_messages: Final = await _sent_messages(
|
||||
model_to_use, messages, tools, system, messages_format, context_management
|
||||
)
|
||||
if isinstance(sent_messages, AnthropicContextManagementError):
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
request_model=request_model,
|
||||
model_used=model_to_use,
|
||||
tokenizer_type="openai_api",
|
||||
error=True,
|
||||
error_message=sent_messages.message,
|
||||
status_code=sent_messages.status_code,
|
||||
rejected_before_provider_call=True,
|
||||
)
|
||||
|
||||
resolved: Final = _count_credential(deployment, request_model or model_to_use)
|
||||
if isinstance(resolved, OpenAIError):
|
||||
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
|
||||
|
||||
input_items, instructions, counted_tools, text = self._responses_request_parts(
|
||||
model_to_use=model_to_use,
|
||||
messages=messages,
|
||||
messages=sent_messages,
|
||||
tools=tools,
|
||||
system=system,
|
||||
messages_format=messages_format,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
|
||||
)
|
||||
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
|
||||
from litellm.llms.laya.common_utils import laya_response_model
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
|
||||
|
|
@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler:
|
|||
response_model: Final = (
|
||||
laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model
|
||||
)
|
||||
logged_model: Final = response_model or request_model or "unknown"
|
||||
registry_request_model: Final = (
|
||||
eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model
|
||||
)
|
||||
logged_model: Final = response_model or registry_request_model or "unknown"
|
||||
model_name: Final = f"{custom_llm_provider}/{logged_model}"
|
||||
usage: Final = response.usage or _TypeSafeUsage()
|
||||
input_tokens: Final = usage.input_tokens
|
||||
output_tokens: Final = usage.output_tokens
|
||||
candidate_models: Final = (
|
||||
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
|
||||
if custom_llm_provider == "eu_jev"
|
||||
else (response_model, registry_request_model)
|
||||
)
|
||||
candidate_model_keys: Final = tuple(
|
||||
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
|
||||
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
|
||||
)
|
||||
pricing: Final = _pricing_for(candidate_model_keys)
|
||||
response_cost: Final = (
|
||||
|
|
|
|||
|
|
@ -7975,6 +7975,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream_options=kwargs.get("stream_options"),
|
||||
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
|
||||
and kwargs["_litellm_strip_stream_usage"] is True,
|
||||
)
|
||||
else:
|
||||
## OpenAI / Azure Text Completion Returns here
|
||||
|
|
@ -8824,8 +8826,9 @@ def speech(
|
|||
extra_headers: Final = kwargs.get("extra_headers", None)
|
||||
model_info: Final = kwargs.get("model_info", None)
|
||||
shared_session: Final = kwargs.get("shared_session", None)
|
||||
caller_api_base: Final = api_base or base_url
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url
|
||||
model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base
|
||||
)
|
||||
kwargs.pop("tags", [])
|
||||
|
||||
|
|
@ -8880,13 +8883,13 @@ def speech(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
|
||||
if custom_llm_provider == "edenai":
|
||||
litellm_params_dict["api_base"] = api_base
|
||||
if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None:
|
||||
litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base
|
||||
response = base_llm_http_handler.text_to_speech_handler(
|
||||
model=model,
|
||||
input=input,
|
||||
voice=voice if isinstance(voice, str) else None,
|
||||
text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(),
|
||||
text_to_speech_provider_config=text_to_speech_provider_config,
|
||||
text_to_speech_optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params_dict,
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
|||
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
|
||||
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
|
||||
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
|
||||
Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -18346,25 +18346,31 @@
|
|||
"dashscope/qwen-image-2.0": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.035,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-2.0-pro": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.075,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-3.0": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.03,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-3.0-pro": {
|
||||
|
|
@ -18372,7 +18378,54 @@
|
|||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen3-tts-flash": {
|
||||
"input_cost_per_character": 1e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.1-tts-flash": {
|
||||
"input_cost_per_token": 2.3e-07,
|
||||
"output_cost_per_token": 1.87e-06,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.1-tts-plus": {
|
||||
"input_cost_per_character": 3e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.0-tts-plus": {
|
||||
"input_cost_per_character": 2e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.0-tts-flash": {
|
||||
"input_cost_per_character": 1.5e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/wan3.0-video": {
|
||||
|
|
@ -72330,6 +72383,28 @@
|
|||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"eu_jev/eu-jeff-1.0.0": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
"litellm_provider": "eu_jev",
|
||||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
},
|
||||
"eu_jev/eu-jeff-latest": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
"litellm_provider": "eu_jev",
|
||||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
|
|||
|
|
@ -1148,9 +1148,9 @@
|
|||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": true,
|
||||
"image_edits": false,
|
||||
"image_edits": true,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"audio_speech": true,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
|
|
@ -2241,6 +2241,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"eu_jev": {
|
||||
"display_name": "eu/jev (`eu_jev`)",
|
||||
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
|
||||
"endpoints": {
|
||||
"chat_completions": false,
|
||||
"responses": false,
|
||||
"text_completion": false,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": false,
|
||||
"interactions": false,
|
||||
"systemone": true,
|
||||
"a2a": false,
|
||||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"laya": {
|
||||
"display_name": "Laya (`laya`)",
|
||||
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",
|
||||
|
|
|
|||
|
|
@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
"/tinyfish/",
|
||||
"/transcribe",
|
||||
"/typesafe/",
|
||||
"/eu_jev/",
|
||||
"/laya/",
|
||||
"/bespoke/",
|
||||
"/openrouter/",
|
||||
|
|
|
|||
|
|
@ -23170,6 +23170,39 @@
|
|||
"title": "CachedTokensDetails",
|
||||
"type": "object"
|
||||
},
|
||||
"ChoiceAnswer": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"choice": {
|
||||
"title": "Choice",
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"title": "Confidence",
|
||||
"type": "number"
|
||||
},
|
||||
"probabilities": {
|
||||
"additionalProperties": {
|
||||
"type": "number"
|
||||
},
|
||||
"title": "Probabilities",
|
||||
"type": "object"
|
||||
},
|
||||
"type": {
|
||||
"const": "choice",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"choice",
|
||||
"confidence",
|
||||
"probabilities"
|
||||
],
|
||||
"title": "ChoiceAnswer",
|
||||
"type": "object"
|
||||
},
|
||||
"Click": {
|
||||
"additionalProperties": true,
|
||||
"description": "A click action.",
|
||||
|
|
@ -23934,6 +23967,112 @@
|
|||
"title": "ErrorResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"EuJevDecisionResponse": {
|
||||
"properties": {
|
||||
"answers": {
|
||||
"additionalProperties": {
|
||||
"discriminator": {
|
||||
"mapping": {
|
||||
"choice": "#/components/schemas/ChoiceAnswer",
|
||||
"noul": "#/components/schemas/NoulAnswer",
|
||||
"score": "#/components/schemas/ScoreAnswer"
|
||||
},
|
||||
"propertyName": "type"
|
||||
},
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/ChoiceAnswer"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/NoulAnswer"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ScoreAnswer"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "One answer per question, under the same names",
|
||||
"title": "Answers",
|
||||
"type": "object"
|
||||
},
|
||||
"meta": {
|
||||
"$ref": "#/components/schemas/EuJevMeta"
|
||||
},
|
||||
"model": {
|
||||
"description": "The release that answered, such as eu-jeff-1.0.0",
|
||||
"title": "Model",
|
||||
"type": "string"
|
||||
},
|
||||
"usage": {
|
||||
"$ref": "#/components/schemas/EuJevUsage"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"model",
|
||||
"answers",
|
||||
"usage",
|
||||
"meta"
|
||||
],
|
||||
"title": "EuJevDecisionResponse",
|
||||
"type": "object"
|
||||
},
|
||||
"EuJevMeta": {
|
||||
"properties": {
|
||||
"cost_eur": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "The exact charge in euros, on authenticated requests",
|
||||
"title": "Cost Eur"
|
||||
},
|
||||
"latency_ms": {
|
||||
"description": "Server-side time for token quoting, credit reservation and inference",
|
||||
"title": "Latency Ms",
|
||||
"type": "integer"
|
||||
},
|
||||
"mode": {
|
||||
"const": "live",
|
||||
"title": "Mode",
|
||||
"type": "string"
|
||||
},
|
||||
"request_id": {
|
||||
"title": "Request Id",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"request_id",
|
||||
"mode",
|
||||
"latency_ms"
|
||||
],
|
||||
"title": "EuJevMeta",
|
||||
"type": "object"
|
||||
},
|
||||
"EuJevUsage": {
|
||||
"properties": {
|
||||
"input_tokens": {
|
||||
"minimum": 0.0,
|
||||
"title": "Input Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"output_tokens": {
|
||||
"minimum": 0.0,
|
||||
"title": "Output Tokens",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input_tokens",
|
||||
"output_tokens"
|
||||
],
|
||||
"title": "EuJevUsage",
|
||||
"type": "object"
|
||||
},
|
||||
"FileSearchTool": {
|
||||
"additionalProperties": true,
|
||||
"description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).",
|
||||
|
|
@ -25594,6 +25733,26 @@
|
|||
"title": "NamespaceTool",
|
||||
"type": "object"
|
||||
},
|
||||
"NoulAnswer": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"noul": {
|
||||
"title": "Noul",
|
||||
"type": "number"
|
||||
},
|
||||
"type": {
|
||||
"const": "noul",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"noul"
|
||||
],
|
||||
"title": "NoulAnswer",
|
||||
"type": "object"
|
||||
},
|
||||
"OperationCreateFile": {
|
||||
"additionalProperties": true,
|
||||
"description": "Instruction describing how to create a file via the apply_patch tool.",
|
||||
|
|
@ -28866,6 +29025,59 @@
|
|||
"title": "Result",
|
||||
"type": "object"
|
||||
},
|
||||
"ScoreAnswer": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"confidence": {
|
||||
"title": "Confidence",
|
||||
"type": "number"
|
||||
},
|
||||
"legend": {
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Legend",
|
||||
"type": "object"
|
||||
},
|
||||
"probabilities": {
|
||||
"additionalProperties": {
|
||||
"type": "number"
|
||||
},
|
||||
"title": "Probabilities",
|
||||
"type": "object"
|
||||
},
|
||||
"score": {
|
||||
"title": "Score",
|
||||
"type": "number"
|
||||
},
|
||||
"type": {
|
||||
"const": "score",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"score",
|
||||
"confidence",
|
||||
"legend",
|
||||
"probabilities"
|
||||
],
|
||||
"title": "ScoreAnswer",
|
||||
"type": "object"
|
||||
},
|
||||
"Screenshot": {
|
||||
"additionalProperties": true,
|
||||
"description": "A screenshot action.",
|
||||
|
|
@ -31758,6 +31970,315 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"/eu_jev/v1/systemone": {
|
||||
"post": {
|
||||
"description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)",
|
||||
"operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"example": {
|
||||
"model": "eu-jeff-latest",
|
||||
"questions": {
|
||||
"frustration": {
|
||||
"criteria": [
|
||||
"calm",
|
||||
"mildly annoyed",
|
||||
"angry"
|
||||
],
|
||||
"instructions": "How frustrated is the customer?",
|
||||
"type": "score"
|
||||
},
|
||||
"is_refund_request": {
|
||||
"instructions": "Is the customer asking for a refund?",
|
||||
"type": "noul"
|
||||
},
|
||||
"team": {
|
||||
"criteria": {
|
||||
"billing": "Payments, invoices, and refunds",
|
||||
"support": "Technical issues and bugs"
|
||||
},
|
||||
"instructions": "Which team should handle this?",
|
||||
"type": "choice"
|
||||
}
|
||||
},
|
||||
"state": "I was charged twice. Can I get a refund?"
|
||||
},
|
||||
"schema": {
|
||||
"properties": {
|
||||
"model": {
|
||||
"default": "eu-jeff-latest",
|
||||
"description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it",
|
||||
"title": "Model",
|
||||
"type": "string"
|
||||
},
|
||||
"questions": {
|
||||
"additionalProperties": {
|
||||
"discriminator": {
|
||||
"propertyName": "type"
|
||||
},
|
||||
"oneOf": [
|
||||
{
|
||||
"properties": {
|
||||
"criteria": {
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
|
||||
},
|
||||
"description": "Two to 64 options, option name to description. A null description uses the option name",
|
||||
"maxProperties": 64,
|
||||
"minProperties": 2,
|
||||
"title": "Criteria",
|
||||
"type": "object"
|
||||
},
|
||||
"instructions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
|
||||
"title": "Instructions"
|
||||
},
|
||||
"type": {
|
||||
"const": "choice",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"instructions",
|
||||
"criteria"
|
||||
],
|
||||
"title": "EuJevChoiceQuestion",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"criteria": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"propertyNames": {
|
||||
"enum": [
|
||||
"true",
|
||||
"false"
|
||||
]
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional descriptions of what true and false mean",
|
||||
"title": "Criteria"
|
||||
},
|
||||
"instructions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
|
||||
"title": "Instructions"
|
||||
},
|
||||
"type": {
|
||||
"const": "noul",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"instructions"
|
||||
],
|
||||
"title": "EuJevNoulQuestion",
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"properties": {
|
||||
"criteria": {
|
||||
"description": "Two to 10 level descriptions, ordered from low to high",
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
|
||||
},
|
||||
"maxItems": 10,
|
||||
"minItems": 2,
|
||||
"title": "Criteria",
|
||||
"type": "array"
|
||||
},
|
||||
"instructions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
|
||||
"title": "Instructions"
|
||||
},
|
||||
"type": {
|
||||
"const": "score",
|
||||
"title": "Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"type",
|
||||
"instructions",
|
||||
"criteria"
|
||||
],
|
||||
"title": "EuJevScoreQuestion",
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "One to eight questions, keyed by names of up to 80 characters",
|
||||
"maxProperties": 8,
|
||||
"minProperties": 1,
|
||||
"propertyNames": {
|
||||
"maxLength": 80,
|
||||
"minLength": 1
|
||||
},
|
||||
"title": "Questions",
|
||||
"type": "object"
|
||||
},
|
||||
"state": {
|
||||
"anyOf": [
|
||||
{
|
||||
"maxLength": 100000,
|
||||
"minLength": 1,
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
}
|
||||
],
|
||||
"description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array",
|
||||
"title": "State"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"state",
|
||||
"questions"
|
||||
],
|
||||
"title": "EuJevDecisionRequest",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/EuJevDecisionResponse"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Eu Jev Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/fal_ai/{endpoint}": {
|
||||
"delete": {
|
||||
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor
|
|||
)
|
||||
from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException
|
||||
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError
|
||||
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.types.services import ServiceTypes
|
||||
|
||||
|
|
@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
|
|||
message=str(e.detail),
|
||||
internal_message=e.internal_message,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=e.status_code,
|
||||
)
|
||||
if isinstance(e, HTTPException):
|
||||
return ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
|
|
@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
|
|||
return ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import (
|
|||
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
|
||||
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
|
||||
|
|
@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str):
|
|||
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"]
|
||||
```
|
||||
"""
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.proxy_server import general_settings, premium_user
|
||||
|
||||
try:
|
||||
|
|
@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request(
|
|||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
|
||||
if RouteChecks.is_builtin_eu_jev_decision_route(route=route):
|
||||
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
|
||||
|
||||
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
|
||||
if requested_model is not None and not isinstance(requested_model, str):
|
||||
raise HTTPException(status_code=400, detail="eu_jev model must be a string")
|
||||
return [f"eu_jev/{eu_jev_registry_model(requested_model)}"]
|
||||
if route == "/cost/predict-cache":
|
||||
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
|
||||
return _dedupe_model_candidates(prediction_models)
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException:
|
|||
f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}"
|
||||
),
|
||||
type=ProxyErrorTypes.auth_provider_unavailable,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,11 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
|
||||
|
||||
|
||||
class OAuth2TokenValidationError(ValueError):
|
||||
"""The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it
|
||||
inactive, or could not be reached."""
|
||||
|
||||
|
||||
class Oauth2Handler:
|
||||
"""
|
||||
Handles OAuth2 token validation.
|
||||
|
|
@ -207,7 +212,7 @@ class Oauth2Handler:
|
|||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
# This will catch any 4xx or 5xx errors
|
||||
raise ValueError(f"Oauth 2.0 Token validation failed: {e}")
|
||||
raise OAuth2TokenValidationError(f"Oauth 2.0 Token validation failed: {e}")
|
||||
except Exception as e:
|
||||
# This will catch any other errors (like network issues)
|
||||
raise ValueError(f"An error occurred during token validation: {e}")
|
||||
raise OAuth2TokenValidationError(f"An error occurred during token validation: {e}")
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
|
|||
_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend")
|
||||
|
||||
_LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp"))
|
||||
_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone"
|
||||
_AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS))
|
||||
|
||||
# Writes that only touch the caller's own credential row (BYOK connections and
|
||||
|
|
@ -301,7 +302,7 @@ class RouteChecks:
|
|||
f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}"
|
||||
),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -503,7 +504,19 @@ class RouteChecks:
|
|||
for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value:
|
||||
if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"):
|
||||
return True
|
||||
return False
|
||||
return RouteChecks.is_builtin_eu_jev_decision_route(route=route)
|
||||
|
||||
@staticmethod
|
||||
def is_builtin_eu_jev_decision_route(route: str) -> bool:
|
||||
if route.rstrip("/") != _EU_JEV_DECISION_ROUTE:
|
||||
return False
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack
|
||||
InitPassThroughEndpointHelpers,
|
||||
)
|
||||
|
||||
return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
|
||||
route=_EU_JEV_DECISION_ROUTE
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool:
|
||||
|
|
|
|||
|
|
@ -1160,6 +1160,24 @@ def _matches_routing_override(token_claims: dict, override: "JWTRoutingOverride"
|
|||
)
|
||||
|
||||
|
||||
def oauth2_introspection_applies(
|
||||
api_key: str, route: str, *, enable_oauth2_auth: bool, enable_jwt_auth: bool, jwt_handler: JWTHandler
|
||||
) -> bool:
|
||||
"""Whether LiteLLM validates ``api_key`` on ``route`` by OAuth2 token introspection, on LLM API, info and
|
||||
lens routes only: a non-JWT token when OAuth2 auth is on globally, or a JWT matched by a routing override."""
|
||||
is_jwt: Final = enable_jwt_auth and jwt_handler.is_jwt(token=api_key)
|
||||
# Routing uses unverified JWT claims only to choose auth path.
|
||||
# Final authentication is enforced by the selected validator.
|
||||
route_jwt_to_oauth2: Final = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler)
|
||||
covered_route: Final = (
|
||||
RouteChecks.is_llm_api_route(route=route)
|
||||
or RouteChecks.is_info_route(route=route)
|
||||
or route == "/lens"
|
||||
or route.startswith("/lens/")
|
||||
)
|
||||
return covered_route and ((enable_oauth2_auth and not is_jwt) or route_jwt_to_oauth2)
|
||||
|
||||
|
||||
def _should_route_jwt_to_oauth2_override(token: str, jwt_handler: JWTHandler) -> bool:
|
||||
routing_overrides: Final = jwt_handler.litellm_jwtauth.routing_overrides
|
||||
if not routing_overrides:
|
||||
|
|
@ -2278,23 +2296,13 @@ async def user_api_key_auth_builder(
|
|||
|
||||
enable_oauth2_auth: Final = general_settings.get("enable_oauth2_auth", False) is True
|
||||
enable_jwt_auth: Final = general_settings.get("enable_jwt_auth", False) is True
|
||||
is_jwt = jwt_handler.is_jwt(token=api_key) if enable_jwt_auth else False
|
||||
|
||||
# Routing uses unverified JWT claims only to choose auth path.
|
||||
# Final authentication is enforced by the selected validator.
|
||||
route_jwt_to_oauth2 = is_jwt and _should_route_jwt_to_oauth2_override(token=api_key, jwt_handler=jwt_handler)
|
||||
|
||||
# OAuth2 applies for:
|
||||
# 1) when global OAuth2 auth is enabled on LLM + info routes
|
||||
# 2) JWT tokens that explicitly match routing_overrides on LLM + info routes
|
||||
route_is_lens: Final = route == "/lens" or route.startswith("/lens/")
|
||||
should_apply_override_oauth2: Final = route_jwt_to_oauth2 and (
|
||||
RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens
|
||||
)
|
||||
should_apply_global_oauth2: Final = enable_oauth2_auth and (
|
||||
RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route) or route_is_lens
|
||||
)
|
||||
if (should_apply_global_oauth2 and not is_jwt) or should_apply_override_oauth2:
|
||||
if oauth2_introspection_applies(
|
||||
api_key,
|
||||
route,
|
||||
enable_oauth2_auth=enable_oauth2_auth,
|
||||
enable_jwt_auth=enable_jwt_auth,
|
||||
jwt_handler=jwt_handler,
|
||||
):
|
||||
from litellm.proxy.proxy_server import premium_user
|
||||
|
||||
if premium_user is not True:
|
||||
|
|
@ -2578,7 +2586,7 @@ async def user_api_key_auth_builder(
|
|||
raise ProxyException(
|
||||
message="Authentication Error, No api key passed in.",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
elif api_key == "":
|
||||
|
|
@ -2586,7 +2594,7 @@ async def user_api_key_auth_builder(
|
|||
raise ProxyException(
|
||||
message="Authentication Error, Malformed API Key passed in. Ensure Key has `Bearer ` prefix.",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -2827,7 +2835,7 @@ async def user_api_key_auth_builder(
|
|||
raise ProxyException(
|
||||
message=f"Authentication Error, Tried to access route={route}, which is only for MASTER KEY",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -2960,7 +2968,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
raise ProxyException(
|
||||
message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
await enforce_key_and_fallback_model_access(
|
||||
|
|
@ -3011,7 +3019,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
"Keys owned by this user cannot be used."
|
||||
),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -3317,7 +3325,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
raise ProxyException(
|
||||
message="Authentication Error, Invalid proxy server token passed",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
if valid_token_dict is not None:
|
||||
|
|
|
|||
|
|
@ -492,7 +492,7 @@ lite codex exec "summarize the repo"
|
|||
|
||||
Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
|
||||
|
||||
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
|
||||
The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug, plus whichever reasoning levels its model group lists on `/model_group/info` the installed Codex can parse. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
|
||||
|
||||
pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. The entry also turns on pi's session header (`x-session-id`, sent by pi 0.80.7 and later), so every chat turn of one pi session shares one session id in the logs and sticks to one deployment when session affinity is on. A custom `pi --session-id` must be at least 8 letters, digits, `_`, or `-` for the proxy to read it. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from dataclasses import dataclass
|
|||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, TypeVar
|
||||
from typing import Final, Literal, TypeAlias, TypeVar
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import click
|
||||
|
|
@ -41,6 +41,7 @@ from .pi import (
|
|||
LITELLM_PROXY_API_KEY_ENV,
|
||||
PI_PROVIDER_NAME,
|
||||
ListingFailure,
|
||||
ModelLimits,
|
||||
PiSyncError,
|
||||
fetch_model_ids,
|
||||
fetch_model_limits,
|
||||
|
|
@ -611,9 +612,27 @@ class _OpenCodeLimit(LiteLLMBaseModel):
|
|||
output: int
|
||||
|
||||
|
||||
class _OpenCodeVariant(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
reasoningEffort: str
|
||||
|
||||
|
||||
class _OpenCodeReasoningOption(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
type: Literal["effort"] = "effort"
|
||||
values: tuple[str, ...]
|
||||
|
||||
|
||||
class _OpenCodeModel(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
name: str
|
||||
limit: _OpenCodeLimit | None = None
|
||||
reasoning: Literal[True] | None = None
|
||||
reasoning_options: tuple[_OpenCodeReasoningOption, ...] | None = None
|
||||
variants: Mapping[str, _OpenCodeVariant] | None = None
|
||||
|
||||
|
||||
class _OpenCodeProviderOptions(LiteLLMBaseModel):
|
||||
|
|
@ -632,15 +651,43 @@ class _OpenCodeConfig(LiteLLMBaseModel):
|
|||
provider: Mapping[str, _OpenCodeProvider]
|
||||
|
||||
|
||||
def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel:
|
||||
if model.max_input_tokens is None or model.max_output_tokens is None:
|
||||
return _OpenCodeModel(name=model.id)
|
||||
_NO_GROUPS: Final[Mapping[str, ModelLimits]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _reasoning_efforts(model_id: str, groups: Mapping[str, ModelLimits]) -> tuple[str, ...] | None:
|
||||
group: Final = groups.get(model_id)
|
||||
return group.supported_reasoning_efforts if group is not None else None
|
||||
|
||||
|
||||
def _opencode_model_entry(model: ListedModel, groups: Mapping[str, ModelLimits]) -> _OpenCodeModel:
|
||||
"""One OpenCode model entry, with the model group's reasoning efforts as its variants when known.
|
||||
|
||||
OpenCode 1.x builds its own variants for a reasoning model and replaces them only through
|
||||
`reasoning_options`. 2.x ignores `reasoning` and `reasoning_options` and builds its own variants
|
||||
unless the entry lists some, so the efforts go in all three, and a model with none lists an
|
||||
empty `variants`."""
|
||||
limit: Final = (
|
||||
_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens)
|
||||
if model.max_input_tokens is not None and model.max_output_tokens is not None
|
||||
else None
|
||||
)
|
||||
efforts: Final = _reasoning_efforts(model.id, groups)
|
||||
if efforts is None:
|
||||
return _OpenCodeModel(name=model.id, limit=limit)
|
||||
if not efforts:
|
||||
return _OpenCodeModel(name=model.id, limit=limit, variants=MappingProxyType({}))
|
||||
return _OpenCodeModel(
|
||||
name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens)
|
||||
name=model.id,
|
||||
limit=limit,
|
||||
reasoning=True,
|
||||
reasoning_options=(_OpenCodeReasoningOption(values=efforts),),
|
||||
variants=MappingProxyType({effort: _OpenCodeVariant(reasoningEffort=effort) for effort in efforts}),
|
||||
)
|
||||
|
||||
|
||||
def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str:
|
||||
def opencode_provider_config(
|
||||
base_url: str, models: Sequence[ListedModel], groups: Mapping[str, ModelLimits] = _NO_GROUPS
|
||||
) -> str:
|
||||
"""OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`.
|
||||
|
||||
One model entry per chat-capable /v1/models row (mode chat, responses, or
|
||||
|
|
@ -648,6 +695,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
|
|||
is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so
|
||||
it never lands in the config text. OpenCode merges this inline config over
|
||||
the user's own files, leaving unrelated keys and providers untouched.
|
||||
`groups` is /model_group/info by model group, where the reasoning efforts come from.
|
||||
"""
|
||||
chat_models: Final = _chat_models(models)
|
||||
provider: Final = _OpenCodeProvider(
|
||||
|
|
@ -657,7 +705,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
|
|||
baseURL=base_url.rstrip("/") + "/v1",
|
||||
apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}",
|
||||
),
|
||||
models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}),
|
||||
models=MappingProxyType({m.id: _opencode_model_entry(m, groups) for m in chat_models}),
|
||||
)
|
||||
config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider}))
|
||||
return config.model_dump_json(exclude_none=True)
|
||||
|
|
@ -672,7 +720,8 @@ def opencode_model_sync_env(
|
|||
) -> Mapping[str, str] | ModelSyncSkipped:
|
||||
"""Env addition that hands OpenCode the proxy's model list, or why it was skipped.
|
||||
|
||||
Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT.
|
||||
Fetches /v1/models with the key, and /model_group/info for each model's reasoning
|
||||
efforts, and packs them into OPENCODE_CONFIG_CONTENT.
|
||||
An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a
|
||||
failed fetch is reported rather than raised: OpenCode still launches on the
|
||||
plain OPENAI_* env, just without a synced model list.
|
||||
|
|
@ -682,16 +731,20 @@ def opencode_model_sync_env(
|
|||
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
|
||||
if isinstance(listing, ModelSyncSkipped):
|
||||
return listing
|
||||
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
|
||||
groups: Final = fetch_model_limits(base_url, api_key, get=get)
|
||||
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing, groups)})
|
||||
|
||||
|
||||
def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow:
|
||||
def _codex_row(
|
||||
model: ListedModel, upstream: ModelUpstream | None, groups: Mapping[str, ModelLimits]
|
||||
) -> CodexCatalogRow:
|
||||
return CodexCatalogRow(
|
||||
id=model.id,
|
||||
mode=model.mode,
|
||||
max_input_tokens=model.max_input_tokens,
|
||||
upstream_model=upstream.model if upstream is not None else None,
|
||||
reasoning_effort=upstream.reasoning_effort if upstream is not None else None,
|
||||
supported_reasoning_efforts=_reasoning_efforts(model.id, groups),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -700,6 +753,8 @@ def codex_model_catalog(
|
|||
stock: Sequence[CodexStockModel],
|
||||
instructions: str,
|
||||
upstreams: Mapping[str, ModelUpstream],
|
||||
groups: Mapping[str, ModelLimits] = _NO_GROUPS,
|
||||
client_version: str | None = None,
|
||||
) -> str | None:
|
||||
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
|
||||
|
||||
|
|
@ -714,17 +769,19 @@ def codex_model_catalog(
|
|||
deployments pin, and a window no larger than its listed max_input_tokens.
|
||||
A model Codex does not know gets the fallback
|
||||
entry, with the same base instructions Codex itself uses so the agent never
|
||||
runs without a system prompt. Apart from aliases, the catalog is the one
|
||||
`/v1/models?client_version=...` serves, built from the installed Codex's own
|
||||
entries instead of the proxy's vendored ones.
|
||||
runs without a system prompt, and offers the reasoning efforts `groups` gives its model group
|
||||
that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is
|
||||
the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries
|
||||
instead of the proxy's vendored ones.
|
||||
"""
|
||||
rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models)
|
||||
rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models)
|
||||
body: Final = codex_models_response_json(
|
||||
rows,
|
||||
stock=MappingProxyType({m.slug: m for m in stock}),
|
||||
instructions=instructions,
|
||||
byte_limit=None,
|
||||
alias_defaults="deployment",
|
||||
client_version=client_version,
|
||||
)
|
||||
return body.json if body.listed else None
|
||||
|
||||
|
|
@ -771,7 +828,7 @@ def _codex_debug_models(
|
|||
encoding="utf-8",
|
||||
timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as e:
|
||||
except (OSError, subprocess.TimeoutExpired, UnicodeError) as e:
|
||||
return ModelSyncSkipped(f"`{name} debug models` failed: {e}")
|
||||
if completed.returncode == 0:
|
||||
return completed.stdout
|
||||
|
|
@ -793,6 +850,25 @@ def _stock_codex_models(
|
|||
return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
|
||||
|
||||
|
||||
def _codex_version(
|
||||
binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
|
||||
) -> str | None:
|
||||
"""The version `codex --version` prints (`codex-cli 0.165.0`), or None when it cannot be read."""
|
||||
try:
|
||||
completed: Final = run(
|
||||
_windows_command(binary, (binary, "--version")),
|
||||
env=dict(env),
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
encoding="utf-8",
|
||||
timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired, UnicodeError):
|
||||
return None
|
||||
words: Final = completed.stdout.split()
|
||||
return words[-1] if completed.returncode == 0 and words else None
|
||||
|
||||
|
||||
def codex_model_sync_args(
|
||||
base_env: Mapping[str, str],
|
||||
base_url: str,
|
||||
|
|
@ -811,7 +887,8 @@ def codex_model_sync_args(
|
|||
atomically replaced on every launch. The Codex at `binary` first lists its
|
||||
own models, so the ones the proxy serves keep that Codex's entries, an alias
|
||||
included when every unpaused deployment /v1/model/info lists for it routes
|
||||
to the same OpenAI model and Codex knows that model, and then
|
||||
to the same OpenAI model and Codex knows that model, and its version, so a
|
||||
model it does not know offers only reasoning efforts it can parse, and then
|
||||
reads the file back once before it is handed over. The key never lands in
|
||||
the file. A failed fetch, read, listing, write or read-back is reported
|
||||
rather than raised: Codex still launches with its built-in catalog and takes
|
||||
|
|
@ -834,7 +911,9 @@ def codex_model_sync_args(
|
|||
if isinstance(stock, ModelSyncSkipped):
|
||||
return stock
|
||||
upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get)
|
||||
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams)
|
||||
groups: Final = fetch_model_limits(base_url, api_key, get=get)
|
||||
version: Final = _codex_version(binary, base_env, run=run)
|
||||
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams, groups, version)
|
||||
if catalog is None:
|
||||
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -50,8 +50,11 @@ class PiSyncError:
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelLimits:
|
||||
"""What /model_group/info says about one model group; `supported_reasoning_efforts` is None when unknown."""
|
||||
|
||||
context_window: int | None
|
||||
max_tokens: int | None
|
||||
supported_reasoning_efforts: tuple[str, ...] | None = None
|
||||
|
||||
|
||||
_NonEmptyString = Annotated[str, StringConstraints(min_length=1)]
|
||||
|
|
@ -76,9 +79,12 @@ class _ModelList(LiteLLMBaseModel):
|
|||
|
||||
|
||||
class _ModelGroup(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model_group: str
|
||||
max_input_tokens: float | None = None
|
||||
max_output_tokens: float | None = None
|
||||
supported_reasoning_efforts: tuple[str, ...] | None = None
|
||||
|
||||
|
||||
class _ModelGroupList(LiteLLMBaseModel):
|
||||
|
|
@ -137,8 +143,8 @@ def fetch_model_limits(
|
|||
*,
|
||||
get: Callable[..., requests.Response] = requests.get,
|
||||
) -> Mapping[str, ModelLimits]:
|
||||
"""Best effort: pi falls back to its own defaults for models without limits,
|
||||
so an unavailable /model_group/info must not block the launch."""
|
||||
"""Best effort: pi, OpenCode and Codex fall back to their own defaults for models without
|
||||
limits or reasoning efforts, so an unavailable /model_group/info must not block the launch."""
|
||||
url: Final = base_url.rstrip("/") + "/model_group/info"
|
||||
try:
|
||||
resp: Final = get(
|
||||
|
|
@ -156,6 +162,7 @@ def fetch_model_limits(
|
|||
group.model_group: ModelLimits(
|
||||
context_window=int(group.max_input_tokens) if group.max_input_tokens else None,
|
||||
max_tokens=int(group.max_output_tokens) if group.max_output_tokens else None,
|
||||
supported_reasoning_efforts=group.supported_reasoning_efforts,
|
||||
)
|
||||
for group in listing.data
|
||||
}
|
||||
|
|
@ -168,6 +175,33 @@ def models_json_path(env: Mapping[str, str]) -> Path:
|
|||
return root / "models.json"
|
||||
|
||||
|
||||
_PI_LADDER_LEVELS: Final = ("minimal", "low", "medium", "high")
|
||||
_PI_OPT_IN_LEVELS: Final = ("xhigh", "max")
|
||||
_REASONING_OFF_EFFORT: Final = "none"
|
||||
|
||||
|
||||
def _thinking_level_map(efforts: tuple[str, ...]) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized
|
||||
"""Each level pi offers maps to a value: pi drops a level mapped to null and shows xhigh and max only when set."""
|
||||
return {
|
||||
"off": _REASONING_OFF_EFFORT if _REASONING_OFF_EFFORT in efforts else None,
|
||||
**{level: level if level in efforts else None for level in _PI_LADDER_LEVELS},
|
||||
**{level: level for level in _PI_OPT_IN_LEVELS if level in efforts},
|
||||
}
|
||||
|
||||
|
||||
def _reasoning_fields(limit: ModelLimits | None) -> Mapping[str, JsonValue]:
|
||||
efforts: Final = limit.supported_reasoning_efforts if limit else None
|
||||
if not efforts:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType(
|
||||
{
|
||||
"reasoning": True,
|
||||
"thinkingLevelMap": _thinking_level_map(efforts),
|
||||
"compat": {"supportsDeveloperRole": False},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _model_entry(
|
||||
model_id: str, limits: Mapping[str, ModelLimits]
|
||||
) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized
|
||||
|
|
@ -178,7 +212,7 @@ def _model_entry(
|
|||
output: Final[dict[str, JsonValue]] = ( # mutable-ok: JSON field
|
||||
{"maxTokens": limit.max_tokens} if limit and limit.max_tokens else {}
|
||||
)
|
||||
return {"id": model_id, **context, **output}
|
||||
return {"id": model_id, **context, **output, **_reasoning_fields(limit)}
|
||||
|
||||
|
||||
def provider_block(
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
error_status_code,
|
||||
openai_error_param,
|
||||
openai_error_type,
|
||||
relayed_provider_specific_fields,
|
||||
)
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING_BYTES,
|
||||
|
|
@ -588,6 +589,7 @@ def model_deployments_support_stream_options(
|
|||
|
||||
|
||||
_STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD)
|
||||
_STREAM_USAGE_TRACKED_ROUTES: Final = frozenset({"acompletion", "atext_completion"})
|
||||
|
||||
|
||||
def stream_usage_tracking_updates(
|
||||
|
|
@ -613,7 +615,7 @@ def stream_usage_tracking_updates(
|
|||
return scrub
|
||||
if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data:
|
||||
return {**scrub, "stream_options": {"include_usage": False}}
|
||||
if always_include is False or route_type != "acompletion":
|
||||
if always_include is False or route_type not in _STREAM_USAGE_TRACKED_ROUTES:
|
||||
return scrub
|
||||
if isinstance(stream_options, dict) and stream_options.get("include_usage") is True:
|
||||
return scrub
|
||||
|
|
@ -4461,7 +4463,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
param=openai_error_param(e),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=_code,
|
||||
provider_specific_fields=getattr(e, "provider_specific_fields", None),
|
||||
provider_specific_fields=getattr(e, "provider_specific_fields", None)
|
||||
or relayed_provider_specific_fields(e),
|
||||
headers=safe_headers,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo
|
|||
lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry
|
||||
Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's
|
||||
``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier``
|
||||
upstream.
|
||||
upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning
|
||||
levels, trimmed to the ones the requesting Codex release can parse.
|
||||
|
||||
The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from
|
||||
https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0)
|
||||
|
|
@ -26,13 +27,16 @@ from pathlib import Path
|
|||
from types import MappingProxyType, NoneType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError
|
||||
from packaging.version import Version
|
||||
from pydantic import ConfigDict, Field, StringConstraints, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort
|
||||
from litellm.types.llms.base_model import LiteLLMBaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
from litellm.router_api import RouterRegistry
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
|
||||
CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
|
||||
|
|
@ -43,6 +47,10 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
|||
_TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
|
||||
_BODY_PREFIX: Final = '{"models":['
|
||||
_BODY_SUFFIX: Final = "]}"
|
||||
_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0")
|
||||
_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"})
|
||||
_PREFERRED_DEFAULT_EFFORT: Final = "medium"
|
||||
_CODEX_UNSET_EFFORT: Final = "none"
|
||||
|
||||
|
||||
class CodexServiceTier(LiteLLMBaseModel):
|
||||
|
|
@ -67,6 +75,15 @@ class _ConfiguredTier(LiteLLMBaseModel):
|
|||
_CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...])
|
||||
|
||||
|
||||
class CodexFallbackReasoningLevel(LiteLLMBaseModel):
|
||||
"""A `ReasoningEffortPreset` as Codex reads it: picking it sends `effort` as the request's reasoning effort."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
effort: str
|
||||
description: str
|
||||
|
||||
|
||||
class CodexTruncationPolicy(LiteLLMBaseModel):
|
||||
mode: Literal["bytes"] = "bytes"
|
||||
limit: int = 10_000
|
||||
|
|
@ -82,10 +99,13 @@ class CodexFallbackModel(LiteLLMBaseModel):
|
|||
same as `codex -m` did.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
slug: str
|
||||
display_name: str
|
||||
description: NoneType = None
|
||||
supported_reasoning_levels: tuple[()] = ()
|
||||
default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None)
|
||||
supported_reasoning_levels: tuple[CodexFallbackReasoningLevel, ...] = ()
|
||||
shell_type: Literal["unified_exec"] = "unified_exec"
|
||||
visibility: Literal["list"] = "list"
|
||||
supported_in_api: Literal[True] = True
|
||||
|
|
@ -153,7 +173,8 @@ class CodexCatalogRow:
|
|||
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
|
||||
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
|
||||
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
|
||||
value per deployment behind the id, None where a deployment sets none."""
|
||||
value per deployment behind the id, None where a deployment sets none. `supported_reasoning_efforts`
|
||||
is the model group's, None when unknown."""
|
||||
|
||||
id: str
|
||||
mode: str | None = None
|
||||
|
|
@ -162,6 +183,7 @@ class CodexCatalogRow:
|
|||
reasoning_effort: str | None = None
|
||||
display_name: str | None = None
|
||||
service_tiers: tuple[object, ...] = ()
|
||||
supported_reasoning_efforts: tuple[str, ...] | None = None
|
||||
search_tool_support: tuple[bool | None, ...] = ()
|
||||
|
||||
|
||||
|
|
@ -261,8 +283,40 @@ def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None:
|
|||
return True if per_deployment and None not in per_deployment else None
|
||||
|
||||
|
||||
def _codex_can_drive(row: CodexCatalogRow) -> bool:
|
||||
return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
|
||||
def _codex_can_drive(model_id: str, mode: str | None) -> bool:
|
||||
return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES)
|
||||
|
||||
|
||||
def _parses_custom_efforts(client_version: str | None) -> bool:
|
||||
try:
|
||||
return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def codex_reasoning_levels(
|
||||
efforts: Sequence[str], client_version: str | None
|
||||
) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]:
|
||||
"""The reasoning levels a fallback entry offers for a model group's efforts, and its default level.
|
||||
|
||||
Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for
|
||||
a version that does not parse, only the efforts every release knows are offered. The default is
|
||||
set whenever a level is offered, because Codex reads a missing one as `none`: `none` itself when the
|
||||
group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the
|
||||
nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium,
|
||||
else the weakest declared level, else the first one."""
|
||||
parses_any: Final = _parses_custom_efforts(client_version)
|
||||
offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES)
|
||||
if not offered:
|
||||
return (), None
|
||||
nearest: Final = nearest_declared_reasoning_effort(_PREFERRED_DEFAULT_EFFORT, offered, "down")
|
||||
levels: Final = tuple(
|
||||
CodexFallbackReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream")
|
||||
for effort in offered
|
||||
)
|
||||
if _CODEX_UNSET_EFFORT in offered:
|
||||
return levels, _CODEX_UNSET_EFFORT
|
||||
return levels, nearest if nearest in offered else offered[0]
|
||||
|
||||
|
||||
def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None:
|
||||
|
|
@ -309,6 +363,7 @@ def _entry(
|
|||
served: frozenset[str],
|
||||
instructions: str,
|
||||
alias_defaults: CodexAliasDefaults,
|
||||
client_version: str | None,
|
||||
) -> CodexCatalogEntry:
|
||||
stock_model: Final = _stock_for(row, stock)
|
||||
stock_tiers: Final = MappingProxyType(
|
||||
|
|
@ -317,9 +372,12 @@ def _entry(
|
|||
configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
|
||||
search_tool: Final = offered_search_tool(row.search_tool_support)
|
||||
if stock_model is None:
|
||||
levels, default_level = codex_reasoning_levels(row.supported_reasoning_efforts or (), client_version)
|
||||
return CodexFallbackModel(
|
||||
slug=row.id,
|
||||
display_name=row.display_name if row.display_name is not None else row.id,
|
||||
default_reasoning_level=default_level,
|
||||
supported_reasoning_levels=levels,
|
||||
priority=index,
|
||||
service_tiers=configured if configured is not None else (),
|
||||
context_window=row.max_input_tokens,
|
||||
|
|
@ -406,6 +464,7 @@ def codex_models_response_json(
|
|||
instructions: str | None = None,
|
||||
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
|
||||
alias_defaults: CodexAliasDefaults = "codex",
|
||||
client_version: str | None = None,
|
||||
) -> CodexCatalogBody:
|
||||
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
|
||||
|
||||
|
|
@ -417,14 +476,15 @@ def codex_models_response_json(
|
|||
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
|
||||
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
|
||||
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
|
||||
them, as Codex would use them for the model's own name.
|
||||
them, as Codex would use them for the model's own name. `client_version` is the requesting Codex's version, which decides the reasoning
|
||||
levels a fallback entry may offer; None counts as a version too old for custom levels.
|
||||
"""
|
||||
known: Final = bundled_codex_models() if stock is None else stock
|
||||
prompt: Final = codex_base_instructions() if instructions is None else instructions
|
||||
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
|
||||
drivable: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode))
|
||||
served: Final = frozenset(row.id for row in drivable)
|
||||
entries: Final = tuple(
|
||||
_entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable)
|
||||
_entry(index, row, known, served, prompt, alias_defaults, client_version) for index, row in enumerate(drivable)
|
||||
)
|
||||
serialized: Final = tuple(_serialized(entry) for entry in entries)
|
||||
kept: Final = _kept_positions(entries, serialized, byte_limit)
|
||||
|
|
@ -435,6 +495,15 @@ def codex_models_response_json(
|
|||
)
|
||||
|
||||
|
||||
def _group_reasoning_efforts(
|
||||
row: ModelInfoResponse, lookup_id: str, registry: RouterRegistry | None, team_id: str | None
|
||||
) -> tuple[str, ...] | None:
|
||||
if registry is None or not _codex_can_drive(row["id"], row.get("mode")):
|
||||
return None
|
||||
group: Final = registry.group(lookup_id, include_hidden=True, team_id=team_id)
|
||||
return group.supported_reasoning_efforts if group is not None else None
|
||||
|
||||
|
||||
def _catalog_row(
|
||||
row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None
|
||||
) -> CodexCatalogRow:
|
||||
|
|
@ -446,6 +515,7 @@ def _catalog_row(
|
|||
upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None,
|
||||
display_name=registry.group(label_id, field="display_name") if registry is not None else None,
|
||||
service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (),
|
||||
supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id),
|
||||
search_tool_support=(
|
||||
registry.resolve(lookup_id, kind="search_tool_support", team_id=team_id) if registry is not None else ()
|
||||
),
|
||||
|
|
@ -463,7 +533,8 @@ def codex_catalog_rows(
|
|||
team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias`
|
||||
reading its target's deployments under its own name; `team_id` is the requesting key's team, so the
|
||||
upstream model that picks Codex's stock entry and the tiers are read only off the deployments its
|
||||
requests can route to. Display names are read under `label_entries`' ids when given."""
|
||||
requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the
|
||||
entry. Display names are read under `label_entries`' ids when given."""
|
||||
lookup_ids: Final = MappingProxyType(dict(entries))
|
||||
label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries))
|
||||
return tuple(
|
||||
|
|
@ -480,9 +551,13 @@ def codex_model_list_body(
|
|||
llm_router: Router | None,
|
||||
team_id: str | None = None,
|
||||
label_entries: Sequence[tuple[str, str]] | None = None,
|
||||
*,
|
||||
client_version: str | None = None,
|
||||
) -> str:
|
||||
"""The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out."""
|
||||
body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries))
|
||||
body: Final = codex_models_response_json(
|
||||
codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version
|
||||
)
|
||||
if body.left_out:
|
||||
verbose_proxy_logger.warning(
|
||||
"Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, "
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import redact_internal_details_from_client_fields
|
||||
from litellm.constants import STRINGIFIED_NONE
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
|
|
@ -28,6 +30,8 @@ _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, sta
|
|||
|
||||
_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600)
|
||||
|
||||
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
|
||||
|
||||
_FAILOVER_REFUSALS: Final = (
|
||||
|
|
@ -92,6 +96,18 @@ def openai_error_param(exc: object) -> str | None:
|
|||
return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
|
||||
|
||||
|
||||
def relayed_provider_specific_fields(
|
||||
exc: object,
|
||||
) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict
|
||||
body: Final = attribute_of(exc, "body")
|
||||
if not isinstance(body, dict):
|
||||
return None
|
||||
fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields")
|
||||
if not isinstance(fields, dict):
|
||||
return None
|
||||
return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields)))
|
||||
|
||||
|
||||
def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException:
|
||||
match exc:
|
||||
case ProxyException():
|
||||
|
|
@ -146,14 +162,43 @@ def _is_failover_refusal(exc: Exception) -> bool:
|
|||
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
|
||||
|
||||
|
||||
def answered_status_code(message: str, status_code: int) -> int:
|
||||
return int(proxy_exception_status_code(message, str(status_code)))
|
||||
|
||||
|
||||
def route_failure_proxy_exception(
|
||||
exc: Exception,
|
||||
message: str,
|
||||
default_status_code: int,
|
||||
headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers
|
||||
openai_code: str | None = None,
|
||||
) -> ProxyException:
|
||||
raised_status: Final = error_status_code(exc, default_status_code)
|
||||
answered_status: Final = answered_status_code(message, raised_status)
|
||||
return ProxyException(
|
||||
message=message,
|
||||
type=_route_failure_error_type(exc, raised_status, answered_status),
|
||||
param=openai_error_param(exc),
|
||||
headers=headers,
|
||||
openai_code=openai_code,
|
||||
code=answered_status,
|
||||
)
|
||||
|
||||
|
||||
def _route_failure_error_type(exc: Exception, raised_status: int, answered_status: int) -> str:
|
||||
if answered_status != raised_status and answered_status in _AUTH_FAILURE_STATUS_CODES:
|
||||
return ProxyErrorTypes.auth_error.value
|
||||
return openai_error_type(exc, answered_status)
|
||||
|
||||
|
||||
def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
|
||||
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
|
||||
answered_status: Final = answered_status_code(message, status_code)
|
||||
return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status))
|
||||
|
||||
|
||||
def status_typed_proxy_exception(exc: HTTPException) -> ProxyException:
|
||||
message: Final = str(exc.detail)
|
||||
answered_status: Final = int(proxy_exception_status_code(message, str(exc.status_code)))
|
||||
answered_status: Final = answered_status_code(message, exc.status_code)
|
||||
return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -435,7 +435,7 @@ class PrismaDBExceptionHandler:
|
|||
return ProxyException(
|
||||
message=PrismaDBExceptionHandler.database_unavailable_message(e),
|
||||
type=ProxyErrorTypes.no_db_connection,
|
||||
param="None",
|
||||
param=None,
|
||||
code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -664,4 +664,7 @@ class GenericGuardrailAPI(CustomGuardrail):
|
|||
GuardrailEventHooks.pre_call,
|
||||
GuardrailEventHooks.post_call,
|
||||
GuardrailEventHooks.during_call,
|
||||
GuardrailEventHooks.pre_mcp_call,
|
||||
GuardrailEventHooks.during_mcp_call,
|
||||
GuardrailEventHooks.post_mcp_call,
|
||||
]
|
||||
|
|
|
|||
|
|
@ -2505,6 +2505,7 @@ def refresh_proxy_server_request_body_snapshot(
|
|||
"litellm_logging_obj",
|
||||
VECTOR_STORE_CALLER_METADATA_KWARG,
|
||||
STREAM_USAGE_INJECTED_FIELD,
|
||||
"_litellm_strip_stream_usage",
|
||||
}
|
||||
)
|
||||
| _TRANSPORT_ONLY_CREDENTIAL_KEYS
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
|
|||
coordination_redis_cache,
|
||||
publish_config_change,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for
|
||||
from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
|
||||
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
|
||||
|
|
@ -4959,7 +4959,7 @@ async def info_key_fn(
|
|||
raise ProxyException(
|
||||
message="Internal server error from /key/info. Details are in the proxy server logs.",
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -8156,7 +8156,7 @@ async def key_health(
|
|||
raise ProxyException(
|
||||
message=f"Key health check failed: {e}",
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy
|
|||
encrypt_callback_vars,
|
||||
is_sensitive_callback_key,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
|
||||
from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports
|
||||
_get_validated_callback_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
convert_key_logging_metadata_to_callback,
|
||||
|
|
@ -463,7 +464,7 @@ async def add_team_callbacks(
|
|||
raise ProxyException(
|
||||
message="Internal Server Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error.value,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -606,7 +607,7 @@ async def delete_team_callback(
|
|||
raise ProxyException(
|
||||
message="Internal Server Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error.value,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
else:
|
||||
|
|
@ -747,7 +748,7 @@ async def disable_team_logging(
|
|||
raise ProxyException(
|
||||
message="Internal Server Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error.value,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -838,7 +839,7 @@ async def get_team_callbacks(
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Internal Server Error({e})"),
|
||||
type=ProxyErrorTypes.internal_server_error.value,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -846,6 +847,6 @@ async def get_team_callbacks(
|
|||
raise ProxyException(
|
||||
message="Internal Server Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error.value,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -68,8 +68,11 @@ module materially harder to understand.
|
|||
behind a single generic branch unless tests prove every mode still behaves
|
||||
correctly.
|
||||
- Be especially careful with legacy `delegate_auth_to_upstream: true`. `auth_type: oauth2`
|
||||
with `delegate_auth_to_upstream: true` is deprecated: LiteLLM admission is required
|
||||
for matching MCP routes. Use `auth_type: oauth_delegate` for client-forwarded OAuth.
|
||||
with `delegate_auth_to_upstream: true` is deprecated but still supported. A bare
|
||||
`Authorization` that is a LiteLLM credential is validated and never forwarded. Any other
|
||||
bearer is the upstream token: it is forwarded, and the caller is admitted as the JWT or
|
||||
custom-auth user it also proves when that user is granted the server, else anonymously.
|
||||
New servers should use `auth_type: true_passthrough` or `auth_type: oauth_delegate`.
|
||||
OAuth discovery endpoints stay public so clients can start the RFC 9728 flow.
|
||||
- Keep database-backed fields in sync across migrations, typed models under
|
||||
`litellm/types/mcp.py` or `litellm/types/mcp_server/`, config loading, this
|
||||
|
|
|
|||
|
|
@ -2,14 +2,15 @@ import asyncio
|
|||
import functools
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
import jwt
|
||||
from fastapi import HTTPException
|
||||
from pydantic import SecretStr, TypeAdapter
|
||||
from starlette.datastructures import Headers
|
||||
|
|
@ -45,9 +46,11 @@ from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
|
|||
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_auth
|
||||
from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_policy
|
||||
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
|
||||
from litellm.proxy.auth.oauth2_check import OAuth2TokenValidationError
|
||||
from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401 # legacy module exports
|
||||
_get_bearer_token_or_received_api_key, # pyright: ignore[reportPrivateUsage] # shared x-litellm-api-key parser lives with user_api_key_auth
|
||||
_run_centralized_common_checks, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
oauth2_introspection_applies,
|
||||
run_centralized_common_checks,
|
||||
user_api_key_auth,
|
||||
)
|
||||
|
|
@ -100,6 +103,9 @@ if TYPE_CHECKING:
|
|||
|
||||
_EMPTY_TOOLSET_GRANTS: Final[Mapping[str, Sequence[str]]] = MappingProxyType({})
|
||||
OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None)
|
||||
_PROXY_SETTINGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_CAUSE_CHAIN_MAX_DEPTH: Final = 16
|
||||
_IdentityCheck: TypeAlias = Literal["oauth2", "oauth2_proxy", "jwt"]
|
||||
_CACHED_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
|
|
@ -233,6 +239,136 @@ def _is_legacy_delegate_cold_start(mcp_servers: list[str] | None, client_ip: str
|
|||
return True
|
||||
|
||||
|
||||
def _cause_chain(exc: BaseException) -> Iterator[BaseException]:
|
||||
link: BaseException | None = exc # rebind-ok: advances one link of the cause chain per iteration
|
||||
for _ in range(_CAUSE_CHAIN_MAX_DEPTH):
|
||||
if link is None:
|
||||
return
|
||||
yield link
|
||||
link = link.__cause__ or link.__context__
|
||||
|
||||
|
||||
def _caused_by(exc: BaseException, error_type: type[BaseException]) -> bool:
|
||||
return any(isinstance(link, error_type) for link in _cause_chain(exc))
|
||||
|
||||
|
||||
def _signed_with(token: str, secret: str) -> bool:
|
||||
"""True when ``token`` is a JWT whose HS256 signature verifies with ``secret``, whatever its claims say."""
|
||||
try:
|
||||
jwt.decode(
|
||||
token,
|
||||
secret,
|
||||
algorithms=["HS256"],
|
||||
options={"verify_exp": False, "verify_nbf": False, "verify_iat": False, "verify_aud": False},
|
||||
)
|
||||
except jwt.PyJWTError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def _bearer_is_litellm_credential(token: str) -> bool:
|
||||
"""True when ``token`` is, or may be, a credential LiteLLM issued: the master key or a JWT it signed
|
||||
(dashboard sessions, BYOK sessions), a sealed gateway bearer, or a virtual key the key table holds (a
|
||||
lookup fault counts as held). Only ``sk-`` tokens are looked up, because no other token can be a
|
||||
virtual key."""
|
||||
from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module
|
||||
is_sealed_gateway_bearer,
|
||||
virtual_key_may_exist,
|
||||
)
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module
|
||||
LITELLM_VIRTUAL_KEY_PREFIX,
|
||||
)
|
||||
from litellm.proxy.proxy_server import master_key # noqa: PLC0415 # startup owns the master key
|
||||
|
||||
if master_key and (secrets.compare_digest(token.encode(), master_key.encode()) or _signed_with(token, master_key)):
|
||||
return True
|
||||
if is_sealed_gateway_bearer(token):
|
||||
return True
|
||||
return token.startswith(LITELLM_VIRTUAL_KEY_PREFIX) and await virtual_key_may_exist(token)
|
||||
|
||||
|
||||
def _identity_check(token: str, route: str, settings: Mapping[str, object]) -> _IdentityCheck | None:
|
||||
"""The identity check LiteLLM's own auth would run on ``token`` for ``route``, in its order: OAuth2
|
||||
introspection, then trusted reverse-proxy identity headers, then a JWT whose issuer may be LiteLLM's."""
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module
|
||||
jwt_issuer_may_be_litellm,
|
||||
)
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
|
||||
jwt_handler,
|
||||
premium_user,
|
||||
)
|
||||
|
||||
enable_jwt_auth: Final = settings.get("enable_jwt_auth") is True
|
||||
if oauth2_introspection_applies(
|
||||
token,
|
||||
route,
|
||||
enable_oauth2_auth=settings.get("enable_oauth2_auth") is True,
|
||||
enable_jwt_auth=enable_jwt_auth,
|
||||
jwt_handler=jwt_handler,
|
||||
):
|
||||
return "oauth2" if premium_user is True else None
|
||||
if settings.get("enable_oauth2_proxy_auth") is True:
|
||||
return "oauth2_proxy"
|
||||
if enable_jwt_auth and premium_user is True and JWTHandler.is_jwt(token=token) and jwt_issuer_may_be_litellm(token):
|
||||
return "jwt"
|
||||
return None
|
||||
|
||||
|
||||
async def _verify_identity(check: Literal["oauth2_proxy", "jwt"], token: str, request: Request) -> None:
|
||||
from litellm.proxy.auth.oauth2_proxy_hook import ( # noqa: PLC0415 # proxy import cycle
|
||||
handle_oauth2_proxy_request,
|
||||
)
|
||||
from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration
|
||||
|
||||
match check:
|
||||
case "oauth2_proxy":
|
||||
await handle_oauth2_proxy_request(request=request)
|
||||
case "jwt":
|
||||
await jwt_handler.auth_jwt(token=token) # pyright: ignore[reportUnknownMemberType] # returns a bare dict; only whether it raises matters
|
||||
case _: # pragma: no cover - exhaustiveness guard, unreachable while the Literal holds
|
||||
assert_never(check) # pragma: no cover
|
||||
|
||||
|
||||
async def _upstream_bearer_identity(token: str, litellm_api_key: str, request: Request) -> UserAPIKeyAuth | None:
|
||||
"""The LiteLLM user an upstream bearer also proves, or ``None`` when it proves none.
|
||||
|
||||
Custom auth decides alone, so any rejection is ``None``. Otherwise the identity check LiteLLM would run
|
||||
must vouch for the caller. Reverse-proxy headers and a JWT are verified first, and OAuth2 introspection
|
||||
runs once inside the full check, where a provider that does not vouch is ``None``. Past that, only a
|
||||
user LiteLLM does not know is ``None``; every other refusal (budget, blocked team, disallowed route)
|
||||
propagates."""
|
||||
from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.mcp_server.bridge_token_flow import ( # noqa: PLC0415 # bridge flow imports this module
|
||||
custom_auth_configured,
|
||||
)
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
|
||||
general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict
|
||||
)
|
||||
|
||||
if custom_auth_configured():
|
||||
try:
|
||||
return await user_api_key_auth(api_key=litellm_api_key, request=request)
|
||||
except (HTTPException, ProxyException):
|
||||
return None
|
||||
check: Final = _identity_check(
|
||||
token, get_request_route(request), _PROXY_SETTINGS_ADAPTER.validate_python(general_settings)
|
||||
)
|
||||
if check is None:
|
||||
return None
|
||||
if check != "oauth2":
|
||||
try:
|
||||
await _verify_identity(check, token, request)
|
||||
except Exception: # noqa: BLE001 # any verification failure means LiteLLM does not vouch for this bearer
|
||||
return None
|
||||
try:
|
||||
return await user_api_key_auth(api_key=litellm_api_key, request=request)
|
||||
except (HTTPException, ProxyException, OAuth2TokenValidationError) as exc:
|
||||
if _caused_by(exc, UserNotFoundError) or _caused_by(exc, OAuth2TokenValidationError):
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def _is_litellm_auth_admission_error(exc: Exception) -> bool:
|
||||
if isinstance(exc, HTTPException):
|
||||
return exc.status_code == 401
|
||||
|
|
@ -704,6 +840,19 @@ class MCPRequestHandler:
|
|||
route=request_route,
|
||||
mcp_servers=mcp_servers,
|
||||
)
|
||||
elif oauth2_headers and (
|
||||
legacy_delegate_targets := MCPRequestHandler._legacy_delegate_targets(
|
||||
path=request_route,
|
||||
mcp_servers=mcp_servers,
|
||||
client_ip=IPAddressUtils.get_mcp_client_ip(request),
|
||||
)
|
||||
):
|
||||
validated_user_api_key_auth = await MCPRequestHandler._admit_legacy_delegate_bearer( # rebind-ok: one assignment per admission arm, like its siblings
|
||||
authorization_value=oauth2_headers["Authorization"],
|
||||
litellm_api_key=litellm_api_key,
|
||||
targets=legacy_delegate_targets,
|
||||
request=request,
|
||||
)
|
||||
elif oauth2_headers:
|
||||
# Authorization on a non-delegated server: the bearer must be a real
|
||||
# LiteLLM credential, so a failed validation is a genuine 401/403 and
|
||||
|
|
@ -741,10 +890,14 @@ class MCPRequestHandler:
|
|||
custom_key_header_name: Final = OPTIONAL_STRING_ADAPTER.validate_python(
|
||||
general_settings.get("litellm_key_header_name")
|
||||
)
|
||||
admitted_credential: Final = MCPRequestHandler.caller_admission_credential(
|
||||
headers,
|
||||
validated_user_api_key_auth,
|
||||
custom_key_header_name=custom_key_header_name,
|
||||
admitted_credential: Final = (
|
||||
None
|
||||
if validated_user_api_key_auth.mcp_forwards_caller_authorization
|
||||
else MCPRequestHandler.caller_admission_credential(
|
||||
headers,
|
||||
validated_user_api_key_auth,
|
||||
custom_key_header_name=custom_key_header_name,
|
||||
)
|
||||
)
|
||||
(
|
||||
oauth2_headers,
|
||||
|
|
@ -934,6 +1087,53 @@ class MCPRequestHandler:
|
|||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _legacy_delegate_targets(
|
||||
path: str, mcp_servers: Sequence[str] | None, client_ip: str | None
|
||||
) -> tuple[MCPServer, ...]:
|
||||
"""Every server the request targets when ALL of them are legacy ``delegate_auth_to_upstream``
|
||||
servers, else ``()``. Names resolve the way downstream routing resolves them, so a permissive
|
||||
``x-mcp-servers`` header cannot open the gate for a stricter path target."""
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module
|
||||
global_mcp_server_manager,
|
||||
is_legacy_upstream_delegate,
|
||||
)
|
||||
|
||||
servers: Final = tuple(
|
||||
global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip)
|
||||
for name in MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers)
|
||||
)
|
||||
delegates: Final = tuple(
|
||||
server for server in servers if server is not None and is_legacy_upstream_delegate(server)
|
||||
)
|
||||
return delegates if servers and len(delegates) == len(servers) else ()
|
||||
|
||||
@staticmethod
|
||||
async def _admit_legacy_delegate_bearer(
|
||||
authorization_value: str,
|
||||
litellm_api_key: str,
|
||||
targets: tuple[MCPServer, ...],
|
||||
request: Request,
|
||||
) -> UserAPIKeyAuth:
|
||||
"""Admit a bare ``Authorization`` bearer sent to legacy ``delegate_auth_to_upstream`` servers.
|
||||
|
||||
A LiteLLM credential runs normal admission, and its errors propagate. Any other bearer is the
|
||||
upstream token and is marked for forwarding: a user it also proves who is granted every target is
|
||||
admitted as that user, and every other caller is admitted anonymously."""
|
||||
from litellm.proxy.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # manager imports this module
|
||||
global_mcp_server_manager,
|
||||
)
|
||||
|
||||
token: Final = _normalize_caller_admission_credential(authorization_value)
|
||||
if await _bearer_is_litellm_credential(token):
|
||||
return await user_api_key_auth(api_key=litellm_api_key, request=request)
|
||||
identity: Final = await _upstream_bearer_identity(token, litellm_api_key, request)
|
||||
if identity is not None:
|
||||
allowed: Final = frozenset(await global_mcp_server_manager.get_allowed_mcp_servers(identity))
|
||||
if all(server.server_id in allowed for server in targets):
|
||||
return identity.model_copy(update={"mcp_forwards_caller_authorization": True})
|
||||
return UserAPIKeyAuth().model_copy(update={"mcp_forwards_caller_authorization": True})
|
||||
|
||||
@staticmethod
|
||||
def _single_dcr_bridge_delegate_target(
|
||||
path: str, mcp_servers: list[str] | None, client_ip: str | None
|
||||
|
|
@ -1553,7 +1753,7 @@ class MCPRequestHandler:
|
|||
return expiry >= datetime.now(timezone.utc)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_target_server_names(path: str, mcp_servers_header: list[str] | None) -> list[str]:
|
||||
def _resolve_target_server_names(path: str, mcp_servers_header: Sequence[str] | None) -> Sequence[str]:
|
||||
"""
|
||||
Resolve the target MCP server names exactly as downstream routing
|
||||
does (``server.py::extract_mcp_auth_context``).
|
||||
|
|
|
|||
|
|
@ -61,7 +61,6 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool:
|
|||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
|
||||
jwt_handler,
|
||||
master_key,
|
||||
user_custom_auth,
|
||||
)
|
||||
|
||||
if "x-litellm-api-key" in request.headers:
|
||||
|
|
@ -71,10 +70,26 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool:
|
|||
return "authorization" in request.headers
|
||||
if token.startswith("sk-") or (master_key and secrets.compare_digest(token.encode(), master_key.encode())):
|
||||
return True
|
||||
if user_custom_auth is not None or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled:
|
||||
if custom_auth_configured() or jwt_handler.litellm_jwtauth.oidc_userinfo_enabled:
|
||||
return True
|
||||
if not JWTHandler.is_jwt(token):
|
||||
return await _opaque_bearer_is_gateway_credential(token)
|
||||
return is_sealed_gateway_bearer(token) or await virtual_key_may_exist(token)
|
||||
return jwt_issuer_may_be_litellm(token)
|
||||
|
||||
|
||||
def custom_auth_configured() -> bool:
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the active auth configuration
|
||||
user_custom_auth,
|
||||
)
|
||||
|
||||
return user_custom_auth is not None
|
||||
|
||||
|
||||
def jwt_issuer_may_be_litellm(token: str) -> bool:
|
||||
"""False only when the JWT names an issuer LiteLLM's JWT auth is not configured to accept."""
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.proxy_server import jwt_handler # noqa: PLC0415 # startup owns the active auth configuration
|
||||
|
||||
claims: Final = JWTHandler.get_unverified_claims(token)
|
||||
issuer: Final = claims.get("iss") if claims is not None else None
|
||||
global_issuer: Final = os.getenv("JWT_ISSUER")
|
||||
|
|
@ -86,25 +101,35 @@ async def oauth_authorization_uses_gateway_credential(request: Request) -> bool:
|
|||
)
|
||||
|
||||
|
||||
async def _opaque_bearer_is_gateway_credential(token: str) -> bool:
|
||||
from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle
|
||||
def is_sealed_gateway_bearer(token: str) -> bool:
|
||||
"""True for a bearer only the gateway issues: a bridge or refresh envelope, a v2 GCM blob, or a
|
||||
dashboard UI token. A UI token that cannot be read still counts, so a lookup fault never reads as foreign."""
|
||||
from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.mcp_server.outbound_credentials.envelope import ( # noqa: PLC0415 # envelope imports bridge types
|
||||
is_envelope,
|
||||
is_refresh_envelope,
|
||||
)
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies
|
||||
prisma_client,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
if is_envelope(token) or is_refresh_envelope(token) or token.startswith(V2_GCM_PREFIX):
|
||||
return True
|
||||
try:
|
||||
if ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None:
|
||||
return True
|
||||
return ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None
|
||||
except Exception as exc: # noqa: BLE001 # an identity lookup fault must not permit cookie fallback
|
||||
verbose_logger.debug("OAuth bearer ownership could not be checked (%s)", type(exc).__name__)
|
||||
return True
|
||||
|
||||
|
||||
async def virtual_key_may_exist(token: str) -> bool:
|
||||
"""False only when the key table answers that ``token`` is not a virtual key; a lookup fault counts as held."""
|
||||
from litellm.proxy._types import hash_token # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.auth.resolvers.store import IdentityStore # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # startup owns the identity store dependencies
|
||||
prisma_client,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
try:
|
||||
await IdentityStore(prisma_client, user_api_key_cache).resolve(hashed_token=hash_token(token))
|
||||
except KeyNotFoundError:
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1196,8 +1196,11 @@ def should_strip_caller_authorization(
|
|||
pass-through cold-start case (RFC 9728) the bearer in
|
||||
``Authorization`` is the upstream OAuth token and must be
|
||||
forwarded, so we keep it.
|
||||
- **Delegated OAuth servers**: admission always runs and there is no
|
||||
anonymous path, so the caller's separate ``Authorization`` is
|
||||
- **Legacy delegate servers**: forward when MCP admission marked the
|
||||
caller's ``Authorization`` as the upstream token
|
||||
(``mcp_forwards_caller_authorization``); otherwise follow the
|
||||
delegated OAuth rule below.
|
||||
- **Delegated OAuth servers**: the caller's separate ``Authorization`` is
|
||||
forwarded only when a distinct ``x-litellm-api-key`` carried
|
||||
admission. Without that header the ``Authorization`` *was* the
|
||||
admission credential — a virtual key, an IdP JWT, or an SSO / OIDC /
|
||||
|
|
@ -1206,6 +1209,12 @@ def should_strip_caller_authorization(
|
|||
- **Every other auth-enabled mode**: strip when ``Authorization`` is the
|
||||
credential admission validated.
|
||||
"""
|
||||
if (
|
||||
user_api_key_auth is not None
|
||||
and user_api_key_auth.mcp_forwards_caller_authorization
|
||||
and is_legacy_upstream_delegate(mcp_server)
|
||||
):
|
||||
return False
|
||||
if _authorization_is_admission_jwt(raw_headers, user_api_key_auth):
|
||||
return True
|
||||
if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
|
||||
|
|
@ -1557,6 +1566,18 @@ async def _materialize_auth_headers(auth: httpx2.Auth | None) -> dict[str, str]
|
|||
return {header_name: header_value} if header_value else None
|
||||
|
||||
|
||||
def is_legacy_upstream_delegate(server: MCPServer) -> bool:
|
||||
"""True for an interactive ``auth_type=oauth2`` server with ``delegate_auth_to_upstream=true``, whose
|
||||
caller ``Authorization`` is the upstream token. The flow is resolved, not read from the column, so an
|
||||
unstamped M2M-shaped row stays excluded: its stored client credentials drive egress, and admitting it
|
||||
anonymously would let any caller act as LiteLLM's service account."""
|
||||
return (
|
||||
server.auth_type == MCPAuth.oauth2
|
||||
and server.delegate_auth_to_upstream is True
|
||||
and MCPServerManager.effective_oauth2_flow(server) != "client_credentials"
|
||||
)
|
||||
|
||||
|
||||
def _consumes_caller_authorization(server: MCPServer) -> bool:
|
||||
"""True when this server's egress forwards the caller's request-wide ``Authorization`` upstream:
|
||||
the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated
|
||||
|
|
@ -1726,18 +1747,15 @@ def _warn_overlapping_tool_policies(server: MCPServer) -> None:
|
|||
|
||||
|
||||
def _warn_legacy_delegate_auth_if_applicable(server: MCPServer, *, source: str) -> None:
|
||||
"""Direct legacy delegated OAuth configurations to the admitted replacement."""
|
||||
if server.auth_type != MCPAuth.oauth2:
|
||||
return
|
||||
if getattr(server, "delegate_auth_to_upstream", False) is not True:
|
||||
return
|
||||
if server.has_client_credentials:
|
||||
"""Point legacy delegated OAuth configurations at the client-forwarded replacements."""
|
||||
if not is_legacy_upstream_delegate(server):
|
||||
return
|
||||
label: Final = get_server_prefix(server)
|
||||
verbose_logger.warning(
|
||||
"MCP server %r (id=%s, source=%s) uses deprecated auth_type=oauth2 with "
|
||||
"delegate_auth_to_upstream=true. LiteLLM admission is now required; migrate to "
|
||||
"auth_type=oauth_delegate for client-forwarded OAuth.",
|
||||
"delegate_auth_to_upstream=true. It still works: a caller without a LiteLLM credential is "
|
||||
"admitted anonymously and its upstream token is forwarded. Migrate to auth_type=true_passthrough "
|
||||
"or auth_type=oauth_delegate.",
|
||||
label,
|
||||
server.server_id,
|
||||
source,
|
||||
|
|
@ -3766,12 +3784,12 @@ class MCPServerManager:
|
|||
)
|
||||
)
|
||||
if is_anonymous:
|
||||
passthrough_server_ids: Final = [
|
||||
anonymous_server_ids: Final = [
|
||||
server.server_id
|
||||
for server in self.get_registry().values()
|
||||
if server.auth_type == MCPAuth.true_passthrough
|
||||
if server.auth_type == MCPAuth.true_passthrough or is_legacy_upstream_delegate(server)
|
||||
]
|
||||
combined_servers.update(passthrough_server_ids)
|
||||
combined_servers.update(anonymous_server_ids)
|
||||
|
||||
restrict_allow_all: Final = (
|
||||
resolved_general_settings.get("mcp_allow_all_keys_respects_mcp_scope", False)
|
||||
|
|
|
|||
|
|
@ -588,6 +588,7 @@ if MCP_AVAILABLE:
|
|||
from litellm.proxy.mcp_server.mcp_server_manager import (
|
||||
MCPServerManager,
|
||||
global_mcp_server_manager,
|
||||
is_legacy_upstream_delegate,
|
||||
)
|
||||
|
||||
######################################################
|
||||
|
|
@ -2162,8 +2163,9 @@ if MCP_AVAILABLE:
|
|||
oauth2_headers: Mapping[str, str] | None,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
|
||||
raw_headers: Mapping[str, str] | None,
|
||||
requested_servers: Sequence[str] | None = None,
|
||||
) -> None:
|
||||
"""Probe pass-through upstream servers in parallel before the MCP session starts.
|
||||
"""Probe pass-through and legacy delegate upstream servers in parallel before the MCP session starts.
|
||||
|
||||
Only servers the caller's key is already authorized to reach are probed —
|
||||
the list is derived from _get_allowed_mcp_servers so that a user cannot
|
||||
|
|
@ -2186,6 +2188,18 @@ if MCP_AVAILABLE:
|
|||
mcp_servers=mcp_servers,
|
||||
client_ip=client_ip,
|
||||
)
|
||||
# A legacy delegate is probed only when the single requested name resolves to it the way admission
|
||||
# resolves it, so neither a multi-server connect nor an access group can be failed by one rejected
|
||||
# token; its challenge echoes the requested name, matching the tokenless challenge.
|
||||
requested_scope: Final = requested_servers if requested_servers is not None else mcp_servers
|
||||
single_requested_name: Final = (
|
||||
requested_scope[0] if requested_scope is not None and len(requested_scope) == 1 else None
|
||||
)
|
||||
requested_server: Final = (
|
||||
operations.global_mcp_server_manager.get_mcp_server_by_name(single_requested_name, client_ip=client_ip)
|
||||
if single_requested_name is not None
|
||||
else None
|
||||
)
|
||||
prepared_headers: Final = tuple(
|
||||
(
|
||||
srv,
|
||||
|
|
@ -2201,6 +2215,11 @@ if MCP_AVAILABLE:
|
|||
)
|
||||
for srv in allowed_servers
|
||||
if srv.is_oauth_passthrough
|
||||
or (
|
||||
requested_server is not None
|
||||
and srv.server_id == requested_server.server_id
|
||||
and is_legacy_upstream_delegate(srv)
|
||||
)
|
||||
)
|
||||
probe_headers: Final = tuple(
|
||||
(
|
||||
|
|
@ -2213,7 +2232,7 @@ if MCP_AVAILABLE:
|
|||
for srv, (auth, extra) in prepared_headers
|
||||
)
|
||||
probe_targets: Final[tuple[tuple[MCPServer, str, str], ...]] = tuple(
|
||||
(srv, auth_header, srv.name)
|
||||
(srv, auth_header, srv.name if srv.is_oauth_passthrough else single_requested_name or srv.name)
|
||||
for srv, headers in probe_headers
|
||||
if (
|
||||
auth_header := next(
|
||||
|
|
@ -2311,6 +2330,7 @@ if MCP_AVAILABLE:
|
|||
if context.mcp_server_auth_headers is not None
|
||||
else None,
|
||||
raw_headers=context.raw_headers,
|
||||
requested_servers=context.mcp_servers,
|
||||
)
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import LiteLLMRoutes
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
|
||||
|
||||
class BillableCategory(str, Enum):
|
||||
|
|
@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None:
|
|||
suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None)
|
||||
if suffix_match is not None:
|
||||
return suffix_match
|
||||
if RouteChecks.is_builtin_eu_jev_decision_route(route=path):
|
||||
return path
|
||||
# Deep passthrough paths only: the bare prefix itself is not an inference call.
|
||||
return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None)
|
||||
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ from litellm.proxy.common_request_processing import (
|
|||
open_sse_before_first_byte,
|
||||
resolve_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
|
||||
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
|
||||
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
_safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
|
|
@ -119,6 +120,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import (
|
|||
)
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
HttpPassThroughEndpointHelpers,
|
||||
YieldsToConfiguredPassThroughRoute,
|
||||
create_pass_through_route,
|
||||
create_websocket_passthrough_route,
|
||||
websocket_passthrough_request,
|
||||
|
|
@ -132,6 +134,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
|
|||
is_allowed_to_call_vector_store_endpoint,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool
|
||||
from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
|
|
@ -714,6 +717,85 @@ def typesafe_allowlist_model_names() -> tuple[str, ...]:
|
|||
return ("typesafe/*", *jev_models_typesafe_serves)
|
||||
|
||||
|
||||
_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body(
|
||||
EuJevDecisionRequest,
|
||||
{
|
||||
"model": "eu-jeff-latest",
|
||||
"state": "I was charged twice. Can I get a refund?",
|
||||
"questions": {
|
||||
"team": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team should handle this?",
|
||||
"criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"},
|
||||
},
|
||||
"is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
|
||||
"frustration": {
|
||||
"type": "score",
|
||||
"instructions": "How frustrated is the customer?",
|
||||
"criteria": ["calm", "mildly annoyed", "angry"],
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute)
|
||||
|
||||
|
||||
@eu_jev_router.post(
|
||||
"/eu_jev/v1/systemone",
|
||||
tags=["eu/jev Pass-through", "pass-through"],
|
||||
responses={200: {"model": EuJevDecisionResponse}},
|
||||
openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY},
|
||||
)
|
||||
async def eu_jev_proxy_route(
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> Response:
|
||||
"""
|
||||
Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.
|
||||
|
||||
`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)
|
||||
and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and
|
||||
`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.
|
||||
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)
|
||||
"""
|
||||
body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request))
|
||||
if "custom_body" in body:
|
||||
raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests")
|
||||
if body.get("stream"):
|
||||
raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests")
|
||||
eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev")
|
||||
if eu_jev_credentials is None:
|
||||
raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy")
|
||||
base_url: Final = httpx.URL(
|
||||
(eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com")
|
||||
.rstrip("/")
|
||||
.removesuffix("/v1")
|
||||
)
|
||||
updated_url: Final = base_url.copy_with(
|
||||
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"),
|
||||
)
|
||||
endpoint_func: Final = create_pass_through_route(
|
||||
endpoint="v1/systemone",
|
||||
target=str(updated_url),
|
||||
custom_headers={
|
||||
"Authorization": f"Bearer {eu_jev_credentials.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
custom_llm_provider="eu_jev",
|
||||
is_streaming_request=False,
|
||||
)
|
||||
return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python(
|
||||
await endpoint_func(request, fastapi_response, user_api_key_dict)
|
||||
)
|
||||
|
||||
|
||||
router.include_router(eu_jev_router)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/laya/v1/systemone",
|
||||
tags=["Laya Pass-through", "pass-through"],
|
||||
|
|
|
|||
|
|
@ -26,9 +26,11 @@ from fastapi import (
|
|||
status,
|
||||
)
|
||||
from fastapi.responses import StreamingResponse
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import TypeAdapter
|
||||
from starlette.datastructures import UploadFile as StarletteUploadFile
|
||||
from starlette.routing import BaseRoute, Route
|
||||
from starlette.routing import BaseRoute, Match, Route
|
||||
from starlette.types import Scope
|
||||
from starlette.websockets import WebSocketState
|
||||
from typing_extensions import ReadOnly
|
||||
from websockets.asyncio.client import connect
|
||||
|
|
@ -73,6 +75,7 @@ from litellm.llms.base_llm.managed_resources.utils import (
|
|||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body
|
||||
from litellm.llms.oss_decision import validate_oss_request
|
||||
from litellm.passthrough import BasePassthroughUtils
|
||||
from litellm.proxy._lazy_features import lazy_owned_routes
|
||||
|
|
@ -90,6 +93,7 @@ from litellm.proxy.auth.auth_utils import (
|
|||
get_request_route,
|
||||
request_dispatched_to_pass_through_endpoint,
|
||||
)
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import (
|
||||
ProxyBaseLLMRequestProcessing,
|
||||
|
|
@ -433,9 +437,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
parsed_url: Final = urlparse(url)
|
||||
if custom_llm_provider == "chatgpt":
|
||||
return EndpointType.OPENAI
|
||||
if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith(
|
||||
"/v1/systemone"
|
||||
):
|
||||
if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix(
|
||||
"/"
|
||||
).endswith("/v1/systemone"):
|
||||
return EndpointType.DECISIONS
|
||||
if (
|
||||
("generateContent") in url
|
||||
|
|
@ -1137,6 +1141,16 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
return headers
|
||||
|
||||
|
||||
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
|
||||
|
||||
|
||||
def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | None) -> Mapping[str, str]:
|
||||
if eu_jev_client_body is None:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)})
|
||||
|
||||
|
||||
async def pass_through_request(
|
||||
request: Request,
|
||||
target: str,
|
||||
|
|
@ -1255,7 +1269,12 @@ async def pass_through_request(
|
|||
strip_client_guardrail_information(client_body)
|
||||
if not _key_or_team_allows_client_pricing_override(user_api_key_dict):
|
||||
_strip_client_pricing_overrides(client_body)
|
||||
_parsed_body = client_body
|
||||
eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body)
|
||||
scoped_client_body: Final = {
|
||||
**client_body,
|
||||
**_eu_jev_scoped_model_override(eu_jev_client_body),
|
||||
}
|
||||
_parsed_body = scoped_client_body
|
||||
if custom_llm_provider in ("laya", "bespoke"):
|
||||
decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body)
|
||||
checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request)
|
||||
|
|
@ -1339,6 +1358,12 @@ async def pass_through_request(
|
|||
)
|
||||
_ = validate_oss_request(custom_llm_provider, decision_body)
|
||||
_parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body)
|
||||
if eu_jev_client_body is not None:
|
||||
_parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage
|
||||
eu_jev_upstream_body(
|
||||
TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body
|
||||
)
|
||||
)
|
||||
resolved_timeout: Final = resolve_pass_through_request_timeout(timeout)
|
||||
async_client_obj: Final = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.PassThroughEndpoint,
|
||||
|
|
@ -2240,17 +2265,23 @@ def create_pass_through_route(
|
|||
stream,
|
||||
) = await _parse_request_data_by_content_type(request)
|
||||
|
||||
if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path):
|
||||
if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or (
|
||||
custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.",
|
||||
)
|
||||
|
||||
passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route(
|
||||
route=path, method=request.method
|
||||
serves_configured_route: Final = custom_llm_provider != "eu_jev"
|
||||
passthrough_params: Final = (
|
||||
InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method)
|
||||
if serves_configured_route
|
||||
else None
|
||||
)
|
||||
if (
|
||||
passthrough_params is None
|
||||
serves_configured_route
|
||||
and passthrough_params is None
|
||||
and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None
|
||||
):
|
||||
raise HTTPException(
|
||||
|
|
@ -3392,6 +3423,37 @@ class InitPassThroughEndpointHelpers:
|
|||
del _registered_pass_through_routes[key]
|
||||
verbose_proxy_logger.debug("Removed pass-through route from registry: %s", key)
|
||||
|
||||
@staticmethod
|
||||
def _llm_api_routes_of_route(route_info: Mapping[str, object]) -> tuple[str, ...]:
|
||||
path: Final = route_info.get("path")
|
||||
if not isinstance(path, str):
|
||||
return ()
|
||||
if route_info.get("type") == "subpath":
|
||||
return (path, path.rstrip("/") + "/*")
|
||||
return (path,)
|
||||
|
||||
@staticmethod
|
||||
def llm_api_routes_of_endpoint(endpoint_id: str) -> frozenset[str]:
|
||||
route_infos: Final = (
|
||||
route_info
|
||||
for route_info in _registered_pass_through_routes.values()
|
||||
if route_info["endpoint_id"] == endpoint_id
|
||||
)
|
||||
return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos)))
|
||||
|
||||
@staticmethod
|
||||
def auth_enforced_llm_api_routes() -> frozenset[str]:
|
||||
route_infos: Final = (
|
||||
route_info for route_info in _registered_pass_through_routes.values() if route_info.get("auth") is True
|
||||
)
|
||||
return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos)))
|
||||
|
||||
@staticmethod
|
||||
def drop_llm_api_routes(routes: frozenset[str]) -> None:
|
||||
openai_routes: Final = LiteLLMRoutes.openai_routes.value
|
||||
for route in routes.intersection(openai_routes):
|
||||
openai_routes.remove(route)
|
||||
|
||||
@staticmethod
|
||||
def clear_all_pass_through_routes():
|
||||
"""Clear all pass-through routes from the registry"""
|
||||
|
|
@ -3436,6 +3498,8 @@ class InitPassThroughEndpointHelpers:
|
|||
return True
|
||||
|
||||
comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
|
||||
if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route):
|
||||
return True
|
||||
|
||||
# Fast path: check if any registered route key contains this path
|
||||
# Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}"
|
||||
|
|
@ -3512,6 +3576,25 @@ class InitPassThroughEndpointHelpers:
|
|||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def has_configured_route_ignoring_trailing_slash(route: str) -> bool:
|
||||
bare_route: Final = route.rstrip("/")
|
||||
return any(
|
||||
InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None
|
||||
for form in (bare_route, f"{bare_route}/")
|
||||
)
|
||||
|
||||
|
||||
class YieldsToConfiguredPassThroughRoute(APIRoute):
|
||||
def matches(self, scope: Scope) -> tuple[Match, Scope]:
|
||||
match, child_scope = super().matches(scope)
|
||||
if match is Match.NONE:
|
||||
return match, child_scope
|
||||
configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash(
|
||||
route=get_request_route(Request(scope))
|
||||
)
|
||||
return (Match.NONE, {}) if configured else (match, child_scope)
|
||||
|
||||
|
||||
def _get_combined_pass_through_endpoints(
|
||||
pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint],
|
||||
|
|
@ -3971,9 +4054,7 @@ async def update_pass_through_endpoints(
|
|||
|
||||
# Update the list
|
||||
pass_through_endpoint_data[endpoint_index] = endpoint_dict
|
||||
|
||||
# Remove old routes from registry before they get re-registered
|
||||
InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id)
|
||||
replaced_llm_api_routes: Final = InitPassThroughEndpointHelpers.llm_api_routes_of_endpoint(endpoint_id)
|
||||
|
||||
## Update db
|
||||
updated_data: Final = ConfigFieldUpdate(
|
||||
|
|
@ -4023,6 +4104,9 @@ async def update_pass_through_endpoints(
|
|||
auth=updated_endpoint.auth,
|
||||
timeout=updated_endpoint.timeout,
|
||||
)
|
||||
InitPassThroughEndpointHelpers.drop_llm_api_routes(
|
||||
replaced_llm_api_routes - InitPassThroughEndpointHelpers.auth_enforced_llm_api_routes()
|
||||
)
|
||||
|
||||
return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else [])
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -11,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
|
|||
LiteLLM_ManagedVectorStore,
|
||||
)
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
from litellm.secret_managers.main import (
|
||||
GOOGLE_OIDC_SECRET_PREFIX,
|
||||
UnresolvableSecretReference,
|
||||
|
|
@ -25,12 +27,23 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PassThroughCredentials:
|
||||
api_key: str
|
||||
api_base: str | None
|
||||
|
||||
|
||||
def _get_proxy_llm_router() -> "Router | None":
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return llm_router
|
||||
|
||||
|
||||
def _is_team_deployment(deployment: Mapping[str, object]) -> bool:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id"))
|
||||
|
||||
|
||||
def _get_str_value(values: dict[str, object] | None, key: str) -> str | None:
|
||||
value: Final = values.get(key) if values is not None else None
|
||||
return value if isinstance(value, str) else None
|
||||
|
|
@ -84,12 +97,13 @@ class PassthroughEndpointRouter:
|
|||
custom_llm_provider: str,
|
||||
region_name: str | None,
|
||||
) -> str | None:
|
||||
deployment_api_key: Final = self._get_deployment_api_key(
|
||||
deployment_credentials: Final = self._get_deployment_credentials(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
region_name=region_name,
|
||||
include_team_deployments=True,
|
||||
)
|
||||
if deployment_api_key is not None:
|
||||
return _resolve_oidc_reference(deployment_api_key)
|
||||
if deployment_credentials is not None:
|
||||
return _resolve_oidc_reference(deployment_credentials.api_key)
|
||||
verbose_router_logger.debug(
|
||||
"No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider
|
||||
)
|
||||
|
|
@ -98,29 +112,56 @@ class PassthroughEndpointRouter:
|
|||
)
|
||||
return _resolve_oidc_reference(get_secret_str(_env_variable_name))
|
||||
|
||||
def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None:
|
||||
deployment_credentials: Final = self._get_deployment_credentials(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
region_name=None,
|
||||
include_team_deployments=False,
|
||||
)
|
||||
api_key: Final = _resolve_oidc_reference(
|
||||
deployment_credentials.api_key
|
||||
if deployment_credentials is not None
|
||||
else get_secret_str(
|
||||
self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider)
|
||||
)
|
||||
)
|
||||
if not api_key:
|
||||
return None
|
||||
return _PassThroughCredentials(
|
||||
api_key=api_key,
|
||||
api_base=deployment_credentials.api_base if deployment_credentials is not None else None,
|
||||
)
|
||||
|
||||
def has_credentials(self, custom_llm_provider: str) -> bool:
|
||||
if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None:
|
||||
if (
|
||||
self._get_deployment_credentials(
|
||||
custom_llm_provider=custom_llm_provider, region_name=None, include_team_deployments=True
|
||||
)
|
||||
is not None
|
||||
):
|
||||
return True
|
||||
env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
return bool(os.environ.get(env_variable_name))
|
||||
|
||||
def _get_deployment_api_key(
|
||||
def _get_deployment_credentials(
|
||||
self,
|
||||
custom_llm_provider: str,
|
||||
region_name: str | None,
|
||||
) -> str | None:
|
||||
include_team_deployments: bool,
|
||||
) -> _PassThroughCredentials | None:
|
||||
llm_router: Final = self.llm_router_getter()
|
||||
if llm_router is None:
|
||||
return None
|
||||
deployments: Final = llm_router.registry().deployments() or ()
|
||||
return next(
|
||||
(
|
||||
api_key
|
||||
credentials
|
||||
for deployment in deployments
|
||||
if (
|
||||
api_key := self._resolve_matching_deployment_api_key(
|
||||
if (include_team_deployments or not _is_team_deployment(deployment))
|
||||
and (
|
||||
credentials := self._resolve_matching_deployment_credentials(
|
||||
litellm_params=deployment["litellm_params"],
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
region_name=region_name,
|
||||
|
|
@ -131,12 +172,12 @@ class PassthroughEndpointRouter:
|
|||
None,
|
||||
)
|
||||
|
||||
def _resolve_matching_deployment_api_key(
|
||||
def _resolve_matching_deployment_credentials(
|
||||
self,
|
||||
litellm_params: LiteLLMParamsTypedDict,
|
||||
custom_llm_provider: str,
|
||||
region_name: str | None,
|
||||
) -> str | None:
|
||||
) -> _PassThroughCredentials | None:
|
||||
if litellm_params.get("use_in_pass_through") is not True:
|
||||
return None
|
||||
if self._get_deployment_provider(litellm_params) != custom_llm_provider:
|
||||
|
|
@ -152,7 +193,8 @@ class PassthroughEndpointRouter:
|
|||
)
|
||||
if deployment_region != region_name:
|
||||
return None
|
||||
return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
|
||||
api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
|
||||
return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None
|
||||
|
||||
def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None:
|
||||
model: Final = litellm_params.get("model")
|
||||
|
|
|
|||
|
|
@ -375,7 +375,7 @@ class PassThroughEndpointLogging:
|
|||
kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract
|
||||
elif (
|
||||
self.is_typesafe_route(custom_llm_provider)
|
||||
or custom_llm_provider in ("laya", "bespoke")
|
||||
or custom_llm_provider in ("eu_jev", "laya", "bespoke")
|
||||
or self.is_openrouter_decisions_route(url_route, custom_llm_provider)
|
||||
):
|
||||
from .llm_provider_handlers.typesafe_passthrough_logging_handler import (
|
||||
|
|
|
|||
|
|
@ -201,7 +201,6 @@ from litellm.types.utils import (
|
|||
CountContextManagement,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
TextCompletionResponse,
|
||||
TokenCountMessagesFormat,
|
||||
TokenCountResponse,
|
||||
|
|
@ -384,6 +383,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
|
|||
SensitiveDataMasker,
|
||||
mask_sensitive_keys,
|
||||
)
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.llms.anthropic.messages.convertible_blocks import (
|
||||
deployment_drops_params,
|
||||
drop_non_string_tool_result_texts,
|
||||
|
|
@ -529,9 +529,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
|
|||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
LITELLM_CALL_ID_HEADER,
|
||||
attribute_of,
|
||||
headers_with_litellm_call_id,
|
||||
litellm_call_id_headers,
|
||||
openai_error_param,
|
||||
proxy_exception_for,
|
||||
route_failure_proxy_exception,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.periodic_reload_schedule import (
|
||||
|
|
@ -5190,7 +5193,7 @@ def _build_redis_usage_cache_from_environment() -> RedisCache | None:
|
|||
|
||||
Lets the proxy's coordination Redis (cross-pod tpm/rpm rate limits, spend
|
||||
tracking, pod lock manager) run when the response-cache backend is not a
|
||||
plain Redis KV cache (e.g. a semantic cache, disk, or s3).
|
||||
plain Redis KV cache (e.g. a semantic cache or s3).
|
||||
|
||||
Returns None when the environment carries no connection target (host, url,
|
||||
cluster nodes, or sentinel nodes).
|
||||
|
|
@ -9898,11 +9901,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe
|
|||
# Including it in the SSE response leaks internal details to clients.
|
||||
error_msg: Final = str(e)
|
||||
|
||||
proxy_exception: Final = ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", 500),
|
||||
proxy_exception: Final = route_failure_proxy_exception(
|
||||
e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
error_returned: Final = json.dumps({"error": proxy_exception.to_dict()})
|
||||
yield f"data: {error_returned}\n\n"
|
||||
|
|
@ -10214,29 +10214,6 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
|
|||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
|
||||
|
||||
|
||||
def _is_injected_stream_usage_artifact(chunk: object) -> bool:
|
||||
if not isinstance(chunk, ModelResponseStream):
|
||||
return False
|
||||
if chunk.provider_specific_fields is not None:
|
||||
return False
|
||||
return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or [])
|
||||
|
||||
|
||||
def _is_empty_streaming_choice(choice: StreamingChoices) -> bool:
|
||||
if choice.finish_reason is not None:
|
||||
return False
|
||||
if getattr(choice, "logprobs", None) is not None:
|
||||
return False
|
||||
delta: Final = choice.delta if hasattr(choice, "delta") else None
|
||||
if delta is None:
|
||||
return True
|
||||
return _all_values_none(delta.model_dump().values())
|
||||
|
||||
|
||||
def _all_values_none(values: Iterable[object]) -> bool:
|
||||
return all(value is None for value in values)
|
||||
|
||||
|
||||
async def _apply_streaming_chunk_hooks(
|
||||
*,
|
||||
chunk: Any,
|
||||
|
|
@ -10616,7 +10593,7 @@ async def async_data_generator(
|
|||
fallback_model_from_metadata=fallback_model_from_metadata,
|
||||
)
|
||||
|
||||
if strip_stream_usage and _is_injected_stream_usage_artifact(chunk):
|
||||
if strip_stream_usage and is_injected_stream_usage_artifact(chunk):
|
||||
if pending_fallback_event:
|
||||
yield _format_fallback_metadata_sse_event(
|
||||
fallback_model=fallback_model_from_metadata,
|
||||
|
|
@ -10735,11 +10712,10 @@ async def async_data_generator(
|
|||
{
|
||||
"error": sse_error_payload(client_error)[1]
|
||||
if isinstance(client_error, HTTPException)
|
||||
else ProxyException(
|
||||
message=getattr(client_error, "message", str(client_error)),
|
||||
type=getattr(client_error, "type", "None"),
|
||||
param=getattr(client_error, "param", "None"),
|
||||
code=getattr(client_error, "status_code", 500),
|
||||
else route_failure_proxy_exception(
|
||||
client_error,
|
||||
str(attribute_of(client_error, "message", client_error)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
).to_dict()
|
||||
},
|
||||
default=str,
|
||||
|
|
@ -10997,7 +10973,7 @@ class ProxyStartupEvent:
|
|||
"""
|
||||
Builds a standalone Redis cache from REDIS_* environment variables so
|
||||
use_redis_transaction_buffer can run when the proxy cache backend is not
|
||||
Redis (e.g. disk, s3).
|
||||
Redis (e.g. s3).
|
||||
|
||||
Returns None when the buffer is disabled, or when no Redis host or url
|
||||
is set in the environment.
|
||||
|
|
@ -12360,8 +12336,10 @@ async def model_list(
|
|||
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
|
||||
provider's model catalog. When present, the response is Codex's own
|
||||
catalog shape (`{"models": [...]}`) built from the same listing, with
|
||||
each model's `model_info.service_tiers` as its service tiers; absent,
|
||||
the OpenAI shape below is returned
|
||||
each model's `model_info.service_tiers` as its service tiers and, for a
|
||||
model that Codex release does not know, the reasoning efforts
|
||||
`/model_group/info` reports that it can parse; absent, the OpenAI shape
|
||||
below is returned
|
||||
- include_metadata: Include additional metadata in the response with fallback information
|
||||
- fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy")
|
||||
Defaults to "general" when include_metadata=true
|
||||
|
|
@ -12520,8 +12498,13 @@ async def model_list(
|
|||
admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
if client_version is not None:
|
||||
return Response(
|
||||
content=codex_model_list_body(
|
||||
admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id
|
||||
content=await asyncio.to_thread(
|
||||
codex_model_list_body,
|
||||
admin_listing,
|
||||
admin_entries,
|
||||
llm_router,
|
||||
team_id or user_api_key_dict.team_id,
|
||||
client_version=client_version,
|
||||
),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
|
@ -12596,7 +12579,15 @@ async def model_list(
|
|||
listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above
|
||||
if client_version is not None:
|
||||
return Response(
|
||||
content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels),
|
||||
content=await asyncio.to_thread(
|
||||
codex_model_list_body,
|
||||
listing,
|
||||
entries,
|
||||
llm_router,
|
||||
team_id or user_api_key_dict.team_id,
|
||||
labels,
|
||||
client_version=client_version,
|
||||
),
|
||||
media_type="application/json",
|
||||
)
|
||||
|
||||
|
|
@ -13104,14 +13095,12 @@ async def completion(
|
|||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, ProxyException):
|
||||
raise with_litellm_call_id(e, litellm_call_id)
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=getattr(e, "status_code", 500),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -13332,22 +13321,18 @@ async def moderations(
|
|||
if isinstance(e, ProxyException):
|
||||
raise with_litellm_call_id(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", str(e)),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
else:
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
code=getattr(e, "status_code", 500),
|
||||
)
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
)
|
||||
|
||||
|
||||
async def _audio_speech_chunk_generator(
|
||||
|
|
@ -13485,13 +13470,12 @@ async def audio_speech(
|
|||
detail=e.detail,
|
||||
headers=headers_with_litellm_call_id(e.headers, litellm_call_id),
|
||||
)
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", f"{e}"),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=getattr(e, "status_code", 500),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -13631,23 +13615,19 @@ async def audio_transcriptions(
|
|||
)
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", str(e.detail)),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e.detail)),
|
||||
status.HTTP_400_BAD_REQUEST,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
else:
|
||||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=getattr(e, "status_code", 500),
|
||||
)
|
||||
raise route_failure_proxy_exception(
|
||||
e,
|
||||
str(attribute_of(e, "message", e)),
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
headers=litellm_call_id_headers(litellm_call_id),
|
||||
openai_code=getattr(e, "code", None),
|
||||
)
|
||||
|
||||
|
||||
######################################################################
|
||||
|
|
@ -17247,7 +17227,7 @@ async def model_streaming_metrics(
|
|||
raise ProxyException(
|
||||
message=CommonProxyErrors.db_not_connected_error.value,
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -17381,7 +17361,7 @@ async def model_metrics(
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
startTime = startTime or datetime.now() - timedelta(days=DAYS_IN_A_MONTH)
|
||||
|
|
@ -17496,7 +17476,7 @@ async def model_metrics_slow_responses(
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if api_key is None or api_key == "undefined":
|
||||
|
|
@ -17585,7 +17565,7 @@ async def model_metrics_exceptions(
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -18069,6 +18049,7 @@ def get_model_group_info(
|
|||
registry.deployments(
|
||||
resolve_model_group_alias(registry.aliases(), model) or model,
|
||||
source="routed_group",
|
||||
team_id=team_id,
|
||||
)
|
||||
),
|
||||
}
|
||||
|
|
@ -18798,7 +18779,7 @@ async def login_v2(request: Request):
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", str(e)),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
else:
|
||||
|
|
@ -18806,7 +18787,7 @@ async def login_v2(request: Request):
|
|||
raise ProxyException(
|
||||
message=error_msg,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -18874,7 +18855,7 @@ async def login_v3(request: Request):
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", str(e)),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
else:
|
||||
|
|
@ -18882,7 +18863,7 @@ async def login_v3(request: Request):
|
|||
raise ProxyException(
|
||||
message=error_msg,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -18947,7 +18928,7 @@ async def login_v3_exchange(request: Request):
|
|||
raise ProxyException(
|
||||
message=str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1553,6 +1553,34 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "EU_JEV",
|
||||
"provider_display_name": "eu/jev",
|
||||
"litellm_provider": "eu_jev",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://eu-jev.com",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "eu_jev/eu-jeff-latest"
|
||||
},
|
||||
{
|
||||
"provider": "FalAI",
|
||||
"provider_display_name": "Fal AI",
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.proxy.auth.authorization_dependencies import (
|
|||
LogTeamLookupDependency,
|
||||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.spend_tracking.spend_capture_rate import (
|
||||
ProviderBillingCredentialMissing,
|
||||
|
|
@ -636,7 +637,7 @@ async def view_spend_tags(
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"/spend/tags Error({e})"),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -644,7 +645,7 @@ async def view_spend_tags(
|
|||
raise ProxyException(
|
||||
message="/spend/tags Error" + str(e),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -2142,7 +2143,7 @@ async def global_get_all_tag_names():
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"/spend/all_tag_names Error({e})"),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -2150,7 +2151,7 @@ async def global_get_all_tag_names():
|
|||
raise ProxyException(
|
||||
message="/spend/all_tag_names Error" + str(e),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -2224,7 +2225,7 @@ async def global_view_spend_tags(
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"/spend/tags Error({error_str})"),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -2232,7 +2233,7 @@ async def global_view_spend_tags(
|
|||
raise ProxyException(
|
||||
message="/spend/tags Error" + error_str,
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -2447,9 +2448,9 @@ async def calculate_spend(request: SpendCalculateRequest):
|
|||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", str(e)),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)),
|
||||
param=openai_error_param(e),
|
||||
code=error_status_code(e, status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
if isinstance(e, litellm.exceptions.ModelNotMappedError):
|
||||
raise ProxyException(
|
||||
|
|
@ -2461,9 +2462,9 @@ async def calculate_spend(request: SpendCalculateRequest):
|
|||
error_msg: Final = f"{e}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", 500),
|
||||
type=openai_error_type(e, error_status_code(e, 500)),
|
||||
param=openai_error_param(e),
|
||||
code=error_status_code(e, 500),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2633,7 +2634,7 @@ async def ui_view_spend_logs(
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -2700,7 +2701,7 @@ async def ui_view_spend_logs(
|
|||
raise ProxyException(
|
||||
message="Start date and end date are required",
|
||||
type="bad_request",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
formats: Final = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"]
|
||||
|
|
@ -3681,7 +3682,7 @@ async def view_spend_logs(
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"/spend/logs Error({e})"),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -3689,7 +3690,7 @@ async def view_spend_logs(
|
|||
raise ProxyException(
|
||||
message="/spend/logs Error" + str(e),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -3716,7 +3717,7 @@ async def global_spend_reset():
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -3747,7 +3748,7 @@ async def global_spend_refresh():
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
|
@ -3818,7 +3819,7 @@ async def global_spend_for_internal_user(
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
try:
|
||||
|
|
@ -3879,7 +3880,7 @@ async def global_spend_logs(
|
|||
raise ProxyException(
|
||||
message="Prisma Client is not initialized",
|
||||
type="internal_error",
|
||||
param="None",
|
||||
param=None,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -3923,7 +3924,7 @@ async def global_spend_logs(
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -3931,7 +3932,7 @@ async def global_spend_logs(
|
|||
raise ProxyException(
|
||||
message="/global/spend/logs Error" + error_str,
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
@ -3971,7 +3972,7 @@ async def global_spend():
|
|||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"/global/spend Error({error_str})"),
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
|
|
@ -3979,7 +3980,7 @@ async def global_spend():
|
|||
raise ProxyException(
|
||||
message="/global/spend Error" + error_str,
|
||||
type="internal_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
param=openai_error_param(e),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1617,7 +1617,7 @@ class Router(DeprecatedRouterMembers):
|
|||
self._core_deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
|
||||
self._core_deployment_latency_map = {}
|
||||
### CACHING ###
|
||||
cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache
|
||||
cache_type: Literal["local", "redis", "redis-semantic", "s3"] = "local" # default to an in-memory cache
|
||||
redis_cache = None
|
||||
cache_config: Final[dict[str, Any]] = {}
|
||||
|
||||
|
|
@ -12856,7 +12856,10 @@ class Router(DeprecatedRouterMembers):
|
|||
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
|
||||
reasoning_efforts_initialized = False
|
||||
reasoning_efforts_unknown = False
|
||||
model_list: Final = self._core_get_model_list_of_routed_group(model_group)
|
||||
serves_team_deployments: Final = team_id is not None and bool(
|
||||
self._team_deployments_named(team_id, model_group)
|
||||
)
|
||||
model_list: Final = self._core_get_model_list_of_routed_group(model_group, team_id)
|
||||
model_group_description: Final[str | None] = next(
|
||||
(d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)),
|
||||
None,
|
||||
|
|
@ -12864,7 +12867,8 @@ class Router(DeprecatedRouterMembers):
|
|||
for model in model_list:
|
||||
is_match = False
|
||||
if (
|
||||
"model_name" in model
|
||||
serves_team_deployments
|
||||
or "model_name" in model
|
||||
and model["model_name"] == model_group
|
||||
or "model_name" in model
|
||||
and self._core_pattern_router.route(model_group) is not None
|
||||
|
|
@ -14213,17 +14217,22 @@ class Router(DeprecatedRouterMembers):
|
|||
lambda name: self._core_wildcard_route_deployments(name, team_id),
|
||||
)
|
||||
|
||||
def _core_get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]:
|
||||
def _core_get_model_list_of_routed_group(
|
||||
self, model_group: str, team_id: str | None = None
|
||||
) -> list[DeploymentTypedDict]:
|
||||
"""
|
||||
The deployments a request the router has already resolved to model_group is
|
||||
served from: the ones named model_group, the routing group of that name, or
|
||||
the wildcard route matching it when neither exists.
|
||||
served from: team_id's own deployments under that public name when the team has
|
||||
any, else the ones named model_group, the routing group of that name, or the
|
||||
wildcard route matching it when neither exists.
|
||||
|
||||
Unlike get_model_list, model_group's own model_group_alias entry is not
|
||||
followed. The router resolves an alias exactly once, so a group reached as
|
||||
an alias target is served by its own deployments, never by a second hop:
|
||||
in the chain X -> T -> U a request to X is served from T's deployments.
|
||||
"""
|
||||
if team_id is not None and self._team_deployments_named(team_id, model_group):
|
||||
return self._core_get_all_deployments(model_name=model_group, team_id=team_id)
|
||||
own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns
|
||||
list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -419,7 +419,7 @@ class RouterRegistry:
|
|||
if source == "named":
|
||||
return r._core_deployments_named(model_name)
|
||||
if source == "routed_group":
|
||||
return r._core_get_model_list_of_routed_group(model_name)
|
||||
return r._core_get_model_list_of_routed_group(model_name, team_id)
|
||||
if source == "wildcard":
|
||||
return r._core_wildcard_route_deployments(model_name, team_id)
|
||||
if source == "routable":
|
||||
|
|
|
|||
|
|
@ -27,20 +27,22 @@ it above.
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias, get_args
|
||||
|
||||
from typing_extensions import assert_never
|
||||
from typing import Final, TypeAlias, get_args
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params
|
||||
from litellm.litellm_core_utils.reasoning_effort_strength import (
|
||||
REASONING_EFFORT_STRENGTH_ORDER,
|
||||
ReasoningEffortRounding,
|
||||
nearest_declared_reasoning_effort,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.types.llms.openai import REASONING_EFFORT
|
||||
|
||||
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
|
||||
RequestedReasoningEffort: TypeAlias = str | Mapping[str, object] | None
|
||||
|
||||
REASONING_EFFORT_ADVERTISEMENT_ORDER: Final[tuple[str, ...]] = get_args(REASONING_EFFORT)
|
||||
|
|
@ -115,35 +117,6 @@ def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) -
|
|||
return declared_reasoning_efforts(entry)
|
||||
|
||||
|
||||
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
|
||||
_STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)})
|
||||
|
||||
|
||||
def nearest_declared_reasoning_effort(
|
||||
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
|
||||
) -> str:
|
||||
"""Rounding up takes the weakest declared level at least as strong as the request, else the
|
||||
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
|
||||
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
|
||||
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
|
||||
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
|
||||
a model that cannot turn it off says so itself. A level outside the strength order is likewise
|
||||
returned as is for upstream to judge."""
|
||||
ranked: Final = sorted(
|
||||
(effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort]
|
||||
)
|
||||
if requested in ranked or requested not in _STRENGTH_RANK or not ranked:
|
||||
return requested
|
||||
strength: Final = _STRENGTH_RANK[requested]
|
||||
match rounding:
|
||||
case "up":
|
||||
return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= strength), ranked[-1])
|
||||
case "down":
|
||||
return next((effort for effort in reversed(ranked) if _STRENGTH_RANK[effort] <= strength), ranked[0])
|
||||
case _:
|
||||
assert_never(rounding)
|
||||
|
||||
|
||||
_ROUNDING_KEY: Final = "reasoning_effort_rounding"
|
||||
_DEFAULT_EFFORT: Final = "default"
|
||||
_EFFORT_KEY: Final = "effort"
|
||||
|
|
@ -163,7 +136,7 @@ def deployment_reasoning_effort_levels(deployment_model_info: object) -> tuple[s
|
|||
|
||||
def deployment_rounds_reasoning_effort(deployment_model_info: object) -> bool:
|
||||
declared: Final = deployment_reasoning_effort_levels(deployment_model_info)
|
||||
return declared is not None and any(level in _STRENGTH_RANK for level in declared)
|
||||
return declared is not None and any(level in REASONING_EFFORT_STRENGTH_ORDER for level in declared)
|
||||
|
||||
|
||||
def with_adaptive_thinking_as_effort(
|
||||
|
|
|
|||
|
|
@ -98,6 +98,8 @@ class Feature(str, Enum):
|
|||
"""Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature."""
|
||||
TOOL_CUSTOM_FIELD = "tool_custom_field"
|
||||
"""Tools carrying a non-spec custom object, which Python drops or hoists for the provider."""
|
||||
STRICT_TOOLS_RETRY = "strict_tools_retry"
|
||||
"""Strict tools under drop_params, which Python sends again without strict if Claude rejects their limits."""
|
||||
CLIENT_HEADERS = "client_headers"
|
||||
"""Proxy-forwarded headers that Python merges into extra_headers."""
|
||||
DEPLOYMENT_HOOK = "deployment_hook"
|
||||
|
|
@ -179,6 +181,7 @@ MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
|
|||
Feature.STREAM_COST_INJECTION,
|
||||
Feature.THINKING_SIGNATURE_RETRY,
|
||||
Feature.TOOL_CUSTOM_FIELD,
|
||||
Feature.STRICT_TOOLS_RETRY,
|
||||
}
|
||||
|
||||
RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
|
||||
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
|
||||
|
|
@ -48,6 +49,13 @@ def tool_custom_field(request: NativeCall) -> bool:
|
|||
return any("custom" in (optional_mapping(tool) or {}) for tool in tools)
|
||||
|
||||
|
||||
def strict_tools_retry(request: NativeCall) -> bool:
|
||||
tools: Final = optional_sequence(request.resolved.get("tools")) or ()
|
||||
return drops_unsupported_params(request.resolved.get("drop_params")) and any(
|
||||
(optional_mapping(tool) or {}).get("strict") is True for tool in tools
|
||||
)
|
||||
|
||||
|
||||
def stream_cost_injection(request: NativeCall) -> bool:
|
||||
"""Python adds response cost to streaming usage when the global setting is truthy."""
|
||||
return streaming(request) and bool(litellm.include_cost_in_streaming_usage)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
|
||||
from litellm.litellm_core_utils.stream_usage_chunks import is_injected_stream_usage_artifact
|
||||
from litellm.types.utils import Choices, ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse
|
||||
|
||||
|
||||
class TextCompletionStreamWrapper:
|
||||
|
|
@ -10,11 +11,18 @@ class TextCompletionStreamWrapper:
|
|||
model,
|
||||
stream_options: dict | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
usage_injected_by_proxy: bool = False,
|
||||
):
|
||||
self.completion_stream = completion_stream
|
||||
self.model = model
|
||||
self.stream_options = stream_options
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
self.usage_injected_by_proxy: Final = usage_injected_by_proxy
|
||||
self.include_usage: Final = (
|
||||
not usage_injected_by_proxy
|
||||
and stream_options is not None
|
||||
and stream_options.get("include_usage", False) is True
|
||||
)
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
|
@ -22,7 +30,7 @@ class TextCompletionStreamWrapper:
|
|||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
def convert_to_text_completion_object(self, chunk: ModelResponse):
|
||||
def convert_to_text_completion_object(self, chunk: ModelResponse | ModelResponseStream):
|
||||
try:
|
||||
response: Final = TextCompletionResponse()
|
||||
response["id"] = chunk.get("id", None)
|
||||
|
|
@ -43,8 +51,7 @@ class TextCompletionStreamWrapper:
|
|||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
if self.include_usage:
|
||||
response.usage = chunk.get("usage", None)
|
||||
else:
|
||||
response.usage = None
|
||||
|
|
@ -53,6 +60,9 @@ class TextCompletionStreamWrapper:
|
|||
except Exception as e:
|
||||
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
|
||||
|
||||
def _is_injected_usage_artifact(self, chunk: ModelResponseStream) -> bool:
|
||||
return self.usage_injected_by_proxy and is_injected_stream_usage_artifact(chunk)
|
||||
|
||||
def __next__(self):
|
||||
# model_response = ModelResponse(stream=True, model=self.model)
|
||||
TextCompletionResponse()
|
||||
|
|
@ -60,8 +70,9 @@ class TextCompletionStreamWrapper:
|
|||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
|
||||
continue
|
||||
return self.convert_to_text_completion_object(chunk=chunk)
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
|
|
@ -81,8 +92,9 @@ class TextCompletionStreamWrapper:
|
|||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
if isinstance(chunk, ModelResponseStream) and self._is_injected_usage_artifact(chunk):
|
||||
continue
|
||||
return self.convert_to_text_completion_object(chunk=chunk)
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ class LiteLLMCacheType(str, Enum):
|
|||
REDIS_SEMANTIC = "redis-semantic"
|
||||
VALKEY_SEMANTIC = "valkey-semantic"
|
||||
S3 = "s3"
|
||||
DISK = "disk"
|
||||
QDRANT_SEMANTIC = "qdrant-semantic"
|
||||
AZURE_BLOB = "azure-blob"
|
||||
GCS = "gcs"
|
||||
|
|
|
|||
|
|
@ -204,9 +204,11 @@ class MCPServer(LiteLLMBaseModel):
|
|||
access_groups: list[str] | None = None
|
||||
allow_all_keys: bool = False
|
||||
available_on_public_internet: bool = True
|
||||
# Legacy opt-in to upstream-delegated authentication for ``oauth2``
|
||||
# servers. LiteLLM admission still applies; use ``oauth_delegate`` for the
|
||||
# supported client-forwarded OAuth flow.
|
||||
# Deprecated opt-in to upstream-delegated authentication for ``oauth2``
|
||||
# servers. A bare ``Authorization`` that is not a LiteLLM credential is the
|
||||
# upstream token and is forwarded; see
|
||||
# ``MCPRequestHandler._admit_legacy_delegate_bearer``. New servers should use
|
||||
# ``true_passthrough`` or ``oauth_delegate``.
|
||||
#
|
||||
# Honored only for ``auth_type == oauth2``; ignored for any other
|
||||
# ``auth_type``. OAuth pass-through for non-oauth2 servers
|
||||
|
|
|
|||
100
litellm/types/passthrough_endpoints/eu_jev.py
Normal file
100
litellm/types/passthrough_endpoints/eu_jev.py
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Annotated, Literal, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, Field
|
||||
|
||||
from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer
|
||||
from litellm.types.llms.base import LiteLLMPydanticObjectBase
|
||||
|
||||
EuJevEntry: TypeAlias = Annotated[
|
||||
DecisionsJSON | None,
|
||||
Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"),
|
||||
]
|
||||
|
||||
|
||||
class EuJevChoiceQuestion(LiteLLMPydanticObjectBase):
|
||||
type: Literal["choice"]
|
||||
instructions: EuJevEntry
|
||||
criteria: Annotated[
|
||||
Mapping[str, EuJevEntry],
|
||||
Field(
|
||||
min_length=2,
|
||||
max_length=64,
|
||||
description="Two to 64 options, option name to description. A null description uses the option name",
|
||||
),
|
||||
]
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class EuJevNoulQuestion(LiteLLMPydanticObjectBase):
|
||||
type: Literal["noul"]
|
||||
instructions: EuJevEntry
|
||||
criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = (
|
||||
None
|
||||
)
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class EuJevScoreQuestion(LiteLLMPydanticObjectBase):
|
||||
type: Literal["score"]
|
||||
instructions: EuJevEntry
|
||||
criteria: Annotated[
|
||||
Sequence[EuJevEntry],
|
||||
Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"),
|
||||
]
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
EuJevQuestion: TypeAlias = Annotated[
|
||||
EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class EuJevDecisionRequest(LiteLLMPydanticObjectBase):
|
||||
model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = (
|
||||
"eu-jeff-latest"
|
||||
)
|
||||
state: Annotated[
|
||||
Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object],
|
||||
Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"),
|
||||
]
|
||||
questions: Annotated[
|
||||
Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion],
|
||||
Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"),
|
||||
]
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")]
|
||||
|
||||
|
||||
class EuJevUsage(LiteLLMPydanticObjectBase):
|
||||
input_tokens: Annotated[int, Field(ge=0)]
|
||||
output_tokens: Annotated[int, Field(ge=0)]
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class EuJevMeta(LiteLLMPydanticObjectBase):
|
||||
request_id: str
|
||||
mode: Literal["live"]
|
||||
latency_ms: Annotated[
|
||||
int, Field(description="Server-side time for token quoting, credit reservation and inference")
|
||||
]
|
||||
cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
||||
class EuJevDecisionResponse(LiteLLMPydanticObjectBase):
|
||||
model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")]
|
||||
answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")]
|
||||
usage: EuJevUsage
|
||||
meta: EuJevMeta
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
|
@ -169,6 +169,10 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
|
|||
# above; a forged value could at most narrow, but the stripping keeps the field's provenance
|
||||
# single-owner so its meaning stays trustworthy.
|
||||
mcp_session_resource_server_ids: tuple[str, ...] | None = Field(default=None, exclude=True)
|
||||
# Set only by MCP admission when a legacy delegate_auth_to_upstream caller's Authorization is the
|
||||
# upstream token rather than a LiteLLM credential, so egress forwards it. Stripped from validated
|
||||
# input like the markers above: a forged value would forward a LiteLLM credential upstream.
|
||||
mcp_forwards_caller_authorization: bool = Field(default=False, exclude=True)
|
||||
mcp_toolset_id: str | None = Field(default=None, exclude=True)
|
||||
authenticated_by_custom_auth: bool = Field(default=False, exclude=True)
|
||||
via_virtual_key: bool = Field(
|
||||
|
|
@ -230,6 +234,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response ob
|
|||
values.pop("mcp_explicit_grants_only", None)
|
||||
values.pop("mcp_source_team_rpm_limits", None)
|
||||
values.pop("mcp_session_resource_server_ids", None)
|
||||
values.pop("mcp_forwards_caller_authorization", None)
|
||||
values.pop("mcp_toolset_id", None)
|
||||
values.pop("via_virtual_key", None)
|
||||
values.pop("authenticated_by_custom_auth", None)
|
||||
|
|
|
|||
|
|
@ -4,9 +4,11 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
|
|||
|
||||
import datetime
|
||||
import enum
|
||||
import functools
|
||||
import re
|
||||
from collections.abc import Container, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Annotated,
|
||||
|
|
@ -989,9 +991,10 @@ class AlertingConfig(LiteLLMBaseModel):
|
|||
alerting_threshold: float | None = 300
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
|
||||
"""Resolve a class's annotations, keeping each resolved annotation opaque."""
|
||||
return get_type_hints(model_class)
|
||||
return MappingProxyType(get_type_hints(model_class))
|
||||
|
||||
|
||||
class ModelGroupInfo(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -4489,6 +4489,7 @@ class LlmProviders(str, Enum):
|
|||
STRANDS_DECIDER = "strands_decider"
|
||||
LAYA = "laya"
|
||||
BESPOKE = "bespoke"
|
||||
EU_JEV = "eu_jev"
|
||||
MISTRAL = "mistral"
|
||||
MILVUS = "milvus"
|
||||
GROQ = "groq"
|
||||
|
|
|
|||
|
|
@ -9968,11 +9968,12 @@ class ProviderConfigManager:
|
|||
return litellm.LayaDecisionsConfig()
|
||||
if provider == LlmProviders.BESPOKE:
|
||||
return litellm.BespokeDecisionsConfig()
|
||||
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI):
|
||||
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV):
|
||||
configs: Final = {
|
||||
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
|
||||
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
|
||||
LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig,
|
||||
LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig,
|
||||
}
|
||||
return configs[provider]()
|
||||
return None
|
||||
|
|
@ -10923,19 +10924,8 @@ class ProviderConfigManager:
|
|||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_image_edit_config(
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
) -> BaseImageEditConfig | None:
|
||||
if LlmProviders.OPENAI == provider:
|
||||
from litellm.llms.openai.image_edit import OpenAIImageEditConfig
|
||||
|
||||
return OpenAIImageEditConfig()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config
|
||||
|
||||
return get_hosted_vllm_image_edit_config(model=model)
|
||||
elif LlmProviders.AZURE == provider:
|
||||
def _get_model_independent_image_edit_config(provider: LlmProviders) -> BaseImageEditConfig | None:
|
||||
if LlmProviders.AZURE == provider:
|
||||
from litellm.llms.azure.image_edit.transformation import (
|
||||
AzureImageEditConfig,
|
||||
)
|
||||
|
|
@ -10953,6 +10943,36 @@ class ProviderConfigManager:
|
|||
)
|
||||
|
||||
return BlackForestLabsImageEditConfig()
|
||||
elif LlmProviders.LITELLM_PROXY == provider:
|
||||
from litellm.llms.litellm_proxy.image_edit.transformation import (
|
||||
LiteLLMProxyImageEditConfig,
|
||||
)
|
||||
|
||||
return LiteLLMProxyImageEditConfig()
|
||||
elif LlmProviders.DASHSCOPE == provider:
|
||||
from litellm.llms.dashscope.image_edit.transformation import (
|
||||
DashScopeImageEditConfig,
|
||||
)
|
||||
|
||||
return DashScopeImageEditConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_provider_image_edit_config(
|
||||
model: str,
|
||||
provider: LlmProviders,
|
||||
) -> BaseImageEditConfig | None:
|
||||
model_independent_config: Final = ProviderConfigManager._get_model_independent_image_edit_config(provider)
|
||||
if model_independent_config is not None:
|
||||
return model_independent_config
|
||||
if LlmProviders.OPENAI == provider:
|
||||
from litellm.llms.openai.image_edit import OpenAIImageEditConfig
|
||||
|
||||
return OpenAIImageEditConfig()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config
|
||||
|
||||
return get_hosted_vllm_image_edit_config(model=model)
|
||||
elif LlmProviders.FAL_AI == provider:
|
||||
from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config
|
||||
|
||||
|
|
@ -10965,12 +10985,6 @@ class ProviderConfigManager:
|
|||
from litellm.llms.gemini.image_edit import get_gemini_image_edit_config
|
||||
|
||||
return get_gemini_image_edit_config(model)
|
||||
elif LlmProviders.LITELLM_PROXY == provider:
|
||||
from litellm.llms.litellm_proxy.image_edit.transformation import (
|
||||
LiteLLMProxyImageEditConfig,
|
||||
)
|
||||
|
||||
return LiteLLMProxyImageEditConfig()
|
||||
elif LlmProviders.VERTEX_AI == provider:
|
||||
from litellm.llms.vertex_ai.image_edit import (
|
||||
get_vertex_ai_image_edit_config,
|
||||
|
|
@ -11192,6 +11206,12 @@ class ProviderConfigManager:
|
|||
return AWSPollyTextToSpeechConfig()
|
||||
elif litellm.LlmProviders.EDENAI == provider:
|
||||
return litellm.EdenAITextToSpeechConfig()
|
||||
elif litellm.LlmProviders.DASHSCOPE == provider:
|
||||
from litellm.llms.dashscope.text_to_speech.transformation import (
|
||||
DashScopeTextToSpeechConfig,
|
||||
)
|
||||
|
||||
return DashScopeTextToSpeechConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -18346,25 +18346,31 @@
|
|||
"dashscope/qwen-image-2.0": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.035,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-2.0-pro": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.075,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-3.0": {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"output_cost_per_image": 0.03,
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-image-3.0-pro": {
|
||||
|
|
@ -18372,7 +18378,54 @@
|
|||
"mode": "image_generation",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
"/v1/images/generations",
|
||||
"/v1/images/edits"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen3-tts-flash": {
|
||||
"input_cost_per_character": 1e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.1-tts-flash": {
|
||||
"input_cost_per_token": 2.3e-07,
|
||||
"output_cost_per_token": 1.87e-06,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.1-tts-plus": {
|
||||
"input_cost_per_character": 3e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.0-tts-plus": {
|
||||
"input_cost_per_character": 2e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/qwen-audio-3.0-tts-flash": {
|
||||
"input_cost_per_character": 1.5e-05,
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"dashscope/wan3.0-video": {
|
||||
|
|
@ -72330,6 +72383,28 @@
|
|||
"source": "https://docs.typesafe.ai/models",
|
||||
"supports_box_questions": true
|
||||
},
|
||||
"eu_jev/eu-jeff-1.0.0": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
"litellm_provider": "eu_jev",
|
||||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
},
|
||||
"eu_jev/eu-jeff-latest": {
|
||||
"input_cost_per_token": 4.15e-08,
|
||||
"litellm_provider": "eu_jev",
|
||||
"max_input_tokens": 32768,
|
||||
"mode": "evaluation",
|
||||
"output_cost_per_token": 0.0,
|
||||
"source": "https://eu-jev.com/docs",
|
||||
"supported_endpoints": [
|
||||
"/v1/systemone"
|
||||
]
|
||||
},
|
||||
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
|
|||
|
|
@ -78,11 +78,6 @@ id = "GHSA-r4j5-9xpp-3985"
|
|||
ignoreUntil = 2026-11-27
|
||||
reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-w8v5-vhqr-4h9v"
|
||||
ignoreUntil = 2026-11-27
|
||||
reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-45c3-73f7-pv4m"
|
||||
ignoreUntil = 2026-11-27
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ authors = [
|
|||
]
|
||||
dependencies = [
|
||||
"fastuuid>=0.14.0,<1.0; python_version < '3.15'",
|
||||
"filelock>=3.16.1,<4.0",
|
||||
"filelock>=3.16.1,<5",
|
||||
"httpx[http2]>=0.28.0,<1.0",
|
||||
"openai>=2.20.0,<4.0.0",
|
||||
"python-dateutil>=2.8.2,<3.0",
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue