From dacaaad72ed25663ad7f40941c48f653a45289be Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 29 Oct 2026 18:20:47 -0700 Subject: [PATCH 01/34] build: allow filelock 4.x and sync package metadata (#49137) * build: allow filelock 4.x filelock 3.x is the only line the current cap permits, and three CVEs (CVE-2026-112940, CVE-2026-112913, CVE-2026-112907) are fixed only in 4.2.1+. The cap means every downstream that depends on litellm is held on a vulnerable filelock with no way out, since pip has no override. litellm uses FileLock(path, timeout=..., mode=...) and Timeout, in litellm/proxy/client/cli/commands/configure_profiles.py and claude_settings.py. Both keyword arguments still exist in 4.2.4 and Timeout is still exported, so the two call sites are unaffected. The CVEs are in the SQLite-backed ReadWriteLock / AsyncReadWriteLock, which litellm does not use. <5 rather than a floor bump because litellm supports Python 3.10 and filelock 4.x requires >=3.11: widening the cap lets 3.11+ resolve 4.x while 3.10 keeps a 3.x filelock automatically, through filelock's own requires_python. * build: sync core filelock bound and dependency lock --------- Co-authored-by: Speculator55005 <50082482+fas89@users.noreply.github.com> --- packaging/litellm-core/pyproject.toml | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index db50589a675..1141e69c0aa 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "fastuuid>=0.14.0,<1.0; python_version < '3.15'", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<5", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dateutil>=2.8.2,<3.0", diff --git a/pyproject.toml b/pyproject.toml index bff00a2dbf1..de1abeddd7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0; python_version < '3.15'", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<5", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dotenv>=1.0.0,<2.0", diff --git a/uv.lock b/uv.lock index f36fd3495a9..2ba97d15327 100644 --- a/uv.lock +++ b/uv.lock @@ -5242,7 +5242,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", marker = "python_full_version < '3.15'", specifier = ">=0.14.0,<1.0" }, - { name = "filelock", specifier = ">=3.16.1,<4.0" }, + { name = "filelock", specifier = ">=3.16.1,<5" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, From 636e70708984527aa2a4b89fdeb16886f7cff278 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:20:52 -0700 Subject: [PATCH 02/34] fix(proxy)!: stop answering with the literal string "None" as error type and param on the proxy-server, auth, spend, and key health routes (#39540) * fix(proxy): stop answering with the literal string "None" on the proxy-server, auth, and health routes The error objects these routes return defaulted `type` and `param` to the four-character string "None", so a client's error handler matched no known OpenAI type and fell into its generic branch. Route them through the shared openai_error_payload helpers instead, so `type` comes from the carried type or the HTTP status and `param` serializes as JSON null. * fix(proxy): stop answering with the literal string "None" on the spend and management routes The error objects these routes return defaulted `type` and `param` to the four-character string "None", so a client's error handler matched no known OpenAI type and fell into its generic branch. Route them through the shared openai_error_payload helpers instead, so `type` comes from the carried type or the HTTP status and `param` serializes as JSON null. * test(proxy): pin the mid-stream failure frame and the DB outage body to a JSON null param * fix(proxy): type upstream timeouts as timeout_error and update integration asserts for the null param A 408 with no carried type now answers timeout_error instead of invalid_request_error, so a client branching on type keeps retrying it. Three integration suites still compared auth rejections against param="None". * fix(proxy): keep a 408 on main's invalid_request_error type and pin the transcription file cap's 400 * fix(proxy): type a route failure by the status it answers with after the tag-routing and no-deployment remap * fix(proxy): keep auth_error on proxy-raised 401/403 route failures and ban the string None as an error type or param * fix(proxy): type a hook's 401 or 403 on completions, moderations and transcription by its status, as chat does Only a status the proxy remapped into 401 or 403, such as a tag-routing denial, keeps auth_error. * test(proxy): build the transcription file-cap test on a real router so it holds after the router refactor --- .github/workflows/test-linting.yml | 6 + litellm/proxy/auth/auth_exception_handler.py | 7 +- litellm/proxy/auth/handle_jwt.py | 2 +- litellm/proxy/auth/route_checks.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 12 +- .../common_utils/openai_error_payload.py | 31 ++++- litellm/proxy/db/exception_handler.py | 2 +- .../key_management_endpoints.py | 6 +- .../team_callback_endpoints.py | 11 +- litellm/proxy/proxy_server.py | 106 ++++++++--------- .../spend_management_endpoints.py | 49 ++++---- .../check_no_none_string_error_fields.py | 71 ++++++++++++ .../test_check_no_none_string_error_fields.py | 38 +++++++ .../test_guardrail_submissions.py | 2 +- .../test_jwt_key_mapping_lifecycle.py | 4 +- .../authorization/test_memory_ownership.py | 2 +- .../authorization/test_team_admin_gate.py | 2 +- .../test_unknown_key_generic_401.py | 8 +- .../test_user_self_update_guard.py | 2 +- .../authorization/test_warmed_policy.py | 2 +- .../authorization/test_workflow_ownership.py | 2 +- .../management/test_health_routing.py | 2 +- .../management/test_scim_admin_group.py | 2 +- .../test_user_delete_revokes_keys.py | 2 +- ...st_arize_otel_v2_openinference_sad_edge.py | 2 +- .../test_presidio_entity_masking.py | 2 +- .../security/test_config_overrides.py | 6 +- .../spend/test_global_spend_reset.py | 2 +- .../proxy/auth/test_auth_exception_handler.py | 14 +++ tests/unit/proxy/auth/test_handle_jwt.py | 1 + .../unit/proxy/auth/test_user_api_key_auth.py | 27 +++++ .../test_internal_user_endpoints.py | 2 +- .../test_key_management_endpoints.py | 17 +++ .../test_team_endpoints.py | 2 +- .../proxy_server/test_exception_handlers.py | 2 +- .../proxy/proxy_server/test_routes_audio.py | 107 +++++++++++++++++- .../proxy_server/test_routes_completions.py | 38 +++++++ .../proxy_server/test_streaming_helpers.py | 35 +++++- .../test_spend_management_endpoints.py | 16 +++ ...test_proxy_server_endpoints_and_startup.py | 90 +++++++++++++++ 40 files changed, 606 insertions(+), 130 deletions(-) create mode 100644 tests/code_coverage_tests/check_no_none_string_error_fields.py create mode 100644 tests/code_coverage_tests/test_check_no_none_string_error_fields.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index bd7343ea222..88c66f0011d 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -630,6 +630,12 @@ jobs: - name: check_migrations_lock_timeout run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py + - name: check_no_none_string_error_fields + run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py + + - name: test_check_no_none_string_error_fields + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py + - name: check_no_publicly_known_master_key run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index ed743764fce..2ec2274db2b 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor ) from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError +from litellm.proxy.common_utils.openai_error_payload import openai_error_param from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException: message=str(e.detail), internal_message=e.internal_message, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=e.status_code, ) if isinstance(e, HTTPException): return ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), ) if isinstance(e, ProxyException): @@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: return ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_401_UNAUTHORIZED, ) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9d800a1fd63..7bd7b1226b1 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException: f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}" ), type=ProxyErrorTypes.auth_provider_unavailable, - param="None", + param=None, code=status.HTTP_503_SERVICE_UNAVAILABLE, ) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 56be44600fd..105acb237af 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -302,7 +302,7 @@ class RouteChecks: f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2b6471952fd..67410e867d6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2586,7 +2586,7 @@ async def user_api_key_auth_builder( raise ProxyException( message="Authentication Error, No api key passed in.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) elif api_key == "": @@ -2594,7 +2594,7 @@ async def user_api_key_auth_builder( raise ProxyException( message="Authentication Error, Malformed API Key passed in. Ensure Key has `Bearer ` prefix.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2835,7 +2835,7 @@ async def user_api_key_auth_builder( raise ProxyException( message=f"Authentication Error, Tried to access route={route}, which is only for MASTER KEY", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2968,7 +2968,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e raise ProxyException( message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) await enforce_key_and_fallback_model_access( @@ -3019,7 +3019,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e "Keys owned by this user cannot be used." ), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3325,7 +3325,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e raise ProxyException( message="Authentication Error, Invalid proxy server token passed", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) if valid_token_dict is not None: diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index f92fbeef9a1..53b2a05d40f 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -147,8 +147,37 @@ def _is_failover_refusal(exc: Exception) -> bool: return any(refusal in message for refusal in _FAILOVER_REFUSALS) +def answered_status_code(message: str, status_code: int) -> int: + return int(proxy_exception_status_code(message, str(status_code))) + + +def route_failure_proxy_exception( + exc: Exception, + message: str, + default_status_code: int, + headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers + openai_code: str | None = None, +) -> ProxyException: + raised_status: Final = error_status_code(exc, default_status_code) + answered_status: Final = answered_status_code(message, raised_status) + return ProxyException( + message=message, + type=_route_failure_error_type(exc, raised_status, answered_status), + param=openai_error_param(exc), + headers=headers, + openai_code=openai_code, + code=answered_status, + ) + + +def _route_failure_error_type(exc: Exception, raised_status: int, answered_status: int) -> str: + if answered_status != raised_status and answered_status in _AUTH_FAILURE_STATUS_CODES: + return ProxyErrorTypes.auth_error.value + return openai_error_type(exc, answered_status) + + def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: - answered_status: Final = int(proxy_exception_status_code(message, str(status_code))) + answered_status: Final = answered_status_code(message, status_code) return ProxyException( message=message, type=_catch_all_error_type(exc, answered_status), diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 3f7ea334a99..baf53604ea5 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -435,7 +435,7 @@ class PrismaDBExceptionHandler: return ProxyException( message=PrismaDBExceptionHandler.database_unavailable_message(e), type=ProxyErrorTypes.no_db_connection, - param="None", + param=None, code=HTTPStatus.SERVICE_UNAVAILABLE.value, ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 69f28b6d388..a6814b593ca 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -75,7 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) -from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for +from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache @@ -4959,7 +4959,7 @@ async def info_key_fn( raise ProxyException( message="Internal server error from /key/info. Details are in the proxy server logs.", type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -8156,7 +8156,7 @@ async def key_health( raise ProxyException( message=f"Key health check failed: {e}", type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index d1e1c4bd25e..5b778392664 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -46,6 +46,7 @@ from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy encrypt_callback_vars, is_sensitive_callback_key, ) +from litellm.proxy.common_utils.openai_error_payload import openai_error_param from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports _get_validated_callback_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export convert_key_logging_metadata_to_callback, @@ -463,7 +464,7 @@ async def add_team_callbacks( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -606,7 +607,7 @@ async def delete_team_callback( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) else: @@ -747,7 +748,7 @@ async def disable_team_logging( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -838,7 +839,7 @@ async def get_team_callbacks( raise ProxyException( message=getattr(e, "detail", f"Internal Server Error({e})"), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -846,6 +847,6 @@ async def get_team_callbacks( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bd221bca98..b0ccf74c63b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -529,9 +529,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, + attribute_of, headers_with_litellm_call_id, litellm_call_id_headers, + openai_error_param, proxy_exception_for, + route_failure_proxy_exception, with_litellm_call_id, ) from litellm.proxy.common_utils.periodic_reload_schedule import ( @@ -9898,11 +9901,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe # Including it in the SSE response leaks internal details to clients. error_msg: Final = str(e) - proxy_exception: Final = ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + proxy_exception: Final = route_failure_proxy_exception( + e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR ) error_returned: Final = json.dumps({"error": proxy_exception.to_dict()}) yield f"data: {error_returned}\n\n" @@ -10712,11 +10712,10 @@ async def async_data_generator( { "error": sse_error_payload(client_error)[1] if isinstance(client_error, HTTPException) - else ProxyException( - message=getattr(client_error, "message", str(client_error)), - type=getattr(client_error, "type", "None"), - param=getattr(client_error, "param", "None"), - code=getattr(client_error, "status_code", 500), + else route_failure_proxy_exception( + client_error, + str(attribute_of(client_error, "message", client_error)), + status.HTTP_500_INTERNAL_SERVER_ERROR, ).to_dict() }, default=str, @@ -13096,14 +13095,12 @@ async def completion( log_llm_api_exception(e, litellm_call_id) if isinstance(e, ProxyException): raise with_litellm_call_id(e, litellm_call_id) - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) @@ -13324,22 +13321,18 @@ async def moderations( if isinstance(e, ProxyException): raise with_litellm_call_id(e, litellm_call_id) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_400_BAD_REQUEST, headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", 500), ) + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, + headers=litellm_call_id_headers(litellm_call_id), + ) async def _audio_speech_chunk_generator( @@ -13477,13 +13470,12 @@ async def audio_speech( detail=e.detail, headers=headers_with_litellm_call_id(e.headers, litellm_call_id), ) - raise ProxyException( - message=getattr(e, "message", f"{e}"), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) @@ -13623,23 +13615,19 @@ async def audio_transcriptions( ) log_llm_api_exception(e, litellm_call_id) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e.detail)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e.detail)), + status.HTTP_400_BAD_REQUEST, headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - headers=litellm_call_id_headers(litellm_call_id), - openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, + headers=litellm_call_id_headers(litellm_call_id), + openai_code=getattr(e, "code", None), + ) ###################################################################### @@ -17239,7 +17227,7 @@ async def model_streaming_metrics( raise ProxyException( message=CommonProxyErrors.db_not_connected_error.value, type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -17373,7 +17361,7 @@ async def model_metrics( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) startTime = startTime or datetime.now() - timedelta(days=DAYS_IN_A_MONTH) @@ -17488,7 +17476,7 @@ async def model_metrics_slow_responses( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) if api_key is None or api_key == "undefined": @@ -17577,7 +17565,7 @@ async def model_metrics_exceptions( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18791,7 +18779,7 @@ async def login_v2(request: Request): raise ProxyException( message=getattr(e, "detail", str(e)), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: @@ -18799,7 +18787,7 @@ async def login_v2(request: Request): raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18867,7 +18855,7 @@ async def login_v3(request: Request): raise ProxyException( message=getattr(e, "detail", str(e)), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: @@ -18875,7 +18863,7 @@ async def login_v3(request: Request): raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18940,7 +18928,7 @@ async def login_v3_exchange(request: Request): raise ProxyException( message=str(e), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 38dcc3b06df..89200792c0b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -50,6 +50,7 @@ from litellm.proxy.auth.authorization_dependencies import ( LogTeamLookupDependency, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, @@ -636,7 +637,7 @@ async def view_spend_tags( raise ProxyException( message=getattr(e, "detail", f"/spend/tags Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -644,7 +645,7 @@ async def view_spend_tags( raise ProxyException( message="/spend/tags Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2142,7 +2143,7 @@ async def global_get_all_tag_names(): raise ProxyException( message=getattr(e, "detail", f"/spend/all_tag_names Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -2150,7 +2151,7 @@ async def global_get_all_tag_names(): raise ProxyException( message="/spend/all_tag_names Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2224,7 +2225,7 @@ async def global_view_spend_tags( raise ProxyException( message=getattr(e, "detail", f"/spend/tags Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -2232,7 +2233,7 @@ async def global_view_spend_tags( raise ProxyException( message="/spend/tags Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2447,9 +2448,9 @@ async def calculate_spend(request: SpendCalculateRequest): if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", str(e)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)), + param=openai_error_param(e), + code=error_status_code(e, status.HTTP_400_BAD_REQUEST), ) if isinstance(e, litellm.exceptions.ModelNotMappedError): raise ProxyException( @@ -2461,9 +2462,9 @@ async def calculate_spend(request: SpendCalculateRequest): error_msg: Final = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + type=openai_error_type(e, error_status_code(e, 500)), + param=openai_error_param(e), + code=error_status_code(e, 500), ) @@ -2633,7 +2634,7 @@ async def ui_view_spend_logs( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2700,7 +2701,7 @@ async def ui_view_spend_logs( raise ProxyException( message="Start date and end date are required", type="bad_request", - param="None", + param=None, code=status.HTTP_400_BAD_REQUEST, ) formats: Final = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"] @@ -3681,7 +3682,7 @@ async def view_spend_logs( raise ProxyException( message=getattr(e, "detail", f"/spend/logs Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3689,7 +3690,7 @@ async def view_spend_logs( raise ProxyException( message="/spend/logs Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3716,7 +3717,7 @@ async def global_spend_reset(): raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3747,7 +3748,7 @@ async def global_spend_refresh(): raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3818,7 +3819,7 @@ async def global_spend_for_internal_user( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) try: @@ -3879,7 +3880,7 @@ async def global_spend_logs( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3923,7 +3924,7 @@ async def global_spend_logs( raise ProxyException( message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3931,7 +3932,7 @@ async def global_spend_logs( raise ProxyException( message="/global/spend/logs Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3971,7 +3972,7 @@ async def global_spend(): raise ProxyException( message=getattr(e, "detail", f"/global/spend Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3979,7 +3980,7 @@ async def global_spend(): raise ProxyException( message="/global/spend Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/tests/code_coverage_tests/check_no_none_string_error_fields.py b/tests/code_coverage_tests/check_no_none_string_error_fields.py new file mode 100644 index 00000000000..3a75b56fc8d --- /dev/null +++ b/tests/code_coverage_tests/check_no_none_string_error_fields.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import ast +import sys +from collections.abc import Iterator +from itertools import chain +from pathlib import Path +from typing import Final + +ERROR_FIELDS: Final = frozenset({"type", "param"}) +SOURCE_ROOT: Final = Path(__file__).resolve().parents[2] / "litellm" + + +def _is_none_string(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value == "None" + + +def _names_an_error_field(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value in ERROR_FIELDS + + +def _keyword_defaults_to_none_string(call: ast.Call) -> bool: + return any(keyword.arg in ERROR_FIELDS and _is_none_string(keyword.value) for keyword in call.keywords) + + +def _getattr_defaults_to_none_string(call: ast.Call) -> bool: + return ( + isinstance(call.func, ast.Name) + and call.func.id == "getattr" + and len(call.args) == 3 + and _names_an_error_field(call.args[1]) + and _is_none_string(call.args[2]) + ) + + +def _dict_holds_none_string(node: ast.Dict) -> bool: + return any(_names_an_error_field(key) and _is_none_string(value) for key, value in zip(node.keys, node.values)) + + +def _is_violation(node: ast.AST) -> bool: + match node: + case ast.Call(): + return _keyword_defaults_to_none_string(node) or _getattr_defaults_to_none_string(node) + case ast.Dict(): + return _dict_holds_none_string(node) + case _: + return False + + +def violations(source: str, filename: str) -> Iterator[str]: + return (f"{filename}:{node.lineno}" for node in ast.walk(ast.parse(source, filename=filename)) if _is_violation(node)) + + +def _file_violations(path: Path) -> Iterator[str]: + return violations(path.read_text(encoding="utf-8"), str(path.relative_to(SOURCE_ROOT.parent))) + + +def main() -> int: + found: Final = tuple(chain.from_iterable(_file_violations(path) for path in sorted(SOURCE_ROOT.rglob("*.py")))) + if not found: + return 0 + print( + 'An error\'s "type" or "param" is set to the string "None", which OpenAI clients read as a real ' + "error class or field name. Use a named error type and a real None for param instead:" + ) + print("\n".join(found)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/test_check_no_none_string_error_fields.py b/tests/code_coverage_tests/test_check_no_none_string_error_fields.py new file mode 100644 index 00000000000..5d2f04b0942 --- /dev/null +++ b/tests/code_coverage_tests/test_check_no_none_string_error_fields.py @@ -0,0 +1,38 @@ +import importlib.util +from pathlib import Path +from typing import Final + +import pytest + +GUARD_PATH: Final = Path(__file__).resolve().parent / "check_no_none_string_error_fields.py" +_spec: Final = importlib.util.spec_from_file_location("check_no_none_string_error_fields", GUARD_PATH) +assert _spec is not None and _spec.loader is not None +guard: Final = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(guard) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param('raise ProxyException(message="m", type="None", param=None, code=500)\n', id="type_keyword"), + pytest.param('raise ProxyException(message="m", type="x", param="None", code=500)\n', id="param_keyword"), + pytest.param('ProxyException(message="m", type=getattr(e, "type", "None"), param=None)\n', id="type_getattr"), + pytest.param('ProxyException(message="m", type="x", param=getattr(e, "param", "None"))\n', id="param_getattr"), + pytest.param('error = {"message": "m", "type": "x", "param": "None"}\n', id="dict_literal"), + ], +) +def test_a_none_string_error_field_is_reported_with_its_line(source: str) -> None: + assert tuple(guard.violations("x = 1\n" + source, "proxy.py")) == ("proxy.py:2",) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param('ProxyException(message="m", type="auth_error", param=None, code=401)\n', id="real_none_param"), + pytest.param('ProxyException(message="None", type="x", param=getattr(e, "param", None))\n', id="none_message"), + pytest.param('label = getattr(e, "provider", "None")\n', id="other_attribute"), + pytest.param('row = {"model": "None"}\n', id="other_dict_key"), + ], +) +def test_a_real_none_or_an_unrelated_none_string_passes(source: str) -> None: + assert tuple(guard.violations(source, "proxy.py")) == () diff --git a/tests/integration/authorization/test_guardrail_submissions.py b/tests/integration/authorization/test_guardrail_submissions.py index f120028f46d..16b9cbf31dc 100644 --- a/tests/integration/authorization/test_guardrail_submissions.py +++ b/tests/integration/authorization/test_guardrail_submissions.py @@ -310,7 +310,7 @@ def test_non_admin_keys_scoped_to_the_submission_routes_get_403_from_the_admin_c f"Your user_id={_mask_user_id(user_id)}" ), type="auth_error", - param="None", + param=None, code="401", ) ) diff --git a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py index 715f3731175..da830293145 100644 --- a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py +++ b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py @@ -114,7 +114,7 @@ class _ChatRequest(BaseModel): class _ProxyError(BaseModel): message: str type: str - param: str + param: str | None code: str @@ -398,7 +398,7 @@ def _assert_unmapped(response: httpx.Response, client_id: str) -> None: assert response.status_code == 403, response.text error: Final = _ErrorResponse.model_validate_json(response.content) expected: Final = f"JWT Key Mapping: No registered mapping for {_CLIENT_CLAIM}='{client_id}'. Access denied." - assert error.error == _ProxyError(message=expected, type="auth_error", param="None", code="403"), response.text + assert error.error == _ProxyError(message=expected, type="auth_error", param=None, code="403"), response.text def _mapping_info(gateway: Gateway, identity: str) -> _JWTMappingResponse: diff --git a/tests/integration/authorization/test_memory_ownership.py b/tests/integration/authorization/test_memory_ownership.py index bad7c3ccb37..249b74363a2 100644 --- a/tests/integration/authorization/test_memory_ownership.py +++ b/tests/integration/authorization/test_memory_ownership.py @@ -24,7 +24,7 @@ def _admin_only_error(route: str, user_id: str) -> dict[str, JsonValue]: f"keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_team_admin_gate.py b/tests/integration/authorization/test_team_admin_gate.py index eff4060631e..adc374a57d7 100644 --- a/tests/integration/authorization/test_team_admin_gate.py +++ b/tests/integration/authorization/test_team_admin_gate.py @@ -600,7 +600,7 @@ def _admin_only_route(route: str) -> tuple[int, JsonValue]: "message": "Authentication Error, Only proxy admin can be used to generate, delete, update info for new " f"keys/users/teams. Route={route}. Your role=internal_user. Your user_id=", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_unknown_key_generic_401.py b/tests/integration/authorization/test_unknown_key_generic_401.py index 8e6ff3fa3ab..025f11d3973 100644 --- a/tests/integration/authorization/test_unknown_key_generic_401.py +++ b/tests/integration/authorization/test_unknown_key_generic_401.py @@ -40,7 +40,7 @@ class _ProxyError(BaseModel): message: str type: str - param: str + param: str | None code: str @@ -260,12 +260,12 @@ def _control_error(response: httpx.Response) -> _ProxyError: def test_requests_without_an_sk_key_keep_their_own_401(gateway: Gateway, peer: Gateway, side: Side) -> None: client: Final = _side(gateway, peer, side).client assert _control_error(client.get("/v1/models")) == _ProxyError( - message="Authentication Error, No api key passed in.", type="auth_error", param="None", code="401" + message="Authentication Error, No api key passed in.", type="auth_error", param=None, code="401" ) assert _control_error(client.get("/v1/models", headers={"Authorization": "Bearer not-a-real-key"})) == _ProxyError( message="LiteLLM Virtual Key expected. Received=not-****-key, expected to start with 'sk-'.", type="auth_error", - param="None", + param=None, code="401", ) @@ -277,7 +277,7 @@ def test_a_blocked_key_keeps_its_own_401_on_both_proxies(gateway: Gateway, peer: expected: Final = _ProxyError( message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", type="auth_error", - param="None", + param=None, code="401", ) for candidate in (gateway, peer): diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py index d058b3c6998..2c495beb77e 100644 --- a/tests/integration/authorization/test_user_self_update_guard.py +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -65,7 +65,7 @@ def _ui_session_route_error(user_id: str) -> dict[str, JsonValue]: f"Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_warmed_policy.py b/tests/integration/authorization/test_warmed_policy.py index b4ffc10c3c4..0de4141143e 100644 --- a/tests/integration/authorization/test_warmed_policy.py +++ b/tests/integration/authorization/test_warmed_policy.py @@ -679,7 +679,7 @@ def _expected_scim_write_denial(route: str, user_id: str, user_role: str) -> dic f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_workflow_ownership.py b/tests/integration/authorization/test_workflow_ownership.py index 386d4f1ed27..70919760874 100644 --- a/tests/integration/authorization/test_workflow_ownership.py +++ b/tests/integration/authorization/test_workflow_ownership.py @@ -57,7 +57,7 @@ def _route_refusal(response: httpx.Response, user_id: str) -> None: f"keys/users/teams. Route=/v1/workflows/runs. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, response.text diff --git a/tests/integration/management/test_health_routing.py b/tests/integration/management/test_health_routing.py index 3a60efe4342..b74d32380d0 100644 --- a/tests/integration/management/test_health_routing.py +++ b/tests/integration/management/test_health_routing.py @@ -151,7 +151,7 @@ def test_public_readiness_reports_stalled_database_lookups(gateway: Gateway, tmp "Please retry shortly." ), "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/integration/management/test_scim_admin_group.py b/tests/integration/management/test_scim_admin_group.py index ffbec93d4a8..92016af1ebb 100644 --- a/tests/integration/management/test_scim_admin_group.py +++ b/tests/integration/management/test_scim_admin_group.py @@ -133,7 +133,7 @@ def _expected_scim_write_denial(route: str, user_id: str, user_role: str) -> dic f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 324963a0da9..513a80b7319 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -64,7 +64,7 @@ def _blocked_key_error() -> dict[str, JsonValue]: "error": { "message": "Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py index a05b3f0f045..94e9d46ec8f 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py @@ -177,7 +177,7 @@ def test_arize_otel_v2_d4_unauthenticated_request_has_no_span(gateway: Gateway, "error": { "message": "Authentication Error, No api key passed in.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, body diff --git a/tests/integration/observability/test_presidio_entity_masking.py b/tests/integration/observability/test_presidio_entity_masking.py index ab966873389..ff36e4623f7 100644 --- a/tests/integration/observability/test_presidio_entity_masking.py +++ b/tests/integration/observability/test_presidio_entity_masking.py @@ -325,7 +325,7 @@ def test_long_apply_guardrail_alias_accepts_an_llm_api_scoped_key(gateway: Gatew "error": { "message": "Authentication Error, No api key passed in.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, unauthenticated.text diff --git a/tests/integration/security/test_config_overrides.py b/tests/integration/security/test_config_overrides.py index 72546c54222..70f9226ea79 100644 --- a/tests/integration/security/test_config_overrides.py +++ b/tests/integration/security/test_config_overrides.py @@ -100,7 +100,7 @@ def _assert_non_admin_refusal(response: httpx.Response, route: str, user_id: str f"new keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, response.text @@ -244,7 +244,7 @@ def test_saved_vault_config_reads_back_masked_and_refuses_non_admins( "message": "user not allowed to access this route, role= proxy_admin_viewer. " "Trying to access: /config_overrides/hashicorp_vault", "type": "auth_error", - "param": "None", + "param": None, "code": "403", } }, viewer_post.text @@ -386,7 +386,7 @@ def test_saved_cyberark_config_reads_back_masked_and_refuses_non_admins( "message": "user not allowed to access this route, role= proxy_admin_viewer. " "Trying to access: /config_overrides/cyberark", "type": "auth_error", - "param": "None", + "param": None, "code": "403", } }, viewer_post.text diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py index c400d90e184..cac66471dea 100644 --- a/tests/integration/spend/test_global_spend_reset.py +++ b/tests/integration/spend/test_global_spend_reset.py @@ -104,7 +104,7 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: "error": { "message": "Authentication Error, Tried to access route=/global/spend/reset, which is only for MASTER KEY", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, refused.text diff --git a/tests/unit/proxy/auth/test_auth_exception_handler.py b/tests/unit/proxy/auth/test_auth_exception_handler.py index 92b4c93118f..2523284ec2c 100644 --- a/tests/unit/proxy/auth/test_auth_exception_handler.py +++ b/tests/unit/proxy/auth/test_auth_exception_handler.py @@ -1298,3 +1298,17 @@ def test_as_proxy_exception_keeps_jwt_scope_denial_message_shape(): assert converted.to_dict() == plain.to_dict() assert converted.internal_message == denial.internal_message + + +@pytest.mark.parametrize( + "auth_failure", + [ + pytest.param(HTTPException(status_code=401, detail="Authentication Error, invalid key"), id="http_exception"), + pytest.param(Exception("Invalid proxy server token passed"), id="bare_exception"), + ], +) +def test_auth_failure_error_body_carries_a_json_null_param(auth_failure: Exception) -> None: + body = json.loads(json.dumps({"error": _as_proxy_exception(auth_failure).to_dict()})) + + assert body["error"]["type"] == ProxyErrorTypes.auth_error.value + assert body["error"]["param"] is None diff --git a/tests/unit/proxy/auth/test_handle_jwt.py b/tests/unit/proxy/auth/test_handle_jwt.py index a0ea6ce0403..51cf15996cd 100644 --- a/tests/unit/proxy/auth/test_handle_jwt.py +++ b/tests/unit/proxy/auth/test_handle_jwt.py @@ -4368,6 +4368,7 @@ async def test_removed_signing_key_stops_being_trusted_once_the_stale_window_exp assert exc_info.value.code == "503" assert exc_info.value.type == ProxyErrorTypes.auth_provider_unavailable + assert exc_info.value.param is None @pytest.mark.asyncio diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 117428ee27b..81ea0e3f60b 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -1310,6 +1310,33 @@ async def test_auth_rejects_wrong_or_empty_present_custom_header( assert exc_info.value.code == "401" +@pytest.mark.asyncio +@pytest.mark.parametrize("api_key", [None, ""], ids=["no_key", "no_bearer_prefix"]) +async def test_auth_rejection_without_a_usable_key_has_a_json_null_param( + api_key: str | None, monkeypatch: pytest.MonkeyPatch +): + from fastapi import Request + + from litellm.proxy import proxy_server + from litellm.proxy._types import ProxyErrorTypes, ProxyException + + monkeypatch.setattr(proxy_server, "master_key", "sk-test-master") + monkeypatch.setattr(proxy_server, "general_settings", {}) + request = Request( + scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []}, + receive=AsyncMock(return_value={"type": "http.request", "body": b"", "more_body": False}), + ) + request._url = URL(url="/chat/completions") + + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth(request=request, api_key=api_key) + + body = json.loads(json.dumps({"error": exc_info.value.to_dict()})) + assert body["error"]["code"] == "401" + assert body["error"]["type"] == ProxyErrorTypes.auth_error.value + assert body["error"]["param"] is None + + @pytest.mark.asyncio async def test_user_api_key_from_query_param(): """Ensure user_api_key_auth reads API key from `key` query parameter.""" diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index d1bbf444d2c..78a0a765fe6 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4970,7 +4970,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 859824e4a72..b0ef7e64e60 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -23379,6 +23379,23 @@ def test_key_info_database_not_connected_sanitized_500( proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) +def test_key_health_failure_body_carries_a_json_null_param(): + import litellm.proxy.proxy_server as ps + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="hashed-key", metadata={"logging": ["not-a-callback-object"]} + ) + try: + response = client.post("/key/health", headers={"Authorization": "Bearer sk-test"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 500 + error = response.json()["error"] + assert error["type"] == "internal_server_error" + assert error["param"] is None + + @pytest.mark.asyncio async def test_update_key_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None: from fastapi import Request diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index c6d12ade2a0..e33743bb149 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -19095,7 +19095,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/proxy_server/test_exception_handlers.py b/tests/unit/proxy/proxy_server/test_exception_handlers.py index b34c2d2fb08..0f9d379afa1 100644 --- a/tests/unit/proxy/proxy_server/test_exception_handlers.py +++ b/tests/unit/proxy/proxy_server/test_exception_handlers.py @@ -438,7 +438,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index de76c7257cf..59d9490f786 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -14,9 +14,15 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi import HTTPException +from litellm import Router from litellm.proxy import proxy_server +from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import RouterErrors + +_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']" @pytest.fixture @@ -208,7 +214,8 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path): with auth_as(): response = client.post(path, json=payload) assert response.status_code == 500 - assert len(response.content) > 0 + error = response.json()["error"] + assert (error["type"], error["param"]) == ("internal_server_error", None) def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection): @@ -245,6 +252,104 @@ def test_audio_transcription_happy_path(client, auth_as, patched_transcription, } +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_over_the_file_size_cap_is_rejected_as_a_bad_request( + client, auth_as, patched_transcription, monkeypatch, path +): + from litellm.proxy.common_utils.http_parsing_utils import check_file_size_under_limit + + capped_router = Router( + model_list=[ + {"model_name": "whisper-1", "litellm_params": {"model": "openai/whisper-1", "max_file_size_mb": 0.000001}} + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", capped_router) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", check_file_size_under_limit) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post(path, files=files, data={"model": "whisper-1"}) + assert response.status_code == 400 + error = response.json()["error"] + assert "File size is too large" in error["message"] + assert error["param"] == "file" + + +def test_audio_speech_tag_routing_denial_is_typed_as_a_proxy_auth_error( + client, auth_as, patched_speech_error, monkeypatch +): + async def _deny(*args, **kwargs): + raise ValueError(_TAG_ROUTING_DENIAL) + + monkeypatch.setattr(proxy_server, "route_request", _deny) + with auth_as(): + response = client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"}) + assert response.status_code == 401 + error = response.json()["error"] + assert (error["type"], error["param"]) == ("auth_error", None) + + +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + pytest.param(ValueError(_TAG_ROUTING_DENIAL), 401, "auth_error", id="tag_routing_denial"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + 403, + "permission_error", + id="hook_policy_block", + ), + ], +) +def test_audio_transcription_failure_is_typed_by_who_rejected_it( + client, auth_as, patched_transcription, monkeypatch, failure, expected_status, expected_type +): + async def _deny(*args, **kwargs): + raise failure + + monkeypatch.setattr(proxy_server, "route_request", _deny) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"}) + assert response.status_code == expected_status + error = response.json()["error"] + assert (error["type"], error["param"]) == (expected_type, None) + + +@pytest.mark.parametrize( + "rejection", + [ + pytest.param( + ProxyException( + message="Invalid type for 'metadata': expected an object, but got an integer instead.", + type=ProxyErrorTypes.bad_request_error, + param="metadata", + code=400, + ), + id="bad_metadata", + ), + pytest.param( + ProxyException( + message="tag budgets are unavailable", type=ProxyErrorTypes.no_db_connection, param=None, code=503 + ), + id="tag_budget_store_down", + ), + ], +) +def test_audio_transcription_keeps_the_status_of_a_proxy_rejection( + client, auth_as, patched_transcription, monkeypatch, rejection +): + async def _reject(data, **kwargs): + raise rejection + + monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _reject) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"}) + assert response.status_code == int(rejection.code) + assert response.json()["error"] == rejection.to_dict() + + @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) def test_audio_transcription_error(client, auth_as, patched_transcription_error, path): """Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (error).""" diff --git a/tests/unit/proxy/proxy_server/test_routes_completions.py b/tests/unit/proxy/proxy_server/test_routes_completions.py index b5c60c23c02..2b585357549 100644 --- a/tests/unit/proxy/proxy_server/test_routes_completions.py +++ b/tests/unit/proxy/proxy_server/test_routes_completions.py @@ -12,8 +12,10 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock import pytest +from fastapi import HTTPException from litellm.proxy import common_request_processing, proxy_server +from litellm.types.router import RouterErrors from .conftest import normalize # type: ignore[import-not-found] @@ -124,3 +126,39 @@ def test_completion_pipeline_error(client, auth_as, completion_pipeline_raises, response = client.post(path, json=payload) assert response.status_code == 500 assert response.headers.get("content-type", "").startswith("application/json") + + +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + pytest.param(ValueError("boom"), 500, "internal_server_error", id="unexpected_failure"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + 403, + "permission_error", + id="hook_policy_block", + ), + pytest.param(HTTPException(status_code=401, detail="key expired"), 401, "authentication_error", id="hook_401"), + pytest.param( + ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-4o and tags=['x']"), + 401, + "auth_error", + id="tag_routing_denial", + ), + ], +) +def test_completion_failure_body_types_the_status_it_answers_with( + client, auth_as, monkeypatch, failure, expected_status, expected_type +): + monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock())) + + async def _raise(self, *args, **kwargs): + raise failure + + monkeypatch.setattr(common_request_processing.ProxyBaseLLMRequestProcessing, "base_process_llm_request", _raise) + with auth_as(): + response = client.post("/v1/completions", json={"model": "gpt-4o", "prompt": "hi"}) + assert response.status_code == expected_status + error = response.json()["error"] + assert (error["type"], error["param"], error["code"]) == (expected_type, None, str(expected_status)) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 22ada990fe6..c37475bd539 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -32,7 +32,7 @@ import litellm import litellm.proxy.proxy_server as ps from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, @@ -946,6 +946,39 @@ def _error_of(frame: str) -> dict[str, object]: return json.loads(frame.removeprefix("data: "))["error"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "raised, expected_error", + [ + pytest.param( + ProxyException(message="Budget has been exceeded", type="budget_exceeded", param=None, code=429), + {"message": "Budget has been exceeded", "type": "budget_exceeded", "param": None, "code": "429"}, + id="proxy_exception_keeps_its_status", + ), + pytest.param( + RuntimeError("upstream blew up"), + {"message": "upstream blew up", "type": "internal_server_error", "param": None, "code": "500"}, + id="unexpected_failure", + ), + ], +) +async def test_a_mid_stream_failure_frame_carries_the_status_the_error_was_raised_with( + monkeypatch, raised: Exception, expected_error: dict[str, object] +): + _patch_logging_flags(monkeypatch) + monkeypatch.setattr(ps.proxy_logging_obj, "post_call_failure_hook", AsyncMock(return_value=None)) + + frames: Final = [ + frame + async for frame in async_data_generator( + response=_async_iter_raises(raised), user_api_key_dict=_user_auth(), request_data={} + ) + ] + + error_frames: Final = [frame for frame in frames if isinstance(frame, str) and frame.startswith('data: {"error":')] + assert [_error_of(frame.removesuffix("\n\n")) for frame in error_frames] == [expected_error] + + @pytest.mark.asyncio @pytest.mark.parametrize( "detail,expected_message,expected_fields", diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 8d53817f51e..6598b94e60f 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -8268,3 +8268,19 @@ async def test_global_spend_refresh_runs_the_refresh_on_the_writer_when_a_reader assert [statement for statement in journal.statements if statement[1].startswith("REFRESH")] == [ ("writer", 'REFRESH MATERIALIZED VIEW "MonthlyGlobalSpend";') ] + + +def test_spend_calculate_rejection_body_is_openai_shaped(client): + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user", api_key="hashed-k" + ) + try: + response = client.post("/spend/calculate", json={}, headers={"Authorization": "Bearer sk-test"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] is None + assert error["code"] == "400" diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 023ef0b4606..62a90109d26 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -7255,6 +7255,39 @@ async def test_async_data_generator_closes_the_guardrail_chain_before_returning_ assert cleanup_ran == [True] +@pytest.mark.asyncio +async def test_async_data_generator_midstream_failure_frame_is_openai_shaped_with_a_json_null_param(): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import async_data_generator + + async def chain_failing_after_first_chunk(**_kwargs): + yield {"choices": [{"delta": {"content": "Hello"}}]} + raise ValueError("audit hook could not reach its sink") + + proxy_logging = _guarded_chain_logging(chain_failing_after_first_chunk) + proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging): + frames = [ + frame + async for frame in async_data_generator( + MagicMock(), MagicMock(spec=UserAPIKeyAuth), {"model": "gpt-4o-mini"} + ) + ] + + error_bodies = [json.loads(frame.removeprefix("data: ")) for frame in frames if '"error"' in frame] + assert error_bodies == [ + { + "error": { + "message": "audit hook could not reach its sink", + "type": "internal_server_error", + "param": None, + "code": "500", + } + } + ] + + @pytest.mark.asyncio async def test_async_data_generator_closes_the_guardrail_chain_while_a_keepalive_read_is_pending(): from litellm.proxy._types import UserAPIKeyAuth @@ -14222,6 +14255,63 @@ async def test_moderations_reraises_proxy_exception_unwrapped(): mock_logging.post_call_failure_hook.assert_awaited_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "expected_type", "expected_code"), + [ + pytest.param(HTTPException(status_code=400, detail="bad input"), "invalid_request_error", "400", id="http_400"), + pytest.param(HTTPException(status_code=429, detail="slow down"), "rate_limit_error", "429", id="http_429"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + "permission_error", + "403", + id="hook_policy_block", + ), + pytest.param(Exception("provider down"), "internal_server_error", "500", id="bare_exception"), + pytest.param( + ValueError("Not allowed to access model due to tags configuration. Passed model=m and tags=['x']"), + "auth_error", + "401", + id="tag_routing_denial", + ), + ], +) +async def test_moderations_failure_body_is_openai_shaped(failure, expected_type, expected_code): + from litellm.proxy._types import ProxyException + + async def passthrough_add_litellm_data(data, **kwargs): + return data + + request = MagicMock() + request.body = AsyncMock(return_value=b'{"input": "hi"}') + fake_logging = MagicMock() + fake_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + fake_logging.post_call_failure_hook = AsyncMock() + + with ( + patch.object( + proxy_server_module, "add_litellm_data_to_request", new=passthrough_add_litellm_data + ), # test-quality-ok: the route reads this module global, no injection point + patch.object( + proxy_server_module, "route_request", new=AsyncMock(side_effect=failure) + ), # test-quality-ok: fakes the provider failure so the real route's error shaping is observable + patch.object( + proxy_server_module, "proxy_logging_obj", new=fake_logging + ), # test-quality-ok: module global, no injection point + pytest.raises(ProxyException) as raised, + ): + await proxy_server_module.moderations( + request=request, + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + body = json.loads(json.dumps({"error": raised.value.to_dict()})) + assert body["error"]["type"] == expected_type + assert body["error"]["param"] is None + assert body["error"]["code"] == expected_code + + @pytest.mark.asyncio async def test_moderations_response_carries_litellm_call_id_header(): from fastapi import Response From 23ad8870b46ab984a922ae72ea7b9410542e11e0 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:24:04 -0700 Subject: [PATCH 03/34] fix(token_counter): price PDF pages for OpenAI and Azure models the way they bill (#47554) * test(token_counter): price PDF pages for OpenAI and Azure models the way they bill * fix(token_counter): price PDF pages for OpenAI and Azure models the way they bill * fix(token_counter): count azure, data-zone and dated GPT names with their family's encoding * fix(token_counter): price only drawn PDF images and keep other gateways' GPT pricing Detect raster images from the page content stream, including inline images on pages with empty resources, and walk each form once. Strip only openai/, azure/ and Azure data-zone prefixes, bound the model family scan, read Azure-only chat models from the cost map, and keep cl100k_base for listed non-chat OpenAI models. * fix(token_counter): price GPT names behind any prefix like the bare name and walk shared forms per resources Prices a PDF by the model name's last path segment, the same name the proxy counts with, keeps listed non-chat models on Claude pricing, walks each form once per inherited resources dictionary so a shared child form drawing an image is not missed, and resolves PDF pricing only when a message carries a PDF * fix(token_counter): keep the rendered-page count for image pages, parse each form once, and match chat models only A PDF page that draws a raster image keeps the rendered-page estimate, since newer GPT models bill such a page near it. The image walk parses each form once and stops at a fixed number of drawn XObjects, so a form shared under many resources dictionaries no longer slows the count. The family prefix match skips non-chat entries, and ft: fine-tunes count like their base model * fix(token_counter): count Azure-only chat models with o200k like other OpenAI chat models * fix(token_counter): count reservation and project ITPM with the serving deployment's model Budget reservation and project ITPM counted with the requested group name, so a GPT-named group served by Claude reserved a PDF at OpenAI's price. They now count with each deployment's model minus its provider prefix, the name count_tokens uses, and take the largest. A custom OpenAI tokenizer with no model counts with cl100k again instead of raising, and listed Azure gpt- chat entries price as OpenAI. * perf(token_counter): find drawn PDF images with a byte scan instead of parsing every operator pypdf parsed every operator of a page's content stream to find the images it draws, so a compressed page of a million operators took about 30 seconds to price for an OpenAI model. A linear scan of the decoded stream for name Do and inline images finds the same draws. * fix(token-counter): count each team deployment by its PDF pricing and honor base_model Budget reservation and the TPM limiter now look up the deployments that serve the caller's team, by base_model when set. A deployment that reads PDF pages and prices them unlike the group name counts with its own model, any other with the group name. A comment between an XObject name and Do now reads as whitespace, and the timing tests moved to CodSpeed benchmarks. * fix(benchmarks): install pypdf in the benchmark environment The PDF token-count benchmarks parse pages with pypdf, which only the proxy-runtime extra installed, so CodSpeed took the no-parser path. * fix(proxy): count PDF pages for deployments served under multi-segment model names * fix(proxy): count a team's wildcard route the way routing serves it and reserve OpenAI chat's per-PDF charge * fix(proxy): count routing-group and alias members with the caller's team, as routing serves them --- litellm/constants.py | 6 + litellm/litellm_core_utils/token_counter.py | 309 ++++++++++++- .../hooks/parallel_request_limiter_v3.py | 27 +- .../spend_tracking/budget_reservation.py | 36 +- litellm/proxy/spend_tracking/input_tokens.py | 111 ++++- pyproject.toml | 1 + .../test_token_counter_pdf_benchmarks.py | 126 +++++ .../litellm_core_utils/test_token_counter.py | 433 +++++++++++++++++- .../hooks/test_parallel_request_limiter_v3.py | 63 ++- .../spend_tracking/test_budget_reservation.py | 48 +- .../proxy/spend_tracking/test_input_tokens.py | 170 +++++++ tests/unit/rust_bridge/test_token_counter.py | 3 +- uv.lock | 2 + 13 files changed, 1272 insertions(+), 63 deletions(-) create mode 100644 tests/benchmarks/test_token_counter_pdf_benchmarks.py diff --git a/litellm/constants.py b/litellm/constants.py index d02114f56b1..466942d35be 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -753,7 +753,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches" ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568 ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000 ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 +OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 +OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," +PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 +PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000 +OPENAI_MODEL_NAME_MAX_DASHES: Final = 16 +OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"}) VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = { "low": 1, diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 5ad4e130e62..ea3e645ffdf 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -7,9 +7,11 @@ import re import struct from bisect import bisect_right from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from itertools import accumulate +from dataclasses import dataclass +from functools import cached_property +from itertools import accumulate, chain, islice from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast import anyio import anyio.lowlevel @@ -33,7 +35,12 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_CHAT_MODES, + OPENAI_MODEL_NAME_MAX_DASHES, + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, + PDF_MAX_FORM_XOBJECT_DEPTH, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, TOKEN_COUNTER_MAX_EXACT_CHARS, @@ -606,6 +613,35 @@ TokenCounterFunction = Callable[[str], int] Type for a function that counts tokens in a string. """ +PdfPagePricing: TypeAlias = Literal["openai", "claude"] + + +class _PdfBox(Protocol): + @property + def width(self) -> float: ... + + @property + def height(self) -> float: ... + + +class _PdfContent(Protocol): + def get_data(self) -> bytes: ... + + +class _PdfPage(Protocol): + @property + def mediabox(self) -> _PdfBox: ... + + def extract_text(self) -> str: ... + + def get_contents(self) -> _PdfContent | None: ... + + +class _PdfReader(Protocol): + @property + def pages(self) -> Sequence[_PdfPage]: ... + + EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -662,7 +698,7 @@ class _MessageCountParams: if uses_legacy_message_accounting(model): self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n self.tokens_per_name = -1 # if there's a name, the role is omitted - elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms: + elif _is_openai_chat_model(actual_model): self.tokens_per_message = 3 self.tokens_per_name = 1 else: @@ -671,6 +707,63 @@ class _MessageCountParams: self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) self.tool_section = chat_template_tool_section(model) + self._model = model + + @cached_property + def pdf_pricing(self) -> PdfPagePricing: + return pdf_page_pricing(self._model) + + +def _is_openai_chat_model(model: str) -> bool: + return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms + + +def _openai_deployment_name(model: str) -> str: + name: Final = model.rpartition("/")[2] + return name.split(":")[1] if name.startswith("ft:") else name + + +def _openai_model_family(model: str) -> str | None: + words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES) + prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) + listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) + return None if listed is None or _is_listed_non_chat_model(listed) else listed + + +def _untyped_cost_map() -> object: + return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped + + +def _cost_map_entry(model: str) -> Mapping[str, object]: + cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {} + return _pdf_dictionary(cost_map.get(model)) or {} + + +def _is_azure_openai_chat_model(model: str) -> bool: + name: Final = _openai_deployment_name(model) + entry: Final = _cost_map_entry(f"azure/{name}") + return ( + entry.get("litellm_provider") == "azure" + and entry.get("mode") in OPENAI_CHAT_MODES + and (entry.get("supports_pdf_input") is True or name.startswith("gpt-")) + ) + + +def _is_openai_or_azure_model(model: str) -> bool: + if _is_listed_non_chat_model(_openai_deployment_name(model)): + return False + return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) + + +def pdf_page_pricing(model: str | None) -> PdfPagePricing: + return "openai" if _is_openai_or_azure_model(model or "") else "claude" + + +def reads_pdf_pages(model: str) -> bool: + segments: Final = model.split("/") + names: Final = ("/".join(segments[start:]) for start in range(len(segments))) + stated: Final = (_cost_map_entry(name).get("supports_pdf_input") for name in names) + return next((support for support in stated if support is not None), False) is True def token_counter( @@ -842,6 +935,7 @@ def _count_messages( value, use_default_image_token_count, default_token_count, + params.pdf_pricing, ) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -986,7 +1080,7 @@ def _get_exact_count_function( return count_tokens if tokenizer_json["type"] == "openai_tokenizer": - encoding: Final = openai_tokenizer_encoding(model) + encoding: Final = openai_tokenizer_encoding(model or "") def encode_length(text: str) -> int: return _encoding_count(encoding, text) @@ -1006,6 +1100,11 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": return get_encoding(openai_tokenizer_encoding_name(model)) +def _is_listed_non_chat_model(model: str) -> bool: + modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}")) + return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes) + + def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding. @@ -1021,7 +1120,9 @@ def openai_tokenizer_encoding_name(model: str) -> str: try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" + return _suffixed_variant_encoding_name(model_to_use) or ( + "cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base" + ) def unresolved_tokenizer_model(model: str) -> str: @@ -1079,7 +1180,16 @@ def single_encoding_openai_model(served: Sequence[str | None]) -> str | None: def _openai_model_name(model: str) -> str | None: - return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model) + return ( + _listed_openai_model_name(model) + or _tiktoken_o200k_model_name(model) + or _versioned_openai_model_name(model) + or _azure_openai_chat_model_name(model) + ) + + +def _azure_openai_chat_model_name(model: str) -> str | None: + return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else None _VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)") @@ -1249,7 +1359,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int: return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN) -def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: +def _pdf_dictionary(node: object) -> Mapping[str, object] | None: + return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key + + +def _pdf_value(container: Mapping[str, object], key: str) -> object: + return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in [] + + +_PdfXObjects: TypeAlias = Mapping[str, object] | None +_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfContentSummary: + drawn_names: tuple[str, ...] + draws_inline_image: bool + + +_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfWalk: + frontier: tuple[_PdfDrawing, ...] + seen: frozenset[tuple[int, int]] + summaries: Mapping[int, _PdfContentSummary | None] + draws: int + + +def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects: + resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources")) + if resources is None: + return inherited + return _pdf_dictionary(_pdf_value(resources, "/XObject")) + + +_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r " +_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%" +_PDF_XOBJECT_DRAW: Final = re.compile( + rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])" +) +_PDF_INLINE_IMAGE: Final = re.compile(rb"(? str | None: + try: + return raw.decode(charset) + except UnicodeDecodeError: + return None + + +def _pdf_name(escaped: bytes) -> str: + raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped) + decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS) + return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1")) + + +def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary: + data: Final = content.get_data() + draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1) + return _PdfContentSummary( + drawn_names=tuple(_pdf_name(draw[1]) for draw in draws), + draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None, + ) + + +def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None: + from pypdf.generic import StreamObject + + if not isinstance(form, StreamObject): + return None + return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped + + +def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]: + summary, xobjects = drawing + if xobjects is None: + return () + resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names) + return tuple((xobject, xobjects) for xobject in resolved if xobject is not None) + + +def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]: + xobject, inherited = use + return id(xobject), id(_pdf_xobjects(xobject, inherited)) + + +def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None: + draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier) + if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE: + return None + drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier))) + if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): + return None + forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"} + unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen) + unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries} + summaries: Final = MappingProxyType( + {**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}} + ) + drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen) + return _PdfWalk( + frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None), + seen=walk.seen | frozenset(forms), + summaries=summaries, + draws=draws, + ) + + +def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: + contents: Final = page.get_contents() + page_dictionary: Final = _pdf_dictionary(page) + if contents is None or page_dictionary is None: + return False + walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned + frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),), + seen=frozenset(), + summaries=MappingProxyType({}), + draws=0, + ) + for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): + if not walk.frontier: + return False + if any(summary.draws_inline_image for summary, _ in walk.frontier): + return True + if (next_walk := _pdf_next_walk(walk)) is None: + return True + walk = next_walk + return bool(walk.frontier) + + +def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + if _pdf_page_draws_raster_image(page): + return _claude_pdf_page_tokens(page, count_function) + text: Final = page.extract_text() or "" + return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 + + +def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens( + float(page.mediabox.width), float(page.mediabox.height) + ) + + +def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int: + match pdf_pricing: + case "openai": + return _openai_pdf_page_tokens(page, count_function) + case "claude": + return _claude_pdf_page_tokens(page, count_function) + + +def _count_inline_pdf_tokens( + data_url: str, + count_function: TokenCounterFunction, + pdf_pricing: PdfPagePricing = "claude", +) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None try: @@ -1258,12 +1526,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image") return None try: - reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) - return sum( - count_function(page.extract_text() or "") - + _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) - for page in reader.pages + reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed + _PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) ) + return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None @@ -1273,8 +1539,9 @@ def _count_opaque_document_tokens( data_url: str, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: - pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function) + pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing) if pdf_tokens is not None: return pdf_tokens return calculate_img_tokens( @@ -1289,6 +1556,7 @@ def _count_document_tokens( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: source: Final = document["source"] metadata_tokens: Final = sum( @@ -1301,10 +1569,10 @@ def _count_document_tokens( if isinstance(content, str): return metadata_tokens + count_function(content) return metadata_tokens + _count_content_list( - count_function, content, use_default_image_token_count, default_token_count + count_function, content, use_default_image_token_count, default_token_count, pdf_pricing ) return metadata_tokens + _count_opaque_document_tokens( - _anthropic_image_source_data(source), count_function, use_default_image_token_count + _anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing ) @@ -1312,6 +1580,7 @@ def _count_file_tokens( file_value: object, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" if not isinstance(file_value, Mapping): @@ -1321,7 +1590,9 @@ def _count_file_tokens( name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 if not isinstance(file_data, str) or not file_data: return name_tokens - return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count) + return name_tokens + _count_opaque_document_tokens( + file_data, count_function, use_default_image_token_count, pdf_pricing + ) def _count_anthropic_content( @@ -1329,6 +1600,7 @@ def _count_anthropic_content( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """ Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). @@ -1363,6 +1635,7 @@ def _count_anthropic_content( field_value, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif isinstance(field_value, dict): tokens += count_function(str(field_value)) @@ -1389,6 +1662,7 @@ def _count_content_list( ], use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """Recursively count tokens from a list of content blocks.""" try: @@ -1413,12 +1687,14 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] == "file": num_tokens += _count_file_tokens( c.get("file"), count_function, use_default_image_token_count, + pdf_pricing, ) elif c["type"] in ("tool_use", "tool_result", "search_result"): num_tokens += _count_anthropic_content( @@ -1426,6 +1702,7 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] in ("thinking", "redacted_thinking"): # Claude extended thinking content block diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index ea0871575e6..404aff84994 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -76,7 +76,11 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit -from litellm.proxy.spend_tracking.input_tokens import tokenizer_models +from litellm.proxy.spend_tracking.input_tokens import ( + inline_pdf_file_count, + models_counted_for, + openai_chat_pdf_file_tokens, +) from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -160,10 +164,10 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: return resolve_model_group_alias(llm_router.registry().aliases(), model) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # registry().aliases() is the untyped model_group_alias dict -def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]: +def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]: from litellm.proxy.proxy_server import llm_router - return tokenizer_models(model, llm_router) + return models_counted_for(model, llm_router, team_id) def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: @@ -514,6 +518,7 @@ _TPM_FLOOR_FRACTION: Final = 4 # check on data["input"] must exclude these call types, or a Responses call # gets misclassified as an embedding and skips output-token reservation/caps. RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses") +CHAT_COMPLETION_API_CALL_TYPES: Final = (CallTypes.completion.value, CallTypes.acompletion.value) EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding") TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion") RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value) @@ -885,23 +890,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): time_provider: Callable[[], datetime] | None = None, tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db, model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, + counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router, media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ptu_team_ceiling_resolver: Callable[ [str, str], PTUTeamCeiling | None ] = _resolve_ptu_team_ceiling_via_proxy_router, fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings, force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings, - tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now self._tag_rate_limit_resolver = tag_rate_limit_resolver self._model_group_resolver = model_group_resolver + self._counted_models_resolver = counted_models_resolver self._media_model_resolver = media_model_resolver self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver - self._tokenizer_models_resolver = tokenizer_models_resolver if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( BATCH_RATE_LIMITER_SCRIPT @@ -3948,7 +3953,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): state: Final = data.get("state") return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}" - def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: + def _estimate_precise_input_tokens( + self, data: object, model: str | None, call_type: str | None = None, team_id: str | None = None + ) -> int: """ Model-aware input token estimate for the project ITPM reservation, using ``litellm.token_counter`` -- the same approach the @@ -4014,6 +4021,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): audio_token_estimate: Final = self._estimate_audio_content_tokens(messages) countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages + chat_pdf_files: Final = ( + inline_pdf_file_count(countable_messages) if call_type in CHAT_COMPLETION_API_CALL_TYPES else 0 + ) try: estimate: Final = max( @@ -4029,7 +4039,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): use_default_image_token_count=True, ) ) - for tokenizer_model in self._tokenizer_models_resolver(model or "") + + openai_chat_pdf_file_tokens(tokenizer_model, chat_pdf_files) + for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",)) ), ) return estimate + audio_token_estimate @@ -4086,7 +4097,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): include_media=False, ) raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=requested_model, call_type=call_type + data=data, model=requested_model, call_type=call_type, team_id=user_api_key_dict.team_id ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 57dcb74b36b..fc6fb266d04 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -17,6 +17,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.proxy._types import ( @@ -40,12 +41,19 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model, tokenizer_models +from litellm.proxy.spend_tracking.input_tokens import ( + count_input_tokens, + count_input_tokens_for_model, + inline_pdf_file_count, + models_counted_for, + openai_chat_pdf_file_tokens, +) from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict +from litellm.types.utils import CallTypes @dataclass @@ -271,6 +279,7 @@ async def reserve_budget_for_request( request_body=request_body, route=route, llm_router=llm_router, + team_id=valid_token.team_id, raw_body=raw_body, ) @@ -1592,6 +1601,7 @@ async def count_request_input_tokens( request_body: dict, route: str, llm_router: Router | None, + team_id: str | None, raw_body: bytes | None = None, ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. @@ -1600,21 +1610,35 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - counted_as: Final = MappingProxyType({model: tokenizer_models(model, llm_router) for model in models}) + counted_models: Final = MappingProxyType( + {model: models_counted_for(model, llm_router, team_id) for model in models} + ) counts: Final = await count_input_tokens( request_body=request_body, raw_body=raw_body, - models=tuple(dict.fromkeys(chain.from_iterable(counted_as.values()))), + models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))), + ) + chat_pdf_files: Final = ( + inline_pdf_file_count(request_body.get("messages")) + if CallTypes.acompletion in (get_call_types_for_route(route) or ()) + else 0 ) return MappingProxyType( { - model: count - for model, names in counted_as.items() - if (count := max((counts[name] for name in names if name in counts), default=None)) is not None + model: largest + for model, names in counted_models.items() + if (largest := _largest_count(counts, names, chat_pdf_files)) is not None } ) +def _largest_count(counts: Mapping[str, int], names: Sequence[str], chat_pdf_files: int) -> int | None: + return max( + (counts[name] + openai_chat_pdf_file_tokens(name, chat_pdf_files) for name in names if name in counts), + default=None, + ) + + def _estimate_input_tokens( request_body: dict, route: str, diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 2ae85481f11..59c7fe2a41b 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -13,18 +13,25 @@ from __future__ import annotations import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.token_counter import served_openai_models, unresolved_tokenizer_model +from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX +from litellm.litellm_core_utils.token_counter import ( + deployment_openai_model, + pdf_page_pricing, + reads_pdf_pages, + unresolved_tokenizer_model, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, ) -from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.token_counter import ( @@ -40,6 +47,8 @@ if TYPE_CHECKING: TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 +_OPENAI_PREFIX: Final = "openai/" + _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -52,25 +61,6 @@ def _approximate_input_size(request_body: Mapping[str, object]) -> int: return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) -def tokenizer_models(model: str, llm_router: Router | None) -> tuple[str, ...]: - """The names to count a request for `model` with, keeping the largest count. - - An alias resolves to its target group. A group served only by OpenAI models counts as those models, so it gets - their tokenizer. Any other group counts as its OpenAI models, if any, and as the requested name with the encoding - tiktoken gives that name as written, so its other deployments keep that count without loading their vendor's - tokenizer. A name the router does not index counts as written.""" - if llm_router is None: - return (model,) - group: Final = resolve_model_group_alias(llm_router.registry().aliases(), model) or model # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # registry().aliases() is the untyped model_group_alias dict - served: Final = served_openai_models(llm_router.registry().deployments(group, source="named") or ()) - if not served: - return (model,) - openai_served: Final = tuple(served_model for served_model in served if served_model is not None) - if len(openai_served) == len(served): - return openai_served - return (unresolved_tokenizer_model(model), *openai_served) - - async def count_input_tokens( request_body: dict, raw_body: bytes | None, @@ -154,6 +144,83 @@ def _count_input_tokens_for_models( ) +def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: + """The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment + routing would serve it from: the team's or else the shared ones named `model`, its alias target, or its routing + group's members, else those of the one wildcard route that serves it. + + An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed + `openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A + deployment that reads PDF pages and prices them unlike `model` counts as its model minus the provider prefix, the + name `/v1/messages/count_tokens` counts with. Any other deployment counts as the requested name with the encoding + tiktoken gives that name as written, without loading its vendor's tokenizer. A name no deployment serves counts as + written.""" + deployments: Final = _routed_deployments(model, llm_router, team_id) if llm_router else () + return tuple(dict.fromkeys(_counted_name(model, deployment) for deployment in deployments)) or (model,) + + +def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]: + registry: Final = llm_router.registry() + resolved: Final = registry.resolve(model, kind="alias") or model + routing_group: Final = llm_router.get_routing_group(resolved) + members: Final = routing_group.models if routing_group is not None else (resolved,) + named: Final = tuple( + chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members) + ) + return named or registry.deployments(resolved, source="wildcard", team_id=team_id) + + +def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int: + """What OpenAI chat completions bills on top of the counted pages of `pdf_files` inline PDF files, when + `counted_model`, a name from `models_counted_for`, is served by OpenAI itself.""" + return OPENAI_CHAT_PDF_FILE_TOKENS * pdf_files if counted_model.startswith(_OPENAI_PREFIX) else 0 + + +def inline_pdf_file_count(messages: object) -> int: + return sum(1 for part in _content_parts(messages) if _is_inline_pdf_file(part)) + + +def _content_parts(messages: object) -> Iterator[Mapping[str, object]]: + contents: Final = (_as_list(str_keyed(message).get("content")) for message in _as_list(messages)) + return (str_keyed(part) for part in chain.from_iterable(contents)) + + +def _as_list(value: object) -> Sequence[object]: + return value if isinstance(value, list) else () + + +def _is_inline_pdf_file(part: Mapping[str, object]) -> bool: + file_data: Final = str_keyed(part.get("file")).get("file_data") + return part.get("type") == "file" and isinstance(file_data, str) and file_data.startswith(PDF_DATA_URL_PREFIX) + + +def _counted_name(model: str, deployment: object) -> str: + openai_model: Final = deployment_openai_model(deployment) + if openai_model is not None and _served_by_openai(deployment): + return _OPENAI_PREFIX + openai_model.removeprefix(_OPENAI_PREFIX) + if openai_model is not None: + return openai_model + served: Final = next(iter(deployment_pricing_models(deployment)), model) + if pdf_page_pricing(served) != pdf_page_pricing(model) and reads_pdf_pages(served): + return served.partition("/")[2] or served + return unresolved_tokenizer_model(model) + + +def _served_by_openai(deployment: object) -> bool: + params: Final = str_keyed(str_keyed(deployment).get("litellm_params")) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return False + try: + _, served_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + except litellm.exceptions.BadRequestError: + return False + return served_provider == "openai" + + def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: diff --git a/pyproject.toml b/pyproject.toml index de1abeddd7e..6a21fd9b26b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -311,6 +311,7 @@ benchmarks = [ "pytest-codspeed==4.3.0", "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", + "pypdf>=6.21.2,<7.0", ] [build-system] diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..b2a86459438 --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,126 @@ +"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed. + +Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size.""" + +import base64 +from itertools import accumulate, chain +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import token_counter + +TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." + + +def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str: + escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(), + *xobjects, + ) + header: Final = b"%PDF-1.4\n" + objects: Final = tuple( + f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1) + ) + offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1]))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n" + ).encode() + return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + + +def _form(resources_number: int, drawing: bytes) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _count_pdf(pdf_base64: str) -> int: + document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}} + return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}]) + + +def _page_of_drawing_operators() -> str: + return _pdf_base64("", "<< >>", b" q Q" * 50_000) + + +def _page_of_forms_sharing_resources() -> str: + form_count: Final = 300 + shared_resources: Final = ( + "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" + ).encode() + forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count)) + return _pdf_base64( + TEXT_LINE, + "<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>", + b" q 10 0 0 10 0 0 cm /X0 Do Q", + (forms[0], shared_resources, *forms), + ) + + +def _page_of_parents_sharing_a_child_form() -> str: + parent_count: Final = 100 + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child)) + + +PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators() +PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources() +PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form() +BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b"")) +TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b"")) +LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000 +NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8 +UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE) + + +@pytest.mark.benchmark +def test_pdf_page_of_drawing_operators_and_no_image() -> None: + assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None: + assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None: + assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_token_counter_long_unknown_model_name() -> None: + assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index f03c7421899..c2f8e12a726 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -30,7 +30,11 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import ( + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _encoding_count, @@ -2219,11 +2223,11 @@ def test_anthropic_image_block_without_source_raises(): ) -def _count_user_content(content: list[dict]) -> int: +def _count_user_content(content: list[dict], model: str = "anthropic/claude-fable-5") -> int: from litellm.litellm_core_utils.token_counter import token_counter return token_counter( - model="anthropic/claude-fable-5", + model=model, messages=[{"role": "user", "content": content}], use_default_image_token_count=True, ) @@ -2319,14 +2323,27 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): ) -def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str: +def _pdf_base64( + pages: tuple[str, ...], + width: int = 612, + height: int = 792, + xobjects: tuple[bytes, ...] = (), + drawing: bytes = b"", + page_resources: str | None = None, +) -> str: + """`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every + page's content stream ends with `drawing`. `page_resources` replaces every page's /Resources dictionary.""" + first_xobject: Final = 4 + 2 * len(pages) + page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else "" + resources: Final = page_resources if page_resources is not None else f"<< /Font << /F1 3 0 R >>{page_xobjects} >>" + def page_objects(index: int, text: str) -> tuple[bytes, bytes]: escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") - stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream" page: Final = ( f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] " - f"/Resources << /Font << /F1 3 0 R >> >> /Contents {4 + 2 * index} 0 R >>" + f"/Resources {resources} /Contents {4 + 2 * index} 0 R >>" ).encode() return content, page @@ -2336,6 +2353,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> f"<< /Type /Pages /Count {len(pages)} /Kids [ {kids} ] >>".encode(), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", *chain.from_iterable(page_objects(index, text) for index, text in enumerate(pages)), + *xobjects, ) header: Final = b"%PDF-1.4\n" objects: Final = tuple( @@ -2414,6 +2432,401 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest. assert priced_as_image < priced_by_page +_REVENUE_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." +_HEADCOUNT_LINE: Final = "Headcount is flat and the office lease was renewed for three years." +_ONE_PIXEL_IMAGE: Final = ( + b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 " + b"/Length 1 >>\nstream\n\x00\nendstream" +) + + +_DRAW_X0: Final = b" q 10 0 0 10 0 0 cm /X0 Do Q" +_DRAW_X1: Final = b"/X1 Do" + + +def _form_xobject(resources_number: int, drawing: bytes = _DRAW_X1) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _xobject_resources(xobject_number: int) -> bytes: + return f"<< /XObject << /X1 {xobject_number} 0 R >> >>".encode() + + +def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]: + return {"type": "document", "source": _pdf_source(pdf_base64)} + + +def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]: + file_data: Final = "data:application/pdf;base64," + pdf_base64 + return {"type": "file", "file": {"filename": "report.pdf", "file_data": file_data}} + + +def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]: + return {"type": "tool_result", "tool_use_id": "t1", "content": [_as_pdf_document(pdf_base64)]} + + +def _pdf_cost(model: str, pdf_base64: str, wrap: Callable[[str], Mapping[str, object]] = _as_pdf_document) -> int: + return _count_user_content([wrap(pdf_base64)], model) - _count_user_content([wrap(_pdf_base64(()))], model) + + +def _text_cost(model: str, *lines: str) -> int: + return sum( + _count_user_content([{"type": "text", "text": line}], model) - _count_user_content([], model) for line in lines + ) + + +def _rendered_page_cost(width: int = 612, height: int = 792) -> int: + return _pdf_cost("my-private-model", _pdf_base64(("",), width, height)) + + +@pytest.mark.parametrize( + "wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"] +) +@pytest.mark.parametrize( + "model", + ["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"], +) +def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing( + model: str, wrap: Callable[[str], Mapping[str, object]] +): + """OpenAI models bill a PDF page with text as its text plus 13 tokens and a blank page as nothing, with no page + image. Measured 2026-10-20 on Azure chat completions gpt-5.6-sol (1, 2, and 10 pages of 15 text tokens billed + +28, +56, +280, and 1, 2, or 10 blank pages billed the same as none) and on OpenAI /v1/responses/input_tokens + for gpt-4o, gpt-4.1, gpt-5, gpt-5.5, o3, and o4-mini. + + Before the fix every model got Claude's pricing, so count_tokens answered about 1586 for a one-page text PDF + that Azure gpt-5.6-sol billed at 61. + """ + one_page: Final = _pdf_cost(model, _pdf_base64((_REVENUE_LINE,)), wrap) + + assert one_page == _text_cost(model, _REVENUE_LINE) + 13 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE, _HEADCOUNT_LINE)), wrap) == ( + _text_cost(model, _REVENUE_LINE, _HEADCOUNT_LINE) + 2 * 13 + ) + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,) * 10), wrap) == 10 * one_page + assert _pdf_cost(model, _pdf_base64(("",) * 5), wrap) == 0 + assert _pdf_cost(model, _pdf_base64(("", _REVENUE_LINE, "")), wrap) == one_page + + +@pytest.mark.parametrize( + "xobjects", + [(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)], + ids=["image_on_the_page", "image_inside_a_form"], +) +def test_openai_model_prices_a_page_that_draws_a_raster_image_as_its_text_plus_the_rendered_page( + xobjects: tuple[bytes, ...], +): + """A page embedding a raster image keeps the rendered-page estimate every model got before, so a reservation + never undercounts it: on 2026-10-20 /v1/responses/input_tokens billed a page with one drawn image at 1620 on + gpt-5.6-sol and gpt-6-sol, above OpenAI's own high-detail page image (765 for Letter), while a vector-drawn + chart page billed no image. + """ + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == _rendered_page_cost() + assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == ( + _rendered_page_cost(300, 300) + ) + + +def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws(): + """Pages sharing one /Resources all list its logo, but only the page that paints it bills a page image: measured + 2026-10-20 on Azure gpt-5.6-sol, a 3-page PDF drawing a logo on page 1 billed 403, not three page images.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,))) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + form_listing_an_unused_image: Final = (_form_xobject(7, b""), _xobject_resources(8), _ONE_PIXEL_IMAGE) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=form_listing_an_unused_image, drawing=_DRAW_X0)) == 0 + + +def test_openai_model_prices_no_image_for_forms_that_draw_none(): + """A vector-drawn form XObject bills no page image, and a form that lists itself cannot loop the walk.""" + model: Final = "gpt-4o" + self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6)) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + + +_INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" + + +def test_openai_model_prices_a_page_that_draws_an_inline_image_like_an_image_xobject(): + """An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured + 2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o.""" + model: Final = "gpt-4o" + form_drawing_inline_image: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(_INLINE_IMAGE)} >>\nstream\n".encode() + + _INLINE_IMAGE + + b"\nendstream" + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == ( + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == ( + _rendered_page_cost() + ) + + +def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk(): + """A chain of 2,000 nested form XObjects used to exhaust the stack, which dropped the whole document's text and + priced it as one image. The walk now stops at a fixed depth and prices the page as if it drew an image.""" + model: Final = "gpt-4o" + depth: Final = 2000 + first: Final = 6 + chain_objects: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * level + 1), _xobject_resources(first + 2 * level + 2)) for level in range(depth) + ) + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + + +@pytest.mark.parametrize("drawing", [b" /A Do /B Do", b" /B Do /A Do"]) +def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_one_parent(drawing: bytes): + """A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two + parents share reaches the image only through the parent whose resources list it.""" + model: Final = "gpt-4o" + child_without_resources: Final = ( + b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream" + ) + parents_and_child: Final = ( + _form_xobject(7, b"/C Do"), + b"<< /XObject << /C 10 0 R >> >>", + _form_xobject(9, b"/C Do"), + b"<< /XObject << /C 10 0 R /Y 11 0 R >> >>", + child_without_resources, + _ONE_PIXEL_IMAGE, + ) + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=parents_and_child, + drawing=drawing, + page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>", + ), + ) == _rendered_page_cost() + + +@pytest.mark.parametrize( + ("drawing", "draws_image"), + [(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)], + ids=["comment-before-do", "do-inside-a-comment"], +) +def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool): + """A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after + it nor draws anything itself.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == ( + _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13) + ) + + +@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"]) +def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str): + """A name written with #xx escapes reads the same in the content stream and in /Resources, decoded the way pypdf + decodes it, so an image drawn under such a name still prices the page as rendered.""" + model: Final = "gpt-4o" + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=(_ONE_PIXEL_IMAGE,), + drawing=f" {escaped_name} Do".encode(), + page_resources=f"<< /XObject << {escaped_name} 6 0 R >> >>", + ), + ) == _rendered_page_cost() + + +@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)]) +def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered( + parent_count: int, draws_too_many: bool +): + """A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects + between them, so a page of forms nested to blow up the walk cannot stall the request.""" + model: Final = "gpt-4o" + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2)) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + + assert _pdf_cost( + model, + _pdf_base64( + (_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources + ), + ) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13) + + +_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.6-sol", + "gpt-5.4-mini", + "gpt-6-sol", + "azure/gpt-41", + "azure/gpt-5.6-sol", + "eu/gpt-5.6-sol", + "gpt-5.6-sol-2026-07-09", + ], +) +def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str): + """OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on + /v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a + non-Latin sample like this one that is 249 tokens in o200k_base and 297 in cl100k_base.""" + o200k: Final = tiktoken.get_encoding("o200k_base") + + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE)) + + +@pytest.mark.parametrize(("deployment_name", "model"), [("azure/gpt-4", "gpt-4"), ("eu/gpt-35-turbo", "gpt-35-turbo")]) +def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deployment_name: str, model: str): + assert token_counter_new(model=deployment_name, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"] +) +def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str): + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize( + "model", + [ + "openrouter/openai/gpt-4o", + "github_copilot/gpt-4o", + "azure_ai/gpt-4o", + "azure/responses/gpt-5.6-sol", + "azure/o_series/o3-mini", + ], +) +def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str): + """The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes + a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 on + 2026-10-20, closer to OpenAI's pricing than to Claude's 1534.""" + bare_name: Final = model.rpartition("/")[2] + pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0) + + assert _pdf_cost(model, pdf_with_a_drawn_image) == _pdf_cost(bare_name, pdf_with_a_drawn_image) + assert _pdf_cost(model, _pdf_base64(("",))) == 0 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize( + "model", + [ + "text-embedding-ada-002-v2", + "tts-1", + "whisper-1", + "azure/whisper-1", + "github_copilot/text-embedding-3-small-inference", + "azure_text/gpt-35-turbo-instruct", + "ft:davinci-002:my-org::abc123", + ], +) +def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str): + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) + ) + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + + +@pytest.mark.parametrize( + ("fine_tune", "base_model"), + [ + ("ft:gpt-4o-mini-2024-07-18:my-org::abc123", "gpt-4o-mini-2024-07-18"), + ("openai/ft:gpt-4.1-2025-04-14:my-org:support:def456", "gpt-4.1-2025-04-14"), + ], +) +def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_model: str): + pdf: Final = _pdf_base64((_REVENUE_LINE,)) + + assert _pdf_cost(fine_tune, pdf) == _pdf_cost(base_model, pdf) == _text_cost(base_model, _REVENUE_LINE) + 13 + assert token_counter_new(model=fine_tune, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=base_model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", + [ + "", + "anthropic/claude-fable-5", + "DeepSeek-V4-Flash-0731", + "my-private-model", + "azure/mistral-large-latest", + "command-r-plus", + ], +) +def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str): + """Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps + Claude's text plus rendered page image, 1534 tokens for a blank Letter page.""" + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 1534 + + +def test_messages_reach_token_count_prices_pdf_pages_like_token_counter(): + from litellm.litellm_core_utils.token_counter import messages_reach_token_count + + messages: Final = [{"role": "user", "content": [_as_pdf_document(_pdf_base64((_REVENUE_LINE, "")))]}] + total: Final = token_counter_new(model="azure/gpt-5.6-sol", messages=messages) + + assert total == token_counter_new(model="azure/gpt-5.6-sol", messages=[{"role": "user", "content": []}]) + ( + _text_cost("azure/gpt-5.6-sol", _REVENUE_LINE) + 13 + ) + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total) is True + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total + 1) is False + + def _png_data_url(width: int, height: int) -> str: ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() @@ -2747,6 +3160,14 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t } +def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None: + sample: Final = "A custom tokenizer passed with no model still counts." + + assert litellm.token_counter(model=None, custom_tokenizer=litellm.utils.select_tokenizer("gpt-4o"), text=sample) == ( + len(tiktoken.get_encoding("cl100k_base").encode(sample)) + ) + + def test_process_exits_cleanly_while_a_background_tokenizer_retry_is_downloading(tmp_path: Path) -> None: assert _run_in_memory_hub(HUB_RETRY_DURING_EXIT_SCRIPT, {}, "", tmp_path) == {"retry_reached_hub": True} diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 933d1f48ad4..b66e7f3d091 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -46,7 +46,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, ) -from litellm.proxy.spend_tracking.input_tokens import tokenizer_models +from litellm.proxy.spend_tracking.input_tokens import models_counted_for from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token from litellm.router_utils.ptu_shares import PTUTeamCeiling @@ -9101,6 +9101,63 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit( assert "expensive" in str(denied.value.detail) +@pytest.mark.asyncio +async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_deployments() -> None: + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), + counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"), + ) + messages: Final = _one_page_pdf_message() + caller: Final = UserAPIKeyAuth( + api_key=hash_token("sk-pdf-itpm"), + project_id="pdf-itpm", + project_metadata={"model_itpm_limit": {"gpt-4o": 1_000_000}, "model_otpm_limit": {"gpt-4o": 1_000_000}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=cache, + data={"model": "gpt-4o", "messages": messages, "max_tokens": 10}, + call_type="acompletion", + ) + + stash: Final = get_request_stash() + assert stash is not None + by_name: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), counted_models_resolver=lambda model, team_id: (model,) + ) + claude_count: Final = by_name._estimate_precise_input_tokens({"messages": messages}, "claude-haiku-5-5") + assert claude_count > by_name._estimate_precise_input_tokens({"messages": messages}, "gpt-4o") + assert stash.itpm_reserved_tokens == claude_count + + +def test_project_itpm_adds_what_openai_chat_bills_on_top_of_a_pdf_files_pages_on_chat_completions_only() -> None: + from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + router: Final = Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test"}} + for name, model in (("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol")) + ] + ) + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), + counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id), + ) + data: Final = {"messages": _one_page_pdf_message()} + + assert handler._estimate_precise_input_tokens(data, "on-openai", "acompletion") == ( + handler._estimate_precise_input_tokens(data, "on-azure", "acompletion") + OPENAI_CHAT_PDF_FILE_TOKENS + ) + assert handler._estimate_precise_input_tokens(data, "on-openai", "anthropic_messages") == ( + handler._estimate_precise_input_tokens(data, "on-azure", "anthropic_messages") + ) + + def _extraction_request(state: str, call_id: str) -> dict[str, object]: return { "model": "invoice-reader", @@ -9184,7 +9241,7 @@ _MULTILINGUAL_REQUEST: Final[dict[str, object]] = { def _project_itpm_estimate(router: Router, model: str) -> int: handler: Final = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(DualCache()), - tokenizer_models_resolver=partial(tokenizer_models, llm_router=router), + counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id), ) return handler._estimate_precise_input_tokens(_MULTILINGUAL_REQUEST, model, "acompletion") @@ -9193,7 +9250,7 @@ def test_project_itpm_counts_a_group_mixing_in_a_non_openai_model_by_its_name_as router: Final = Router( model_list=[ {"model_name": "gpt-6.1-sol", "litellm_params": {"model": model, "api_key": "sk-test"}} - for model in ("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5") + for model in ("openai/gpt-6.1-sol", "cohere/command-r-plus") ] ) messages: Final = _MULTILINGUAL_REQUEST["messages"] diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 035b7d9dcd3..a87ac7fa7c0 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -429,6 +429,24 @@ def test_budget_window_start_reads_json_encoded_windows(window: str, expected: d assert get_budget_window_start(window) == expected +@pytest.mark.asyncio +async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_group() -> None: + from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + request_body: Final = {"model": "gpt-4o", "messages": _one_page_pdf_message()} + router: Final = Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}] + ) + + counts: Final = await count_request_input_tokens( + request_body=request_body, route="/chat/completions", llm_router=router, team_id=None + ) + + assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")} + assert counts["gpt-4o"] > count_input_tokens_for_model(request_body=request_body, model="gpt-4o") + + _MULTILINGUAL_MESSAGES: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}] @@ -446,10 +464,38 @@ async def _input_tokens(router: Router, model: str) -> dict[str, int]: request_body={"model": model, "messages": _MULTILINGUAL_MESSAGES}, route="/v1/chat/completions", llm_router=router, + team_id=None, ) ) +async def _reserved_input_tokens(router: Router, model: str, route: str, messages: list[dict[str, object]]) -> int: + counts: Final = await count_request_input_tokens( + request_body={"model": model, "messages": messages}, route=route, llm_router=router, team_id=None + ) + return counts[model] + + +@pytest.mark.asyncio +async def test_an_openai_chat_request_reserves_what_openai_chat_bills_on_top_of_each_pdf_files_pages() -> None: + from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + router: Final = _router_with_groups(("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol")) + two_pdfs: Final = [*_one_page_pdf_message(), *_one_page_pdf_message()] + + assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", two_pdfs) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", two_pdfs) + + 2 * OPENAI_CHAT_PDF_FILE_TOKENS + ) + assert await _reserved_input_tokens(router, "on-openai", "/v1/messages", two_pdfs) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/messages", two_pdfs) + ) + assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) + ) + + @pytest.mark.asyncio async def test_alias_group_of_an_openai_model_reserves_on_that_models_token_count(): router: Final = _router_with_groups(("gpt-6.1-sol", "openai/gpt-6.1-sol"), ("prod-gpt-6", "openai/gpt-6.1-sol")) @@ -516,7 +562,7 @@ async def test_release_named_group_or_alias_no_openai_model_serves_reserves_on_a async def test_group_mixing_in_a_non_openai_model_reserves_on_its_name_as_written_and_its_openai_models( group: str, counted_as: str ): - served: Final = ("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5") + served: Final = ("openai/gpt-6.1-sol", "cohere/command-r-plus") router: Final = _router_with_groups(*((group, model) for model in served)) reserved: Final = (await _input_tokens(router, group))[group] diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index c7f13651471..40aee7bb00d 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import json from types import MappingProxyType from typing import Final @@ -12,7 +13,9 @@ from litellm.proxy.spend_tracking.input_tokens import ( TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, count_input_tokens, count_input_tokens_for_model, + models_counted_for, ) +from litellm.router import Router CL100K_MODEL: Final = "gpt-4" @@ -30,6 +33,173 @@ async def test_large_input_is_still_counted() -> None: assert isinstance(counts, MappingProxyType) +def _one_page_pdf_message() -> list[dict[str, object]]: + stream: Final = b"BT /F1 12 Tf 72 720 Td (Revenue grew eleven percent while churn fell.) Tj ET" + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 4 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> " + b"/Contents 5 0 R >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + ) + objects: Final = tuple(f"{n} 0 obj\n".encode() + body + b"\nendobj\n" for n, body in enumerate(bodies, start=1)) + header: Final = b"%PDF-1.4\n" + offsets: Final = tuple(len(header) + sum(len(obj) for obj in objects[:index]) for index in range(len(objects))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{offsets[-1] + len(objects[-1])}\n%%EOF\n" + ).encode() + pdf: Final = base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + return [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"filename": "report.pdf", "file_data": f"data:application/pdf;base64,{pdf}"}}, + {"type": "text", "text": "Summarize the report."}, + ], + } + ] + + +def _router(*deployments: tuple[str, str]) -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test", "api_base": "https://x"}} + for name, model in deployments + ] + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model", "counted_with"), + [ + ("gpt-4o", "anthropic/claude-haiku-5-5", "claude-haiku-5-5"), + ("pdf-reports", "azure/gpt-5.6-sol", "azure/gpt-5.6-sol"), + ("pdf-reports", "azure/responses/gpt-5.6-sol", "azure/responses/gpt-5.6-sol"), + ("pdf-reports", "openrouter/openai/gpt-5.6-sol", "openrouter/openai/gpt-5.6-sol"), + ("pdf-reports", "openai/gpt-5.6-sol", "openai/gpt-5.6-sol"), + ("pdf-reports", "gpt-5.6-sol", "openai/gpt-5.6-sol"), + ], + ids=[ + "gpt-named-group-served-by-claude", + "custom-group-served-by-azure-gpt", + "custom-group-served-by-azure-responses-gpt", + "custom-group-served-by-openrouter-gpt", + "custom-group-served-by-openai-gpt", + "custom-group-served-by-unprefixed-openai-gpt", + ], +) +def test_a_model_group_counts_like_the_deployment_that_serves_it( + group: str, deployment_model: str, counted_with: str +) -> None: + request_body: Final = {"model": group, "messages": _one_page_pdf_message()} + + assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,) + assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model( + request_body=request_body, model=group + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model"), + [ + ("my-claude", "anthropic/claude-haiku-5-5"), + ("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"), + ("my-cohere", "cohere/command-r-plus"), + ("gpt-4o", "cohere/command-r-plus"), + ], + ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"], +) +def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group( + group: str, deployment_model: str +) -> None: + assert models_counted_for(group, _router((group, deployment_model)), None) == (group,) + + +def test_a_custom_named_deployment_counts_with_its_base_model(): + router: Final = Router( + model_list=[ + { + "model_name": "reports", + "litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"}, + "model_info": {"base_model": "azure/gpt-5.6-sol"}, + }, + {"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}}, + ] + ) + + assert models_counted_for("reports", router, None) == ("azure/gpt-5.6-sol",) + assert models_counted_for("drafts", router, None) == ("drafts",) + + +def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is(): + router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol")) + + assert models_counted_for("mixed", router, None) == ("mixed", "azure/gpt-5.6-sol") + assert models_counted_for("not-deployed", router, None) == ("not-deployed",) + assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",) + + +def test_a_team_request_counts_with_the_deployment_its_team_is_served_by(): + router: Final = Router( + model_list=[ + {"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}, + { + "model_name": "reports-team-a", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "reports"}, + }, + ] + ) + + assert models_counted_for("reports", router, "team-a") == ("azure/gpt-5.6-sol",) + assert models_counted_for("reports", router, "team-b") == ("reports",) + + +@pytest.mark.parametrize("requested", ["reports", "latest-reports"], ids=["routing-group", "alias-of-routing-group"]) +def test_a_team_request_to_a_routing_group_counts_with_the_member_deployment_its_team_is_served_by(requested: str): + router: Final = Router( + model_list=[ + { + "model_name": "drafts", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x", "api_base": "https://x"}, + }, + { + "model_name": "drafts-team-a", + "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "drafts"}, + }, + ], + routing_groups=[{"group_name": "reports", "models": ["drafts"], "routing_strategy": "simple-shuffle"}], + model_group_alias={"latest-reports": "reports"}, + ) + + assert models_counted_for(requested, router, "team-a") == models_counted_for( + requested, _router((requested, "anthropic/claude-haiku-5-5")), None + ) + assert models_counted_for(requested, router, "team-b") == ("azure/gpt-5.6-sol",) + + +def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard(): + router: Final = Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "anthropic/*", "api_key": "x"}}, + { + "model_name": "team-a-gpt", + "litellm_params": {"model": "azure/gpt-*", "api_key": "x", "api_base": "https://x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "gpt-*"}, + }, + ] + ) + + assert models_counted_for("gpt-5.6-sol", router, "team-a") == ("azure/gpt-5.6-sol",) + assert models_counted_for("gpt-5.6-sol", router, "team-b") == models_counted_for("gpt-5.6-sol", router, None) + assert "azure/gpt-5.6-sol" not in models_counted_for("gpt-5.6-sol", router, "team-b") + + def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: model: Final = "claude-sonnet-5-5" body_without_tool_choice: Final = { diff --git a/tests/unit/rust_bridge/test_token_counter.py b/tests/unit/rust_bridge/test_token_counter.py index 068d389ebbd..ba9c0b90498 100644 --- a/tests/unit/rust_bridge/test_token_counter.py +++ b/tests/unit/rust_bridge/test_token_counter.py @@ -144,9 +144,10 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gemini/gemini-2.5-pro", "cl100k_base"), ("mistral/mistral-large-latest", "cl100k_base"), ("my-router-alias", "cl100k_base"), - ("azure/gpt-4o", "o200k_base"), ("command-r-plus", "cl100k_base"), ("gpt-4o", "o200k_base"), + ("azure/gpt-4o", "o200k_base"), + ("eu/gpt-5.6-sol", "o200k_base"), ("gpt-4o-mini", "o200k_base"), ("gpt-4o-2024-08-06", "o200k_base"), ("gpt-4.1", "o200k_base"), diff --git a/uv.lock b/uv.lock index 2ba97d15327..87b8fc95486 100644 --- a/uv.lock +++ b/uv.lock @@ -5109,6 +5109,7 @@ admin-mcp = [ benchmarks = [ { name = "a2a-sdk" }, { name = "mcp" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -5337,6 +5338,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1 benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, + { name = "pypdf", specifier = ">=6.21.2,<7.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ] From 89ca8d83f89b876db0353069034a1377a94cde05 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:28:16 -0700 Subject: [PATCH 04/34] fix(ci): measure PR gates from the target branch's current tip (#49078) * fix(ci): measure PR gates from the target branch's current tip The lint gate base, the ui lint and ui unit test scopes, and the spend logs guard compared the event's pull_request.base.sha with the head to find the merge base. GitHub can leave base.sha weeks behind the base branch, and when that old commit is an ancestor of a head that merged newer main, compare returns the old commit itself, so the strict gates blamed main's own code on the PR. Compare base.ref with the head instead, passed through env, so GitHub resolves the branch tip when the job runs. This is the same base make check uses locally. The ui unit test push guard now keys on BASE_REF, which is empty on push and workflow_dispatch. The regression test runs every workflow step that reads merge_base_commit against a fake gh backed by a temp repo with a stale base.sha, for main and a litellm_/ base with a slash. * fix(ci): check out the base branch's current guard code in the PR guards * fix(ci): read the PR's current base branch so re-runs survive a deleted base Every merge-base step now asks the pulls API for the PR's base branch instead of reading the event's base.ref, which a re-run keeps after a stacked base is merged and deleted. The guard checkouts go back to base.sha, which stays reachable. * test(ci): mark the step script path Final * fix(ci): keep the spend logs guard on main and run both guard scripts from the base branch tip * fix(ci): measure from the event's base branch unless it was deleted, and test what each step hands on A re-run after a retarget off main kept following the PR's current base, so lint, ui-unit and the cost map guard could pass without measuring against main. They now read the current base only when the event's base branch no longer exists. The tests run each merge base step whole and check the value its consumer gets * fix(ci): follow a renamed base branch to its new name before the compare --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/cost-map-guard.yml | 10 +- .github/workflows/spend-logs-guard.yml | 6 +- .github/workflows/test-linting.yml | 19 +- .github/workflows/test-unit.yml | 8 +- tests/unit/test_lint_workflow_diff_gates.py | 393 +++++++++++++++++++- tests/unit/test_select_ui_test_scope.py | 65 +++- 6 files changed, 461 insertions(+), 40 deletions(-) diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index fb86237e504..f26c649aad2 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -8,6 +8,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -21,16 +22,19 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - - name: Fetch the pull request head, its merge base, and the base branch tip + - name: Check out the base branch tip and fetch the pull request head and its merge base id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')" + base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)" + target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')" merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target" + git checkout -q --detach "$target" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" echo "target=$target" >> "$GITHUB_OUTPUT" - name: Set up uv diff --git a/.github/workflows/spend-logs-guard.yml b/.github/workflows/spend-logs-guard.yml index cc3bda9b880..bdba95f6a5d 100644 --- a/.github/workflows/spend-logs-guard.yml +++ b/.github/workflows/spend-logs-guard.yml @@ -19,17 +19,17 @@ jobs: steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - ref: ${{ github.event.pull_request.base.sha }} + ref: ${{ github.event.pull_request.base.ref }} persist-credentials: false - name: Fetch the pull request head, its merge base and its reviews id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} run: | - merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" + merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 88c66f0011d..c3388430589 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -48,11 +48,13 @@ jobs: if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; } - MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$MERGE_BASE" retry git fetch --no-tags --depth=1 origin "$MERGE_BASE" echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" @@ -264,6 +266,7 @@ jobs: name: ui permissions: contents: read + pull-requests: read runs-on: ubuntu-latest timeout-minutes: 8 defaults: @@ -281,16 +284,12 @@ jobs: id: changed env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - # base.sha is the base branch tip from when the PR was opened, while - # actions/checkout leaves HEAD on a merge of the PR into the *current* - # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit - # landed since, so a PR that touches no UI file still gets linted - # against hundreds of other people's files. Diff the PR head against its - # own merge base instead, which is exactly what this PR changed. - merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$merge_base" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" : > "$RUNNER_TEMP/prettier_files.txt" diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a5013155eae..9ebb9bab018 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1150,18 +1150,20 @@ jobs: env: CI: "true" GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | full_suite() { npm run test -- --run --pool forks --maxWorkers=14; } - if [ -z "$BASE_SHA" ]; then + if [ -z "$PR_NUMBER" ]; then echo "Push to $GITHUB_REF_NAME: running the full suite" full_suite exit 0 fi - merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$merge_base" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" changed_files=() diff --git a/tests/unit/test_lint_workflow_diff_gates.py b/tests/unit/test_lint_workflow_diff_gates.py index 5d3c0d80aa6..816ef6ce707 100644 --- a/tests/unit/test_lint_workflow_diff_gates.py +++ b/tests/unit/test_lint_workflow_diff_gates.py @@ -1,19 +1,198 @@ from __future__ import annotations import fnmatch +import itertools +import os import re import shlex import subprocess +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, TypeAlias import pytest import yaml +from pydantic import BaseModel, ConfigDict, Field WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml" +WORKFLOWS_DIR: Final = WORKFLOW.parent +REPO_ROOT: Final = WORKFLOW.parents[2] DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|') GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text())) PYTHON_GATES: Final = tuple(gate for gate in GATES if gate[0].startswith(":(glob)")) +REPOSITORY: Final = "BerriAI/litellm" +UI_DASHBOARD: Final = "ui/litellm-dashboard" +STACKED_BASE_REF: Final = "litellm_/stacked" +RELEASE_BASE_REF: Final = "stable/next" +BASE_REFS: Final = ("main", STACKED_BASE_REF, RELEASE_BASE_REF) +RENAMED_BASE_REF: Final = "litellm_renamed_before_the_rerun" +GITHUB_EXPRESSION: Final = re.compile(r"\$\{\{\s*(.+?)\s*\}\}") +FAKE_GH: Final = r"""#!/usr/bin/env bash +set -euo pipefail +endpoint=${2%%\?*} +endpoint=${endpoint#"repos/$GITHUB_REPOSITORY/"} +commit() { git -C "$UPSTREAM" rev-parse --verify --quiet "$1^{commit}"; } +renamed() { git -C "$UPSTREAM" config --get "renamed.$1.to" || printf '%s\n' "$1"; } +case "$1 $endpoint ${3-} ${4-}" in + "api compare/"*"..."*" --jq .merge_base_commit.sha") + basehead=${endpoint#compare/} + git -C "$UPSTREAM" merge-base "$(commit "${basehead%%...*}")" "$(commit "${basehead#*...}")" ;; + "api pulls/1 --jq .base.ref") + printf '%s\n' "$LIVE_BASE_REF" ;; + "api branches/"*) + branch=$(renamed "${endpoint#branches/}") + sha=$(commit "refs/heads/$branch") || { [ "${3-}" = --silent ] || printf '{"message":"Not Found"}\n'; exit 1; } + case "${3-} ${4-}" in + "--silent ") ;; + "--jq .name") printf '%s\n' "$branch" ;; + "--jq .commit.sha") printf '%s\n' "$sha" ;; + *) echo "fake gh cannot answer: $*" >&2; exit 64 ;; + esac ;; + "api pulls/1/reviews --paginate --slurp") + printf '[[]]\n' ;; + *) + echo "fake gh cannot answer: $*" >&2 + exit 64 ;; +esac +""" +FAKE_ARGV_RECORDER: Final = r"""#!/usr/bin/env bash +printf '%s\n' "$@" > "$RUNNER_TEMP/$(basename "$0").argv" +""" + + +class WorkflowStep(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = "" + name: str = "" + uses: str = "" + with_: Mapping[str, str] = Field(default_factory=dict, alias="with") + env: Mapping[str, str] = Field(default_factory=dict) + run: str = "" + + +class RunDefaults(BaseModel): + model_config = ConfigDict(frozen=True) + + working_directory: str = Field(default=".", alias="working-directory") + + +class JobDefaults(BaseModel): + model_config = ConfigDict(frozen=True) + + run: RunDefaults = Field(default_factory=RunDefaults) + + +class WorkflowJob(BaseModel): + model_config = ConfigDict(frozen=True) + + defaults: JobDefaults = Field(default_factory=JobDefaults) + steps: tuple[WorkflowStep, ...] = () + + +class PullRequestTrigger(BaseModel): + model_config = ConfigDict(frozen=True) + + branches: tuple[str, ...] = ("**",) + + +class WorkflowTriggers(BaseModel): + model_config = ConfigDict(frozen=True) + + pull_request: PullRequestTrigger = Field(default_factory=PullRequestTrigger) + + +class Workflow(BaseModel): + model_config = ConfigDict(frozen=True) + + on: WorkflowTriggers + jobs: Mapping[str, WorkflowJob] + + +@dataclass(frozen=True, slots=True) +class MergeBaseStep: + workflow: str + job_id: str + job: WorkflowJob + step: WorkflowStep + base_branches: tuple[str, ...] + + @property + def job_key(self) -> str: + return f"{self.workflow}:{self.job_id}" + + +@dataclass(frozen=True, slots=True) +class StaleBaseUpstream: + path: Path + stale_base: str + head: str + tip_by_ref: Mapping[str, str] + merge_base_by_ref: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class Runner: + root: Path + upstream: StaleBaseUpstream + live_base_ref: str + context: Mapping[str, str] + + @property + def checkout(self) -> Path: + return self.root / "checkout" + + @property + def tools(self) -> Path: + return self.root / "bin" + + @property + def temp(self) -> Path: + return self.root / "runner_temp" + + @property + def github_output(self) -> Path: + return self.root / "github_output" + + @property + def github_env(self) -> Path: + return self.root / "github_env" + + +@dataclass(frozen=True, slots=True) +class StepRun: + step: MergeBaseStep + runner: Runner + + +Observation: TypeAlias = str | frozenset[str] | tuple[str, str] + + +@dataclass(frozen=True, slots=True) +class Consumer: + observe: Callable[[StepRun], Observation] + expect: Callable[[StaleBaseUpstream, str], Observation] + + +def _workflow(path: Path) -> Workflow: + return Workflow.model_validate(yaml.load(path.read_text(), Loader=yaml.BaseLoader)) + + +def _merge_base_steps_in(path: Path) -> Iterator[MergeBaseStep]: + workflow: Final = _workflow(path) + for job_id, job in workflow.jobs.items(): + yield from ( + MergeBaseStep(path.name, job_id, job, step, workflow.on.pull_request.branches) + for step in job.steps + if "merge_base_commit" in step.run + ) + + +MERGE_BASE_STEPS: Final = tuple( + itertools.chain.from_iterable(map(_merge_base_steps_in, sorted(WORKFLOWS_DIR.glob("*.y*ml")))) +) def _git(cwd: Path, *args: str) -> str: @@ -104,6 +283,218 @@ def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manife assert selected == frozenset({trigger}) +def _event_context(pull_request: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType( + { + "github.repository": REPOSITORY, + "github.token": "fake-token", + **{f"github.event.pull_request.{field}": value for field, value in pull_request.items()}, + } + ) + + +def _context_value(context: Mapping[str, str], expression: str) -> str: + if expression not in context: + pytest.fail(f"the fake runner has no value for ${{{{ {expression} }}}}") + return context[expression] + + +def _resolved(text: str, context: Mapping[str, str]) -> str: + return GITHUB_EXPRESSION.sub(lambda expression: _context_value(context, expression.group(1)), text) + + +def _install_tool(path: Path, script: str) -> None: + path.write_text(script) + path.chmod(0o755) + + +def _runner(tmp_path: Path, upstream: StaleBaseUpstream, pull_request: Mapping[str, str], live_base_ref: str) -> Runner: + runner: Final = Runner(tmp_path, upstream, live_base_ref, _event_context(pull_request)) + runner.tools.mkdir() + runner.temp.mkdir() + for name, script in (("gh", FAKE_GH), ("npm", FAKE_ARGV_RECORDER), ("uv", FAKE_ARGV_RECORDER)): + _install_tool(runner.tools / name, script) + return runner + + +def _check_out_like_the_job(job: WorkflowJob, runner: Runner) -> None: + checkout: Final = next(step for step in job.steps if step.uses.startswith("actions/checkout@")) + ref: Final = _resolved(checkout.with_.get("ref", "${{ github.event.pull_request.head.sha }}"), runner.context) + _git(runner.root, "init", "-q", str(runner.checkout)) + _git(runner.checkout, "remote", "add", "origin", f"file://{runner.upstream.path}") + _git(runner.checkout, "fetch", "-q", "--no-tags", "--depth=1", "origin", ref) + _git(runner.checkout, "checkout", "-q", "--detach", "FETCH_HEAD") + + +def _run_step(step: WorkflowStep, job: WorkflowJob, context: Mapping[str, str], runner: Runner) -> None: + result: Final = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _resolved(step.run, context)], + cwd=runner.checkout / job.defaults.run.working_directory, + env={ + **{name: _resolved(value, context) for name, value in step.env.items()}, + "PATH": f"{runner.tools}{os.pathsep}{os.environ['PATH']}", + "HOME": str(runner.root), + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_WORKSPACE": str(REPO_ROOT), + "GITHUB_OUTPUT": str(runner.github_output), + "GITHUB_ENV": str(runner.github_env), + "RUNNER_TEMP": str(runner.temp), + "UPSTREAM": str(runner.upstream.path), + "LIVE_BASE_REF": runner.live_base_ref, + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def _key_values(path: Path) -> Mapping[str, str]: + return MappingProxyType( + {key: value for key, _, value in (line.partition("=") for line in path.read_text().splitlines())} + ) + + +def _argv_recorded(runner: Runner, tool: str) -> tuple[str, ...]: + return tuple((runner.temp / f"{tool}.argv").read_text().splitlines()) + + +def _gate_base_exported(run: StepRun) -> str: + return _key_values(run.runner.github_env)["GATE_BASE_SHA"] + + +def _ui_files_queued_for_prettier(run: StepRun) -> frozenset[str]: + return frozenset((run.runner.temp / "prettier_files.txt").read_text().splitlines()) + + +def _ui_files_passed_to_vitest_related(run: StepRun) -> frozenset[str]: + argv: Final = _argv_recorded(run.runner, "npm") + return frozenset(argv[argv.index("related") + 1 : argv.index("--run")]) + + +def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]: + guard: Final = next(step for step in run.step.job.steps if step.name == "Run the guard") + outputs: Final = { + f"steps.{run.step.step.id}.outputs.{name}": value + for name, value in _key_values(run.runner.github_output).items() + } + _run_step(guard, run.step.job, MappingProxyType({**run.runner.context, **outputs}), run.runner) + argv: Final = _argv_recorded(run.runner, "uv") + return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD") + + +def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str: + return upstream.merge_base_by_ref[ref] + + +def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]: + merge_base: Final = upstream.merge_base_by_ref[ref] + changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head) + return frozenset(changed.split()) + + +def _merge_base_and_tip(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]: + return upstream.merge_base_by_ref[ref], upstream.tip_by_ref[ref] + + +CONSUMERS: Final = MappingProxyType( + { + "test-linting.yml:python": Consumer(_gate_base_exported, _merge_base), + "test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base), + "test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base), + "spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), + "cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), + } +) + + +def _sha(repo: Path, revision: str) -> str: + return _git(repo, "rev-parse", revision).strip() + + +def _commit_ui_file(repo: Path, name: str) -> None: + target: Final = repo / UI_DASHBOARD / "src" / f"{name}.ts" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"export const {name} = 1;\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", name) + + +@pytest.fixture(scope="module") +def upstream(tmp_path_factory: pytest.TempPathFactory) -> StaleBaseUpstream: + path: Final = tmp_path_factory.mktemp("upstream") + _git(path, "init", "-q", "-b", "main") + _git(path, "config", "user.email", "t@t") + _git(path, "config", "user.name", "t") + _commit_ui_file(path, "main_when_the_pull_request_was_opened") + _git(path, "branch", "feature") + _git(path, "branch", RELEASE_BASE_REF) + _commit_ui_file(path, "main_since") + _git(path, "checkout", "-q", "-b", STACKED_BASE_REF) + _commit_ui_file(path, "stacked_base_since") + _git(path, "checkout", "-q", "feature") + _commit_ui_file(path, "pull_request_work") + _git(path, "merge", "-q", "--no-ff", "-m", "merge the newer base", STACKED_BASE_REF) + _git(path, "config", f"renamed.{RENAMED_BASE_REF}.to", STACKED_BASE_REF) + return StaleBaseUpstream( + path=path, + stale_base=_sha(path, "main~1"), + head=_sha(path, "feature"), + tip_by_ref=MappingProxyType({ref: _sha(path, ref) for ref in BASE_REFS}), + merge_base_by_ref=MappingProxyType( + {ref: _git(path, "merge-base", ref, "feature").strip() for ref in BASE_REFS} + ), + ) + + +def test_every_workflow_that_reads_a_merge_base_has_its_steps_discovered() -> None: + assert MERGE_BASE_STEPS + assert frozenset(step.workflow for step in MERGE_BASE_STEPS) == frozenset( + path.name for path in WORKFLOWS_DIR.glob("*.y*ml") if "merge_base_commit" in path.read_text() + ) + + +def _runs_for(step: MergeBaseStep, base_ref: str) -> bool: + return any(fnmatch.fnmatchcase(base_ref, pattern) for pattern in step.base_branches) + + +BASE_REF_CASES: Final = ( + ("main", "main", "main"), + ("stacked", STACKED_BASE_REF, STACKED_BASE_REF), + ("rerun_after_the_stacked_base_was_deleted", "litellm_deleted_after_merge", "main"), + ("rerun_after_the_stacked_base_was_renamed", RENAMED_BASE_REF, STACKED_BASE_REF), + ("rerun_after_a_retarget_off_main_to_a_stacked_branch", "main", STACKED_BASE_REF), + ("rerun_after_a_retarget_off_main_to_a_release_branch", "main", RELEASE_BASE_REF), +) +MERGE_BASE_CASES: Final = tuple( + pytest.param(step, event_base_ref, live_base_ref, id=f"{step.job_key}:{step.step.name}-{case}") + for step, (case, event_base_ref, live_base_ref) in itertools.product(MERGE_BASE_STEPS, BASE_REF_CASES) + if _runs_for(step, event_base_ref) +) + + +@pytest.mark.parametrize(("step", "event_base_ref", "live_base_ref"), MERGE_BASE_CASES) +def test_merge_base_step_hands_its_consumer_the_events_base_unless_that_branch_is_gone( + upstream: StaleBaseUpstream, tmp_path: Path, step: MergeBaseStep, event_base_ref: str, live_base_ref: str +) -> None: + pull_request: Final = MappingProxyType( + { + "number": "1", + "base.ref": event_base_ref, + "base.sha": upstream.stale_base, + "head.sha": upstream.head, + "head.ref": "feature", + "user.type": "User", + } + ) + runner: Final = _runner(tmp_path, upstream, pull_request, live_base_ref) + _check_out_like_the_job(step.job, runner) + _run_step(step.step, step.job, runner.context, runner) + consumer: Final = CONSUMERS[step.job_key] + measured_ref: Final = event_base_ref if event_base_ref in upstream.tip_by_ref else live_base_ref + assert consumer.observe(StepRun(step, runner)) == consumer.expect(upstream, measured_ref) + + UI_API_TYPES_DETECT: Final[str] = next( step["run"] for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"] diff --git a/tests/unit/test_select_ui_test_scope.py b/tests/unit/test_select_ui_test_scope.py index 4605aab956c..8f5a4b464bd 100644 --- a/tests/unit/test_select_ui_test_scope.py +++ b/tests/unit/test_select_ui_test_scope.py @@ -18,8 +18,12 @@ that consumes it, running the step's real shell against stubbed `gh`, `git` and from __future__ import annotations import os +import re import subprocess +from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType +from typing import Final import pytest import yaml @@ -28,6 +32,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SCOPE_SCRIPT = REPO_ROOT / ".github" / "scripts" / "select_ui_test_scope.sh" WORKFLOW = REPO_ROOT / ".github" / "workflows" / "test-unit.yml" STEP_NAME = "Run UI unit tests (Vitest)" +EXPRESSION = re.compile(r"\$\{\{\s*(.+?)\s*\}\}") +PULL_REQUEST_EVENT: Final = MappingProxyType( + {"number": "1", "base.ref": "main", "base.sha": "basesha", "head.sha": "headsha"} +) FULL_SUITE_ARGV = ["run", "test", "--", "--run", "--pool", "forks", "--maxWorkers=14"] @@ -73,13 +81,26 @@ def test_an_empty_change_set_fails_open_to_the_full_suite() -> None: assert scope([]) == "full" -def _step_script() -> str: - workflow = yaml.safe_load(WORKFLOW.read_text()) - steps = workflow["jobs"]["ui-unit"]["steps"] - script = next(step["run"] for step in steps if step.get("name") == STEP_NAME) - resolved = script.replace("${{ github.repository }}", "BerriAI/litellm") - assert "${{" not in resolved, "the step uses an Actions expression this harness does not resolve" - return resolved +def _resolve(text: str, pull_request: Mapping[str, str] | None) -> str: + def value(match: re.Match[str]) -> str: + expression: Final = match.group(1) + if expression == "github.repository": + return "BerriAI/litellm" + if expression == "github.token": + return "token" + field: Final = expression.removeprefix("github.event.pull_request.") + assert field != expression, f"the step uses ${{{{ {expression} }}}}, which this harness does not resolve" + return "" if pull_request is None else pull_request[field] + + return EXPRESSION.sub(value, text) + + +def _step_script_and_env(pull_request: Mapping[str, str] | None) -> tuple[str, dict[str, str]]: + workflow: Final = yaml.safe_load(WORKFLOW.read_text()) + steps: Final = workflow["jobs"]["ui-unit"]["steps"] + step: Final = next(step for step in steps if step.get("name") == STEP_NAME) + env: Final = {name: _resolve(str(value), pull_request) for name, value in step.get("env", {}).items()} + return _resolve(step["run"], pull_request), env def _stub(bin_dir: Path, name: str, body: str) -> None: @@ -88,7 +109,9 @@ def _stub(bin_dir: Path, name: str, body: str) -> None: stub.chmod(0o755) -def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> tuple[int, str, list[list[str]]]: +def _run_step( + tmp_path: Path, changed: list[str], pull_request: Mapping[str, str] | None = PULL_REQUEST_EVENT +) -> tuple[int, str, list[list[str]]]: """Run the workflow step's real shell; return its status, stdout and every npm argv.""" bin_dir = tmp_path / "bin" bin_dir.mkdir() @@ -100,20 +123,22 @@ def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> _stub(bin_dir, "git", 'if [ "$1" = diff ]; then cat "$CHANGED_FILES"; fi') _stub(bin_dir, "npm", 'printf "%s\\n" "$@" >>"$NPM_LOG"; printf "\\0" >>"$NPM_LOG"') - step = tmp_path / "step.sh" - step.write_text(_step_script()) + step_script, step_env = _step_script_and_env(pull_request) + script: Final = tmp_path / "step.sh" + script.write_text(step_script) - env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env["BASE_SHA"] = base_sha - env["HEAD_SHA"] = "headsha" - env["GITHUB_WORKSPACE"] = str(REPO_ROOT) - env["GITHUB_REF_NAME"] = "release_branch" - env["CHANGED_FILES"] = str(changed_file) - env["NPM_LOG"] = str(npm_log) + env: Final = { + **os.environ, + **step_env, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "GITHUB_WORKSPACE": str(REPO_ROOT), + "GITHUB_REF_NAME": "release_branch", + "CHANGED_FILES": str(changed_file), + "NPM_LOG": str(npm_log), + } result = subprocess.run( - ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(step)], + ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], cwd=tmp_path, capture_output=True, text=True, @@ -155,7 +180,7 @@ def test_step_runs_the_full_suite_when_a_changed_file_is_outside_src(tmp_path: P def test_step_runs_the_full_suite_on_a_push(tmp_path: Path) -> None: - returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], base_sha="") + returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], pull_request=None) assert returncode == 0, output assert invocations == [FULL_SUITE_ARGV] From a5b2198dc0f386d23664d46eb0757cba59962c22 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:28:26 -0700 Subject: [PATCH 05/34] fix(wandb): GLM-5.2 max_input_tokens from the CoreWeave Forge page (#49148) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b5d09d6ff4f..e8f6766ed25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -65126,7 +65126,7 @@ "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, "max_output_tokens": 262144, - "max_input_tokens": 1049000, + "max_input_tokens": 262000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, "cache_read_input_token_cost": 1.4e-07, @@ -65134,7 +65134,7 @@ "litellm_provider": "wandb", "mode": "chat", "supports_vision": false, - "source": "https://wandb.ai/site/pricing/tokens/" + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference" }, "deepinfra/openai/gpt-oss-120b-Turbo": { "max_output_tokens": 131072, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b5d09d6ff4f..e8f6766ed25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -65126,7 +65126,7 @@ "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, "max_output_tokens": 262144, - "max_input_tokens": 1049000, + "max_input_tokens": 262000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, "cache_read_input_token_cost": 1.4e-07, @@ -65134,7 +65134,7 @@ "litellm_provider": "wandb", "mode": "chat", "supports_vision": false, - "source": "https://wandb.ai/site/pricing/tokens/" + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference" }, "deepinfra/openai/gpt-oss-120b-Turbo": { "max_output_tokens": 131072, From 594f862bf3c859b1dcfe3fb9496ce22fdd72b49f Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:29:14 -0700 Subject: [PATCH 06/34] fix(streaming): rebuild each choice of an n>1 chat stream on its own (#49002) * fix(streaming): rebuild each choice of an n>1 chat stream on its own stream_chunk_builder folded every choice of an n>1 chat stream into choice 0, so post-call guardrails such as tool_permission re-emitted one choice holding every choice's tool calls, and spend logs and logging callbacks kept only the first choice's text. Each choice is now assembled from its own chunks, while usage, cost, and hidden params still read the whole stream. Cisco AI Defense compares each choice's streamed text on its own so safe n>1 streams still pass through unchanged. * fix(streaming): keep bridged reasoning summaries on choice 0 and infer an unreported choice finish reason * fix(streaming): match MCP follow-ups and bridged summary text to the rebuilt choices MCP auto-execute now answers only the first choice's tool calls, since its follow-up turn continues that choice alone. The Responses bridge streams a space before each reasoning summary part after the first, matching the non-streamed reply. build_base_response now takes the per-choice streams instead of having its choices replaced afterwards * fix(streaming): space every later reasoning summary part, deny per judged call, and rebuild in one text pass * fix(guardrails): key tool_permission denials for custom tool calls too, and type the stream rebuild's messages * fix(mcp): continue the first choice that called a tool when an n>1 reply auto-executes MCP tools * fix(noma): scan each choice of a multi-choice reply on its own * fix(noma): keep masked choices when a later choice's scan fails open on a stream --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../transformation.py | 10 +- .../streaming_chunk_builder_utils.py | 433 ++++++++++-------- .../chat/guardrail_translation/handler.py | 5 - .../cisco_ai_defense/cisco_ai_defense.py | 63 ++- .../guardrails/guardrail_hooks/noma/noma.py | 64 ++- .../guardrail_hooks/tool_permission.py | 25 +- .../mcp/litellm_proxy_mcp_handler.py | 40 +- ...responses_transformation_transformation.py | 39 ++ .../test_streaming_chunk_builder_utils.py | 112 ++++- .../test_cisco_ai_defense_chat.py | 56 +++ .../guardrails/guardrail_hooks/test_noma.py | 200 +++++++- .../guardrail_hooks/test_tool_permission.py | 111 +++++ .../mcp/test_litellm_proxy_mcp_handler.py | 80 ++++ tests/unit/test_main.py | 7 +- 14 files changed, 950 insertions(+), 295 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 580e55273f6..bb59b4617af 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state self._structured_output: Final = structured_output self._first_message_item_done: bool = False + self._reasoning_summary_streamed: bool = False def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place replays_openai_history: bool = False, structured_output: bool = False, + reasoning_summary_streamed: bool = False, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): tool_call_index_map: Per-stream output_index -> sequential tool_call index map replays_openai_history: Record the completed output's item sequence for the next turn structured_output: Only the first message item became content + reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text Returns: ModelResponseStream: OpenAI-formatted streaming chunk @@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) else: raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}") + elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed: + return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))]) elif event_type == "response.reasoning_summary_text.delta": content_part = parsed_chunk.get("delta", None) if content_part: return ModelResponseStream( choices=[ StreamingChoices( - index=cast(int, parsed_chunk.get("summary_index")), + index=0, delta=Delta(reasoning_content=content_part), ) ] @@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): tool_call_index_map=self._tool_call_index_map, replays_openai_history=self._replays_openai_history, structured_output=self._structured_output, + reasoning_summary_streamed=self._reasoning_summary_streamed, ) ) if self._structured_output and _is_message_item_done(chunk): self._first_message_item_done = True + if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"): + self._reasoning_summary_streamed = True return self._with_served_service_tier(self._with_stream_scoped_id(translated)) def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8e1791c3c92..c69d9277d65 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -3,7 +3,7 @@ import builtins import time from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace -from itertools import chain, groupby +from itertools import chain, groupby, takewhile from types import MappingProxyType from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast @@ -257,7 +257,7 @@ def apply_grounding_request_counts( class ChunkProcessor: - def __init__(self, chunks: list, messages: Sequence | None = None): + def __init__(self, chunks: list, messages: Sequence[object] | None = None): self.chunks = self._sort_chunks(chunks) self.messages = messages self.first_chunk = chunks[0] @@ -390,6 +390,30 @@ class ChunkProcessor: case _: return "assistant" + @staticmethod + def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object: + finish_reason = unreported + for chunk in chunks: + if "choices" in chunk and len(chunk["choices"]) > 0: + chunk_finish_reason = None + if hasattr(chunk["choices"][0], "finish_reason"): + chunk_finish_reason = chunk["choices"][0].finish_reason + elif "finish_reason" in chunk["choices"][0]: + chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) + if chunk_finish_reason is not None: + finish_reason = chunk_finish_reason + return finish_reason + + @staticmethod + def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices: + return Choices( + index=index, + finish_reason=ChunkProcessor._finish_reason_of_chunks( + chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop" + ), + message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""}, + ) + @staticmethod def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str: """ @@ -414,7 +438,11 @@ class ChunkProcessor: None, ) - def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse: + def build_base_response( + self, + chunks: Sequence["_BaseChunk"], + choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]], + ) -> ModelResponse: chunk = self.first_chunk id: Final = ChunkProcessor._get_chunk_id(chunks) object: Final = chunk["object"] @@ -426,16 +454,7 @@ class ChunkProcessor: service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks) role: Final = ChunkProcessor._get_role_from_chunks(chunks) - finish_reason = "stop" - for chunk in chunks: - if "choices" in chunk and len(chunk["choices"]) > 0: - chunk_finish_reason = None - if hasattr(chunk["choices"][0], "finish_reason"): - chunk_finish_reason = chunk["choices"][0].finish_reason - elif "finish_reason" in chunk["choices"][0]: - chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) - if chunk_finish_reason is not None: - finish_reason = chunk_finish_reason + finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks) # Initialize the response dictionary response = ModelResponse( @@ -455,13 +474,17 @@ class ChunkProcessor: if service_tier is not None else MappingProxyType({}) ), - "choices": [ - { - "index": 0, - "message": {"role": role, "content": ""}, - "finish_reason": finish_reason, - } - ], + "choices": ( + [ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams] + if len(choice_streams) > 1 + else [ + { + "index": 0, + "message": {"role": role, "content": ""}, + "finish_reason": finish_reason, + } + ] + ), "usage": { "prompt_tokens": 0, # Modify as needed "completion_tokens": 0, # Modify as needed @@ -1520,11 +1543,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]: return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields} +def _stream_choice_index(choice: object) -> int: + index: Final = _field_of(choice, "index") + return index if isinstance(index, int) else 0 + + +def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool: + return any( + "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] + for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks) + ) + + +def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]: + indices: Final = sorted( + frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks))) + ) + if len(indices) <= 1: + return ((0, chunks),) + return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices) + + +def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]: + return tuple( + _BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index]) + for chunk in chunks + ) + + +def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None: + choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices")) + parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk))) + return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None + + +def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]: + tool_call_chunks: Final = [ + chunk + for chunk in chunks + if any( + "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None + for choice in chunk.get("choices") or () + ) + ] + + if tool_call_chunks: + tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks) + yield "content", None + yield "tool_calls", tool_calls_list + + function_call_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "function_call" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["function_call"] is not None + ] + + if function_call_chunks: + yield "content", None + yield "function_call", processor.get_combined_function_call_content(function_call_chunks) + + content_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "content" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["content"] is not None + ] + + if content_chunks: + yield "content", processor.get_combined_content(content_chunks) + + thinking_blocks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "thinking_blocks" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["thinking_blocks"] is not None + ] + + if thinking_blocks: + yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks) + + reasoning_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "reasoning_content" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["reasoning_content"] is not None + ] + + if reasoning_chunks: + yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks) + + typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream + reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks)) + if reasoning_items: + yield "reasoning_items", list(reasoning_items) + + annotation_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "annotations" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["annotations"] is not None + ] + + message_annotations, provider_annotations = ( + partition_annotations( + list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks)) + ) + if annotation_chunks + else (None, ()) + ) + if message_annotations is not None: + yield "annotations", message_annotations + + audio_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "audio" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["audio"] is not None + ] + + if audio_chunks: + yield "audio", processor.get_combined_audio_content(audio_chunks) + + # Handle image chunks from models like gemini-2.5-flash-image + # See: https://github.com/BerriAI/litellm/issues/19478 + image_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "images" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["images"] is not None + ] + + if image_chunks: + # Images come complete in a single chunk, collect all images from all chunks + all_images: Final = [] + for chunk in image_chunks: + all_images.extend(chunk["choices"][0]["delta"]["images"]) + yield "images", all_images + + # Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations) + # See: https://github.com/BerriAI/litellm/issues/17737 + provider_specific_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "provider_specific_fields" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None + ] + + provider_field_dicts: Final = tuple( + fields + for chunk in provider_specific_chunks + for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],) + if isinstance(fields, dict) + ) + streamed_citations: Final = tuple( + fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None + ) + citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {} + provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {} + combined_provider_fields: Final = { + key: value + for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts) + for key, value in fields.items() + if key != "citation" + } + + if combined_provider_fields: + yield "provider_specific_fields", combined_provider_fields + + def stream_chunk_builder( chunks: list, messages: Sequence | None = None, - start_time=None, - end_time=None, + start_time: object = None, + end_time: object = None, logging_obj: Optional["Logging"] = None, count_prompt_tokens: Callable[[], int] | None = None, ) -> ModelResponse | TextCompletionResponse | None: @@ -1561,26 +1761,15 @@ def stream_chunk_builder( usage_model: Final = usage_tokenizer_model( model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None ) - # Initialize the response dictionary - response: Final = processor.build_base_response(chunks) + choice_streams: Final = _choice_streams(chunks) + response: Final = processor.build_base_response(chunks, choice_streams) + rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams))) - # Fast path for the common text-only streaming case: - # avoid repeated multi-pass list scans over chunks. - simple_content_parts: Final[list[str]] = [] - is_simple_text_stream = True - for chunk in chunks: - if not chunk.get("choices"): - continue - - if (part := _simple_text_part(chunk["choices"])) is None: - is_simple_text_stream = False - break - if part: - simple_content_parts.append(part) - - if is_simple_text_stream: - if simple_content_parts: - response["choices"][0]["message"]["content"] = "".join(simple_content_parts) + simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams) + if None not in simple_texts: + for (choice, _), simple_text in zip(rebuilt_choices, simple_texts): + if simple_text: + choice.message.content = simple_text completion_output = get_content_from_model_response(response) usage = processor.calculate_usage( chunks=chunks, @@ -1619,165 +1808,9 @@ def stream_chunk_builder( processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response - tool_call_chunks: Final = [ - chunk - for chunk in chunks - if any( - "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None - for choice in chunk.get("choices") or () - ) - ] - - if len(tool_call_chunks) > 0: - tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks) - _choice = response.choices[0] - _choice.message.content = None - _choice.message.tool_calls = tool_calls_list - - function_call_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "function_call" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["function_call"] is not None - ] - - if len(function_call_chunks) > 0: - _choice = response.choices[0] - _choice.message.content = None - _choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks) - - content_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "content" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["content"] is not None - ] - - if len(content_chunks) > 0: - response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks) - - thinking_blocks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "thinking_blocks" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["thinking_blocks"] is not None - ] - - if len(thinking_blocks) > 0: - response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content( - thinking_blocks - ) - - reasoning_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "reasoning_content" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["reasoning_content"] is not None - ] - - if len(reasoning_chunks) > 0: - response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content( - reasoning_chunks - ) - - typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream - reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks)) - if reasoning_items: - response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items) - - annotation_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "annotations" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["annotations"] is not None - ] - - if len(annotation_chunks) > 0: - # Merge annotations from ALL chunks — providers may spread - # them across multiple streaming chunks or send them only in - # the final chunk. - all_annotations: Final[list[object]] = [] - for ac in annotation_chunks: - all_annotations.extend(ac["choices"][0]["delta"]["annotations"]) - message_annotations, provider_annotations = partition_annotations(all_annotations) - if message_annotations is not None: - response["choices"][0]["message"]["annotations"] = message_annotations - if provider_annotations: - response.choices[0].message.provider_specific_fields = { - **(response.choices[0].message.provider_specific_fields or {}), - "annotations": list(provider_annotations), - } - - audio_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "audio" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["audio"] is not None - ] - - if len(audio_chunks) > 0: - _choice = response.choices[0] - _choice.message.audio = processor.get_combined_audio_content(audio_chunks) - - # Handle image chunks from models like gemini-2.5-flash-image - # See: https://github.com/BerriAI/litellm/issues/19478 - image_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "images" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["images"] is not None - ] - - if len(image_chunks) > 0: - # Images come complete in a single chunk, collect all images from all chunks - all_images: Final = [] - for chunk in image_chunks: - all_images.extend(chunk["choices"][0]["delta"]["images"]) - response["choices"][0]["message"]["images"] = all_images - - # Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations) - # See: https://github.com/BerriAI/litellm/issues/17737 - provider_specific_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "provider_specific_fields" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None - ] - - if len(provider_specific_chunks) > 0: - provider_field_dicts: Final = tuple( - fields - for chunk in provider_specific_chunks - for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],) - if isinstance(fields, dict) - ) - streamed_citations: Final = tuple( - fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None - ) - citation_fields: Final = ( - {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {} - ) - combined_provider_fields: Final = { - key: value - for fields in (citation_fields, *provider_field_dicts) - for key, value in fields.items() - if key != "citation" - } - - if combined_provider_fields: - _choice = response.choices[0] - _choice.message.provider_specific_fields = { - **(_choice.message.provider_specific_fields or {}), - **combined_provider_fields, - } + for choice, choice_chunks in rebuilt_choices: + for field, value in _assembled_message_fields(processor, choice_chunks): + choice.message[field] = value completion_output = get_content_from_model_response(response) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3070bedd30d..5c5204ef2f4 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): responses_so_far: Sequence["ModelResponseStream"], litellm_logging_obj: "LiteLLMLoggingObj | None", ) -> "ModelResponse": - """``stream_chunk_builder`` folds every choice of a stream into one index-0 - choice, so the stream is rebuilt one choice index at a time (every chunk - kept, its choices narrowed to that index, so usage-only chunks still - count) and the rebuilt choices are stitched into one response, each - carrying the index the stream gave it.""" choice_indices: Final = tuple( sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices)) ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index 45e785fd53c..3d555707159 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -22,6 +22,7 @@ import os from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass, replace from datetime import datetime +from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -120,6 +121,11 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" +def _streamed_choice_index(choice: object) -> int: + index: Final[object] = getattr(choice, "index", None) + return index if isinstance(index, int) else 0 + + class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): """ Cisco AI Defense guardrail integration. @@ -585,29 +591,44 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @classmethod def _extract_streaming_chunk_scan_text(cls, chunks: Sequence[object]) -> str: + streamed_choices: Final = tuple( + choice + for choice in chain.from_iterable(getattr(chunk, "choices", None) or () for chunk in chunks) + if getattr(choice, "delta", None) is not None + ) + choice_indices: Final = sorted(frozenset(map(_streamed_choice_index, streamed_choices))) + choice_texts: Final = ( + cls._extract_streamed_deltas_scan_text( + tuple( + getattr(choice, "delta", None) + for choice in streamed_choices + if _streamed_choice_index(choice) == index + ) + ) + for index in choice_indices + ) + return " ".join(text for text in choice_texts if text) + + @classmethod + def _extract_streamed_deltas_scan_text(cls, deltas: Sequence[object]) -> str: original_text = "" argument_text = "" - for chunk in chunks: - choices = getattr(chunk, "choices", None) or [] - for c in choices: - delta: object | None = getattr(c, "delta", None) - if delta is None: - continue - text = getattr(delta, "content", None) - if isinstance(text, str): - original_text += text - reasoning_text = " ".join(cls._extract_message_reasoning_parts(delta)) - if reasoning_text: - original_text += reasoning_text - for tc in getattr(delta, "tool_calls", None) or []: - args = cls._extract_tool_call_arguments(tc) - if args: - argument_text += args - fc: object | None = getattr(delta, "function_call", None) - if fc is not None: - args = cls._extract_function_call_arguments(fc) - if args: - argument_text += args + for delta in deltas: + text = getattr(delta, "content", None) + if isinstance(text, str): + original_text += text + reasoning_text = " ".join(cls._extract_message_reasoning_parts(delta)) + if reasoning_text: + original_text += reasoning_text + for tc in getattr(delta, "tool_calls", None) or []: + args = cls._extract_tool_call_arguments(tc) + if args: + argument_text += args + fc: object | None = getattr(delta, "function_call", None) + if fc is not None: + args = cls._extract_function_call_arguments(fc) + if args: + argument_text += args return " ".join(part for part in (original_text, argument_text) if part) # ------------------------------------------------------------------ diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index 85f476ecd62..d69fe4068e0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -246,20 +246,38 @@ class NomaGuardrail(CustomGuardrail): ) -> str | None: """Shared logic for processing LLM response checks""" - start_time: Final = datetime.now() - extra_data: Final = self.get_guardrail_dynamic_request_body_params(request_data) - if not isinstance(response, litellm.ModelResponse): return None - content = None - for choice in response.choices: - if isinstance(choice, litellm.Choices) and choice.message.content: - content = choice.message.content - break + checked_contents: Final = tuple( + [ + await self._process_choice_check(request_data, response, position, content, user_auth, event_type) + for position, content in self._choice_texts(response) + ] + ) + return next(iter(checked_contents), None) - if not content or not isinstance(content, str): - return None + @staticmethod + def _choice_texts(response: litellm.ModelResponse) -> tuple[tuple[int, str], ...]: + return tuple( + (position, choice.message.content) + for position, choice in enumerate(response.choices) + if isinstance(choice, litellm.Choices) + and isinstance(choice.message.content, str) + and choice.message.content + ) + + async def _process_choice_check( + self, + request_data: dict[str, object], + response: litellm.ModelResponse, + position: int, + content: str, + user_auth: UserAPIKeyAuth, + event_type: GuardrailEventHooks | None, + ) -> str: + start_time: Final = datetime.now() + extra_data: Final = self.get_guardrail_dynamic_request_body_params(request_data) payload: Final = { "input": [ @@ -306,7 +324,7 @@ class NomaGuardrail(CustomGuardrail): anonymized_content: Final = self._extract_anonymized_content(response_json, ASSISTANT_ROLE) if anonymized_content: # Replace the LLM response content with anonymized version - self._replace_llm_response_content(response, anonymized_content) + self._replace_llm_response_content(response, anonymized_content, position) verbose_proxy_logger.debug("Noma guardrail anonymized LLM response: %s", anonymized_content) return anonymized_content @@ -486,21 +504,14 @@ class NomaGuardrail(CustomGuardrail): messages[i]["content"] = anonymized_content break - def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str): - """ - Replace the LLM response content with anonymized version. - - Args: - response: The original LLM response - anonymized_content: The anonymized content to replace with - """ + def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str, position: int): + """Replace the content of the choice at ``position`` with its anonymized version.""" if not isinstance(response, litellm.ModelResponse): return - # Replace content in all choices - for choice in response.choices: - if isinstance(choice, litellm.Choices) and choice.message.content: - choice.message.content = anonymized_content + choice: Final = response.choices[position] + if isinstance(choice, litellm.Choices): + choice.message.content = anonymized_content async def _check_user_message_background( self, @@ -815,6 +826,7 @@ class NomaGuardrail(CustomGuardrail): ) if isinstance(assembled_model_response, ModelResponse): + texts_before_check: Final = self._choice_texts(assembled_model_response) try: processed_response: Final = await self._check_llm_response( request_data, @@ -828,7 +840,11 @@ class NomaGuardrail(CustomGuardrail): if self.block_failures: raise verbose_proxy_logger.error("Noma streaming post-call hook failed: %s", e) - for chunk in all_chunks: + if self._choice_texts(assembled_model_response) == texts_before_check: + for chunk in all_chunks: + yield chunk + return + async for chunk in MockResponseIterator(model_response=assembled_model_response): yield chunk return diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index fa92a0c90e0..e8c094c0c37 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -36,6 +36,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( ) from litellm.types.utils import ( CallTypesLiteral, + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, Function, @@ -744,6 +745,14 @@ class ToolPermissionGuardrail(CustomGuardrail): return ToolResult(tool_use_id=tool_call.id, content=error_message, is_error=True) + @staticmethod + def _judged_tool_call( + tool_call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, + ) -> tuple[str | None, ...]: + if isinstance(tool_call, ChatCompletionMessageCustomToolCall): + return (tool_call.id, tool_call.type, tool_call.custom.name, tool_call.custom.input) + return (tool_call.id, tool_call.type, tool_call.function.name, tool_call.function.arguments) + def _modify_response_with_permission_errors( self, response: ModelResponse, @@ -761,11 +770,10 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tools)) - # Create a mapping of tool_use_id to error result - error_results: Final = {} - for tool_use, error in denied_tools: - error_result = self._create_permission_error_result(tool_use, error) - error_results[tool_use.id] = error_result + error_results: Final = { + self._judged_tool_call(tool_use): self._create_permission_error_result(tool_use, error) + for tool_use, error in denied_tools + } # Modify the response content for choice_index, choice in enumerate(response.choices): @@ -775,9 +783,8 @@ class ToolPermissionGuardrail(CustomGuardrail): # Rewrite tool_calls for tool_call in choice.message.tool_calls or []: - tool_call_id = tool_call.id - if tool_call_id in error_results: - error_result = error_results[tool_call_id] + error_result = error_results.get(self._judged_tool_call(tool_call)) + if error_result is not None: error_messages.append(error_result.content) else: filtered_tool_calls.append(tool_call) @@ -788,7 +795,7 @@ class ToolPermissionGuardrail(CustomGuardrail): getattr(choice.message, "function_call", None), choice_index ) if legacy_tool_call is not None: - legacy_error_result = error_results.get(legacy_tool_call.id) + legacy_error_result = error_results.get(self._judged_tool_call(legacy_tool_call)) if legacy_error_result is not None: choice.message.function_call = None error_messages.append(legacy_error_result.content) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c01d07fb5c3..c88a8c1cc28 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -35,10 +35,7 @@ from litellm.types.llms.openai import ToolParam as ResponsesToolParam from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.utils import ( CallTypes, - ChatCompletionMessageCustomToolCall, - ChatCompletionMessageToolCall, Choices, - Message, ModelResponse, StandardLoggingMCPToolCall, ) @@ -587,29 +584,18 @@ class LiteLLM_Proxy_MCP_Handler: _extract_tool_calls_from_response = extract_tool_calls_from_response + @staticmethod + def _continued_choice(response: ModelResponse) -> Choices | None: + choices: Final = tuple(choice for choice in response.choices if isinstance(choice, Choices)) + return next((choice for choice in choices if choice.message.tool_calls), next(iter(choices), None)) + @staticmethod def extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]: - """Extract tool calls from a chat completion response.""" - tool_calls: Final[list[object]] = [] - - try: - for choice in response.choices: - message: Message | None = getattr(choice, "message", None) - if message is None: - continue - tool_call_entries: ( - Sequence[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None - ) = getattr(message, "tool_calls", None) - if tool_call_entries: - for tool_call in tool_call_entries: - if hasattr(tool_call, "model_dump"): - tool_calls.append(tool_call.model_dump()) - else: - tool_calls.append(tool_call) - except Exception: - verbose_logger.exception("Failed to extract tool calls from chat completion response") - - return tool_calls + """Extract the tool calls of the first choice that made any, since the follow-up turn continues it alone.""" + continued_choice: Final = LiteLLM_Proxy_MCP_Handler._continued_choice(response) + if continued_choice is None: + return [] + return [tool_call.model_dump() for tool_call in continued_choice.message.tool_calls or ()] _extract_tool_calls_from_chat_response = extract_tool_calls_from_chat_response @@ -1018,9 +1004,9 @@ class LiteLLM_Proxy_MCP_Handler: message_to_append: dict | None = None try: - first_choice: Final = response.choices[0] - if isinstance(first_choice, Choices) and getattr(first_choice, "message", None): - message_to_append = first_choice.message.model_dump(exclude_none=True) + continued_choice: Final = LiteLLM_Proxy_MCP_Handler._continued_choice(response) + if continued_choice is not None: + message_to_append = continued_choice.message.model_dump(exclude_none=True) # Ensure tool_calls have arguments field (required by OpenAI API) if message_to_append.get("tool_calls"): for tool_call in message_to_append["tool_calls"]: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index d59f59703ad..4e6ed16e360 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -418,6 +418,45 @@ def test_openai_responses_chunk_parser_reasoning_summary(): assert delta.function_call is None +def test_bridged_reasoning_summary_parts_rebuild_into_the_one_choice_they_streamed_on(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + events = ( + {"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0}, + { + "type": "response.reasoning_summary_text.delta", + "delta": "First part.", + "output_index": 0, + "summary_index": 0, + }, + {"type": "response.reasoning_summary_part.done", "output_index": 0, "summary_index": 0}, + {"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 1}, + { + "type": "response.reasoning_summary_text.delta", + "delta": "Second part.", + "output_index": 0, + "summary_index": 1, + }, + {"type": "response.reasoning_summary_part.added", "output_index": 1, "summary_index": 0}, + { + "type": "response.reasoning_summary_text.delta", + "delta": "Next item.", + "output_index": 1, + "summary_index": 0, + }, + {"type": "response.output_text.delta", "delta": "Answer", "output_index": 2, "content_index": 0}, + ) + + rebuilt = litellm.stream_chunk_builder(chunks=[iterator.chunk_parser(event) for event in events]) + + assert [(choice.index, choice.message.reasoning_content, choice.message.content) for choice in rebuilt.choices] == [ + (0, "First part. Second part. Next item.", "Answer") + ] + + def test_chunk_parser_string_output_text_delta_produces_text(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( OpenAiResponsesToChatCompletionStreamIterator, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index dd1f495b0d0..28587bf4749 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1304,12 +1304,118 @@ def test_stream_chunk_builder_keeps_each_choices_tool_call_arguments_apart(): ] ) - assert [(tool_call.id, tool_call.function.arguments) for tool_call in response.choices[0].message.tool_calls] == [ - ("call_a", '{"fruit": "persimmon"}'), - ("call_b", '{"fruit": "durian"}'), + assert [ + (choice.index, [(tool_call.id, tool_call.function.arguments) for tool_call in choice.message.tool_calls]) + for choice in response.choices + ] == [ + (0, [("call_a", '{"fruit": "persimmon"}')]), + (1, [("call_b", '{"fruit": "durian"}')]), ] +def test_stream_chunk_builder_rebuilds_each_choice_of_a_multi_choice_text_stream_on_its_own(): + def chunk(choice_index: int, content: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + object="chat.completion.chunk", + created=1234567890, + model="gpt-4.1-mini", + choices=[StreamingChoices(index=choice_index, delta=Delta(content=content), finish_reason=finish_reason)], + ) + + response = stream_chunk_builder( + chunks=[ + chunk(0, "The persim"), + chunk(1, "The"), + chunk(1, " persimmon is"), + chunk(0, "mon is ripe.", finish_reason="stop"), + chunk(1, " ripe and", finish_reason="length"), + ] + ) + + assert [(choice.index, choice.message.content, choice.finish_reason) for choice in response.choices] == [ + (0, "The persimmon is ripe.", "stop"), + (1, "The persimmon is ripe and", "length"), + ] + + +def test_stream_chunk_builder_gives_a_choice_without_a_finish_chunk_the_finish_reason_its_content_implies(): + def chunk(choice_index: int, delta: Delta, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + object="chat.completion.chunk", + created=1234567890, + model="gpt-4.1-mini", + choices=[StreamingChoices(index=choice_index, delta=delta, finish_reason=finish_reason)], + ) + + def tool_call(call_id: str) -> list[ChatCompletionDeltaToolCall]: + return [ + ChatCompletionDeltaToolCall( + id=call_id, index=0, type="function", function=Function(name="lookup_fruit", arguments="{}") + ) + ] + + response = stream_chunk_builder( + chunks=[ + chunk(0, Delta(tool_calls=tool_call("call_a"))), + chunk(1, Delta(tool_calls=tool_call("call_b"))), + chunk(2, Delta(content="Kiwis are fuzzy.")), + chunk(0, Delta(), finish_reason="tool_calls"), + ] + ) + + assert [(choice.index, choice.finish_reason) for choice in response.choices] == [ + (0, "tool_calls"), + (1, "tool_calls"), + (2, "stop"), + ] + + +def test_stream_chunk_builder_keeps_each_choices_function_call_annotations_and_images_on_that_choice(): + def chunk(choice_index: int, delta: Delta) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + object="chat.completion.chunk", + created=1234567890, + model="gpt-4.1-mini", + choices=[StreamingChoices(index=choice_index, delta=delta, finish_reason=None)], + ) + + citation: Final = { + "type": "url_citation", + "url_citation": {"start_index": 0, "end_index": 5, "url": "https://example.com/kiwi", "title": "Kiwi"}, + } + image: Final = {"type": "image_url", "index": 0, "image_url": {"url": "data:image/png;base64,a2l3aQ=="}} + + response = stream_chunk_builder( + chunks=[ + chunk(0, Delta(function_call={"name": "lookup_fruit", "arguments": '{"fruit": '})), + chunk(1, Delta(content="Kiwis are")), + chunk(0, Delta(function_call={"arguments": '"kiwi"}'})), + chunk(1, Delta(content=" fuzzy.", annotations=[citation])), + chunk(1, Delta(images=[image])), + ] + ) + + function_choice, cited_choice = response.choices + assert ( + function_choice.index, + function_choice.message.content, + function_choice.message.function_call.name, + function_choice.message.function_call.arguments, + getattr(function_choice.message, "annotations", None), + getattr(function_choice.message, "images", None), + ) == (0, None, "lookup_fruit", '{"fruit": "kiwi"}', None, None) + assert ( + cited_choice.index, + cited_choice.message.content, + cited_choice.message.function_call, + cited_choice.message.annotations, + cited_choice.message.images, + ) == (1, "Kiwis are fuzzy.", None, [citation], [image]) + + def test_get_combined_tool_content_joins_many_dict_shaped_argument_fragments_in_order(): processor = ChunkProcessor.__new__(ChunkProcessor) first_fragments = [f"a{i};" for i in range(300)] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py index 4d1f254aef9..5ce78fd05d1 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py @@ -2108,6 +2108,62 @@ class TestCiscoAIDefenseStreamingBypass: f"Original: {chunks!r}, received: {received!r}" ) + @pytest.mark.asyncio + async def test_streaming_safe_n2_response_inspects_each_choice_and_yields_original_chunks(self): + g = _make_guardrail(event_hook=["pre_call", "post_call"]) + chunks = [ + ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), finish_reason=finish_reason, index=index)], + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + ) + for index, part, finish_reason in ( + (0, "The persim", None), + (1, "The", None), + (0, "mon is ripe.", "stop"), + (1, " persimmon is ripe.", "stop"), + ) + ] + + received, post_mock = await _streaming_setup(g, chunks, cisco_response=_safe_response()) + + inspected = post_mock.call_args.kwargs["json"]["messages"] + assert [m["content"] for m in inspected if m["role"] == "assistant"] == [ + "The persimmon is ripe.", + "The persimmon is ripe.", + ] + assert received == chunks + + @pytest.mark.asyncio + async def test_streaming_safe_n2_response_with_a_function_call_choice_yields_original_chunks(self): + g = _make_guardrail(event_hook=["pre_call", "post_call"]) + chunks = [ + ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=delta, finish_reason=None, index=index)], + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + ) + for index, delta in ( + (0, Delta(function_call={"name": "lookup_fruit", "arguments": '{"fruit": '})), + (1, Delta(content="Kiwis are")), + (0, Delta(function_call={"arguments": '"kiwi"}'})), + (1, Delta(content=" fuzzy.")), + ) + ] + + received, post_mock = await _streaming_setup(g, chunks, cisco_response=_safe_response()) + + inspected = post_mock.call_args.kwargs["json"]["messages"] + assert [m["content"] for m in inspected if m["role"] == "assistant"] == [ + '{"fruit": "kiwi"}', + "Kiwis are fuzzy.", + ] + assert received == chunks + @pytest.mark.asyncio async def test_streaming_redact_does_not_replay_tool_call_arguments(self): g = _make_guardrail( diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py index ee369f72f01..574550e1bdd 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py @@ -1,4 +1,5 @@ import copy +from itertools import chain import os from typing import cast from unittest.mock import AsyncMock, MagicMock, patch @@ -18,7 +19,7 @@ import litellm.proxy.guardrails.guardrail_hooks.noma.noma as noma_legacy_module from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import Choices, Message +from litellm.types.utils import Choices, Delta, Message, ModelResponseStream, StreamingChoices @pytest.fixture @@ -1848,12 +1849,207 @@ class TestNomaAnonymizationLogic: ) anonymize_guardrail._replace_llm_response_content( - response, "Your email is *******" + response, "Your email is *******", 0 ) assert response.choices[0].message.content == "Your email is *******" +def _noma_reply_per_text(verdict_for_text): + def reply(*args, **kwargs): + response = MagicMock() + response.json.return_value = verdict_for_text(kwargs["json"]["input"][0]["content"][0]["text"]) + response.raise_for_status = MagicMock() + return response + + return reply + + +class TestNomaMultiChoiceResponses: + @pytest.mark.asyncio + async def test_streamed_two_choice_reply_is_blocked_when_only_the_second_choice_is_flagged(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + monitor_mode=False, + block_failures=True, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + scanned_texts = [] + + def verdict_for_text(text): + scanned_texts.append(text) + return {"aggregatedScanResult": "launch code" in text, "scanResult": []} + + async def upstream(): + for index, part in ((0, "Kiwis are fuzzy."), (1, "The launch code is 0000.")): + yield ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), finish_reason="stop", index=index)], + created=1234567890, + model="gpt-4.1-mini", + object="chat.completion.chunk", + ) + + with patch.object(guardrail.async_handler, "post", side_effect=_noma_reply_per_text(verdict_for_text)): + with pytest.raises(NomaBlockedMessage): + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=upstream(), + request_data={"litellm_call_id": "call-1", "metadata": {}}, + ): + pass + + assert scanned_texts == ["Kiwis are fuzzy.", "The launch code is 0000."] + + @pytest.mark.asyncio + async def test_each_choice_gets_its_own_anonymized_text(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + anonymize_input=True, + monitor_mode=False, + block_failures=True, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + + def verdict_for_text(text): + return { + "aggregatedScanResult": False, + "scanResult": [ + { + "role": "assistant", + "type": "message", + "results": { + "anonymizedContent": {"anonymized": text.replace("test@example.com", "*******")}, + "sensitiveData": {"PII": {"result": False, "status": "SUCCESS"}}, + }, + } + ], + } + + response = ModelResponse( + id="resp_1", + choices=[ + Choices(index=0, finish_reason="stop", message=Message(role="assistant", content="Mail test@example.com")), + Choices( + index=1, finish_reason="stop", message=Message(role="assistant", content="Write test@example.com") + ), + ], + model="gpt-4.1-mini", + ) + + with patch.object(guardrail.async_handler, "post", side_effect=_noma_reply_per_text(verdict_for_text)): + await guardrail.async_post_call_success_hook( + data={"litellm_call_id": "call-1", "metadata": {}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=response, + ) + + assert [choice.message.content for choice in response.choices] == ["Mail *******", "Write *******"] + + @pytest.mark.asyncio + async def test_fail_open_stream_keeps_redactions_made_before_a_later_choice_fails(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + anonymize_input=True, + monitor_mode=False, + block_failures=False, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + + def verdict_for_text(text): + if text.startswith("Write"): + raise httpx.ConnectError("Noma is unreachable") + return { + "aggregatedScanResult": False, + "scanResult": [ + { + "role": "assistant", + "type": "message", + "results": { + "anonymizedContent": {"anonymized": text.replace("test@example.com", "*******")}, + "sensitiveData": {"PII": {"result": False, "status": "SUCCESS"}}, + }, + } + ], + } + + async def upstream(): + for index, part in ((0, "Mail test@example.com"), (1, "Write back soon")): + yield ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), finish_reason="stop", index=index)], + created=1234567890, + model="gpt-4.1-mini", + object="chat.completion.chunk", + ) + + with patch.object(guardrail.async_handler, "post", side_effect=_noma_reply_per_text(verdict_for_text)): + streamed = [ + chunk + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=upstream(), + request_data={"litellm_call_id": "call-1", "metadata": {}}, + ) + ] + + streamed_choices = tuple(chain.from_iterable(part.choices for part in streamed)) + text_by_choice = { + index: "".join(choice.delta.content or "" for choice in streamed_choices if choice.index == index) + for index in (0, 1) + } + assert text_by_choice == {0: "Mail *******", 1: "Write back soon"} + + @pytest.mark.asyncio + async def test_fail_open_stream_replays_the_original_chunks_when_nothing_was_redacted(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + anonymize_input=True, + monitor_mode=False, + block_failures=False, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + originals = [ + ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), index=0)], + created=1234567890, + model="gpt-4.1-mini", + object="chat.completion.chunk", + ) + for part in ("Kiwis", " are", " fuzzy.") + ] + + async def upstream(): + for chunk in originals: + yield chunk + + with patch.object(guardrail.async_handler, "post", side_effect=httpx.ConnectError("Noma is unreachable")): + streamed = [ + chunk + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=upstream(), + request_data={"litellm_call_id": "call-1", "metadata": {}}, + ) + ] + + assert len(streamed) == len(originals) + assert all(sent is original for sent, original in zip(streamed, originals)) + + class TestNomaAnonymizationFlow: """Test full anonymization flow with real Noma response objects""" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 2ee4c8c72c0..21376068686 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -5,6 +5,7 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics) import json import logging import re +from itertools import chain from typing import Final from unittest.mock import patch @@ -25,10 +26,16 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( PermissionError, ) from litellm.types.utils import ( + ChatCompletionCustomToolCallPayload, + ChatCompletionDeltaToolCall, + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, + Delta, + Function, ModelResponse, ModelResponseStream, + StreamingChoices, ) @@ -425,6 +432,41 @@ class TestToolPermissionGuardrail: assert isinstance(choice.message.content, str) assert "berri" in choice.message.content + @pytest.mark.asyncio + async def test_rewrite_denies_only_the_choice_whose_call_is_denied_when_choices_share_a_call_id(self): + guardrail = ToolPermissionGuardrail( + guardrail_name="choice-guardrail", + rules=[{"id": "deny_bash", "tool_name": "Bash", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + response = ModelResponse( + choices=[ + Choices( + index=index, + finish_reason="tool_calls", + message={ + "tool_calls": [ + {"id": "call_shared", "type": "function", "function": {"name": name, "arguments": "{}"}} + ] + }, + ) + for index, name in enumerate(("Bash", "Read")) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["choice-guardrail"]}, user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + kept: Final = [ + [tool_call.function.name for tool_call in choice.message.tool_calls or ()] + for choice in response.choices + if isinstance(choice, Choices) + ] + assert kept == [[], ["Read"]] + @pytest.mark.asyncio async def test_async_post_call_success_hook_missing_arguments_blocks_param_rule( self, @@ -866,6 +908,46 @@ class TestToolPermissionGuardrail: f"got: {chunks[0].choices[0].delta.content!r}" ) + @pytest.mark.asyncio + async def test_async_post_call_streaming_iterator_hook_re_emits_each_choice_of_an_n2_tool_call_stream(self): + def chunk(choice_index: int, tool_call: ChatCompletionDeltaToolCall) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-n2-tools", + created=1700000000, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[StreamingChoices(index=choice_index, delta=Delta(tool_calls=[tool_call]))], + ) + + def fragment(arguments: str, name: str | None = None) -> ChatCompletionDeltaToolCall: + return ChatCompletionDeltaToolCall( + id="call_shared" if name else None, + index=0, + type="function", + function=Function(name=name, arguments=arguments), + ) + + async def _stream(): + for streamed in ( + chunk(0, fragment("", name="Bash")), + chunk(1, fragment("", name="Bash")), + chunk(0, fragment('{"command": "ls"}')), + chunk(1, fragment('{"command": "pwd"}')), + ): + yield streamed + + re_emitted: Final = [ + chunk + async for chunk in self.guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={} + ) + ] + + assert [ + (choice.index, [(call.function.name, call.function.arguments) for call in choice.delta.tool_calls]) + for choice in chain.from_iterable(streamed.choices for streamed in re_emitted) + ] == [(0, [("Bash", '{"command": "ls"}')]), (1, [("Bash", '{"command": "pwd"}')])] + def test_modify_response_with_permission_errors(self): # Setup a response with one tool_call tool_call = ChatCompletionMessageToolCall( @@ -952,6 +1034,35 @@ class TestToolPermissionGuardrail: assert "Permission denied" in choice.message.content + def test_modify_response_with_permission_errors_keeps_custom_tool_call_in_another_choice(self): + denied_call = ChatCompletionMessageToolCall( + id="call_bash", type="function", function=Function(name="Bash", arguments="{}") + ) + custom_call = ChatCompletionMessageCustomToolCall( + id="call_patch", custom=ChatCompletionCustomToolCallPayload(name="apply_patch", input="*** Begin Patch") + ) + response = ModelResponse( + choices=[ + Choices(index=0, message={"role": "assistant", "content": None, "tool_calls": [denied_call]}), + Choices(index=1, message={"role": "assistant", "content": None, "tool_calls": [custom_call]}), + ] + ) + denied_tools = [ + ( + denied_call, + PermissionError(tool_name="Bash", rule_id="deny_bash", message="Tool 'Bash' denied by rule 'deny_bash'"), + ) + ] + + self.guardrail._modify_response_with_permission_errors(response, denied_tools) + + kept = [ + [call.id for call in choice.message.tool_calls or []] + for choice in response.choices + if isinstance(choice, Choices) + ] + assert kept == [[], ["call_patch"]] + class TestToolPermissionGuardrailIntegration: """Integration tests for Tool Permission Guardrail""" diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 229d5cbe676..2f27622d77c 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -131,6 +131,86 @@ def test_extract_tool_calls_from_chat_response_handles_tool_calls(): assert tool_calls[0]["function"]["name"] == "foo" +def test_chat_follow_up_answers_only_tool_calls_in_its_assistant_turn(): + response = ModelResponse( + id="resp-n2", + choices=[ + { + "index": index, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": "foo", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + } + for index, call_id in enumerate(("call-a", "call-b")) + ], + model="gpt", + created=0, + object="chat.completion", + ) + tool_results = [ + {"tool_call_id": tool_call["id"], "name": tool_call["function"]["name"], "result": "done"} + for tool_call in LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response) + ] + + follow_up = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat( + [{"role": "user", "content": "hi"}], response, tool_results + ) + + assistant_call_ids = [tool_call["id"] for tool_call in follow_up[1]["tool_calls"]] + answered_call_ids = [message["tool_call_id"] for message in follow_up if message["role"] == "tool"] + assert answered_call_ids == assistant_call_ids == ["call-a"] + + +def test_chat_follow_up_continues_the_choice_that_called_a_tool_when_choice_0_answered(): + response = ModelResponse( + id="resp-n2-second", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "no tool needed"}, + "finish_reason": "stop", + }, + { + "index": 1, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call-b", "type": "function", "function": {"name": "foo", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + }, + ], + model="gpt", + created=0, + object="chat.completion", + ) + tool_results = [ + {"tool_call_id": tool_call["id"], "name": tool_call["function"]["name"], "result": "done"} + for tool_call in LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response) + ] + + follow_up = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat( + [{"role": "user", "content": "hi"}], response, tool_results + ) + + assistant_call_ids = [tool_call["id"] for tool_call in follow_up[1]["tool_calls"]] + answered_call_ids = [message["tool_call_id"] for message in follow_up if message["role"] == "tool"] + assert answered_call_ids == assistant_call_ids == ["call-b"] + + +def test_extract_tool_calls_from_chat_response_without_choices_finds_none(): + response = ModelResponse(id="resp-empty", choices=[], model="gpt", created=0, object="chat.completion") + + assert LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response) == [] + + def test_create_follow_up_messages_for_chat_appends_tool_results(): original_messages = [{"role": "user", "content": "hi"}] response = ModelResponse( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 11104cf7bca..571f890a4e6 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -2831,9 +2831,10 @@ def test_stream_chunk_builder_keeps_tool_calls_carried_only_by_a_later_choice_of response = stream_chunk_builder(chunks=chunks) - tool_calls = response.choices[0].message.tool_calls - assert tool_calls is not None - assert [(call.id, call.function.name, call.function.arguments) for call in tool_calls] == [ + text_choice, tool_choice = response.choices + assert (text_choice.index, text_choice.message.content, text_choice.message.tool_calls) == (0, "hello world", None) + assert tool_choice.index == 1 + assert [(call.id, call.function.name, call.function.arguments) for call in tool_choice.message.tool_calls] == [ ("call_1", "lookup_fruit", '{"fruit":"kiwi"}') ] From 326f17594efb084b9d6080551147c1d12aed032e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:29:30 -0700 Subject: [PATCH 07/34] fix(batches): return real batch token counts next to OpenAI's batch usage fields (#47740) * fix(batches): read OpenAI batch usage into the batch token counts * fix(batches): serve OpenAI's batch usage shape from retrieve and the cost poller * fix(batches): keep the chat usage keys next to OpenAI's batch usage fields Batch usage now carries every chat key it had plus OpenAI's five batch fields with the same counts, so poller-billed batches keep prompt_tokens and cache-write tokens, and older proxies still read the new rows. Usage with missing or null detail counts reads its real input and output counts. * test(batches): check the batch usage counts in the e2e terminal-state test * fix(types): apply nested include and exclude to both batch usage shapes * fix(batches): log no cost for a batch retrieve that is not final yet A retrieve now reads OpenAI's batch usage into the chat counts, so a retrieve that saw the batch completed before its output file arrived priced those counts for the logging callbacks, and the final retrieve priced the whole batch again. A batch that is not final now logs no cost, as it did before. Batch usage dumps also pass exclude_unset and exclude_defaults through to the usage keys. * fix(batches): pass through every usage detail field the provider sends Batch dumps rebuilt input_tokens_details and output_tokens_details from the chat keys, which dropped any detail field past cached_tokens and reasoning_tokens that the merge base passed through. The dump now starts from the provider's stored details and only sets the two counts derived from the chat keys * fix(batches): skip sync loggers on the cost poller's own retrieve and honor exclude_none on provider details * fix(batches): keep sync logger events at $0 on poller-owned retrieves, pass unparsed provider counts through, log batch usage in chat shape * fix(batches): store only chat usage extras on batch spend rows Spend rows for batch retrieves copied OpenAI's batch fields (input_tokens, output_tokens and their details) into metadata.additional_usage_values next to the chat counts they duplicate. Build the row from the batch's chat usage, shared with the logging payload through batch_chat_usage. --- litellm/litellm_core_utils/litellm_logging.py | 19 +- .../spend_tracking/spend_tracking_utils.py | 9 +- litellm/types/utils.py | 151 +++++++++++++ tests/e2e/batches/batch_client.py | 13 +- tests/e2e/batches/test_batches_e2e.py | 7 + .../test_litellm_logging.py | 110 ++++++++- .../common_utils/test_check_batch_cost.py | 17 +- .../test_spend_tracking_utils.py | 59 +++++ tests/unit/types/test_types_utils.py | 213 ++++++++++++++++++ 9 files changed, 575 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 34c98efc22f..4dd33fa5565 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -202,6 +202,7 @@ from litellm.types.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, + batch_chat_usage, ) from litellm.types.videos.main import VideoObject from litellm.utils import get_base_model_from_metadata, print_verbose @@ -2091,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass): elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set router_model_id = hidden_params["model_id"] + if isinstance(priced_result, LiteLLMBatch) and ( + not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result) + ): + return 0.0 + # Fallback: extract router_model_id from litellm_params when not available # from the result object. ResponsesAPIResponse objects (used by /v1/responses # streaming) don't carry _hidden_params["model_id"] like ModelResponse does. @@ -3346,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass): finally: self._restore_correlation_context() + def _is_batch_poll_without_logging(self, result: object) -> bool: + litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {} + return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True + def _success_handler_body( self, result: object = None, @@ -3860,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass): ## CALCULATE COST FOR BATCH JOBS if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch): - litellm_params = self.litellm_params or {} - litellm_metadata: Final = litellm_params.get("litellm_metadata") or {} - if ( - litellm_metadata.get("batch_ignore_default_logging", False) is True - ): # polling job will query these frequently, don't spam db logs + if self._is_batch_poll_without_logging(result): return batch_cost: Final = kwargs.get("batch_cost", None) @@ -7481,7 +7487,8 @@ def get_standard_logging_object_payload( # Extract usage as a plain dict, avoiding Pydantic round-trip raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict( response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj, - combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")), + combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")) + or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None), ) usage_dict: Final = ( {**raw_usage_dict, "output_image_count": len(init_response_obj.data)} diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 634a805d72b..09b7ce7ee9a 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -87,6 +87,7 @@ from litellm.types.utils import ( AzureSpillover, CallTypes, CostBreakdown, + LiteLLMBatch, LlmProviders, StandardLoggingGuardrailInformation, StandardLoggingKubernetesPodRouting, @@ -96,6 +97,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, StandardLoggingVectorStoreRequest, VectorStoreSearchResponse, + batch_chat_usage, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -626,7 +628,12 @@ def get_logging_payload( usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) elif not is_unbilled_non_inference_call(call_type, metadata, response_obj_dict): # Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models - _usage: Final = response_obj_dict.get("usage", None) or {} + chat_usage_of_batch: Final = batch_chat_usage(response_obj) if isinstance(response_obj, LiteLLMBatch) else None + _usage: Final = ( + chat_usage_of_batch.model_dump() + if chat_usage_of_batch is not None + else response_obj_dict.get("usage", None) or {} + ) if isinstance(_usage, litellm.Usage): usage = dict(_usage) elif isinstance(_usage, dict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1055ffe7b2f..181ed5ea6db 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -20,6 +20,7 @@ from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import ( FileTypes as FileTypes, ) +from openai.types.batch_usage import BatchUsage, InputTokensDetails, OutputTokensDetails from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion from openai.types.completion_usage import ( CompletionTokensDetails, @@ -44,6 +45,7 @@ from pydantic import ( JsonValue, PrivateAttr, SkipValidation, + ValidationError, field_serializer, field_validator, ) @@ -4785,6 +4787,112 @@ class LiteLLMFineTuningJob(FineTuningJob): self._hidden_params = kwargs.get("_hidden_params", {}) +class _BatchInputTokensDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + cached_tokens: int | None = None + + +class _BatchOutputTokensDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + reasoning_tokens: int | None = None + + +class _BatchTokenCounts(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + output_tokens: int + total_tokens: int | None = None + input_tokens_details: _BatchInputTokensDetails | None = None + output_tokens_details: _BatchOutputTokensDetails | None = None + + +def _parse_batch_token_counts(value: object) -> _BatchTokenCounts | None: + if not isinstance(value, Mapping): + return None + try: + return _BatchTokenCounts.model_validate(value) + except ValidationError: + return None + + +def _prompt_details_with_cached_tokens( + details: PromptTokensDetailsWrapper | None, cached_tokens: int | None +) -> PromptTokensDetailsWrapper | None: + if not cached_tokens: + return details + return (details or PromptTokensDetailsWrapper()).model_copy(update={"cached_tokens": cached_tokens}) + + +def _completion_details_with_reasoning_tokens( + details: CompletionTokensDetailsWrapper | None, reasoning_tokens: int | None +) -> CompletionTokensDetailsWrapper | None: + if not reasoning_tokens: + return details + return (details or CompletionTokensDetailsWrapper()).model_copy(update={"reasoning_tokens": reasoning_tokens}) + + +def _usage_with_batch_token_counts(usage: Usage, counts: _BatchTokenCounts) -> Usage: + return usage.model_copy( + update={ + "prompt_tokens": counts.input_tokens, + "completion_tokens": counts.output_tokens, + "total_tokens": ( + counts.input_tokens + counts.output_tokens if counts.total_tokens is None else counts.total_tokens + ), + "prompt_tokens_details": _prompt_details_with_cached_tokens( + usage.prompt_tokens_details, + counts.input_tokens_details.cached_tokens if counts.input_tokens_details else None, + ), + "completion_tokens_details": _completion_details_with_reasoning_tokens( + usage.completion_tokens_details, + counts.output_tokens_details.reasoning_tokens if counts.output_tokens_details else None, + ), + } + ) + + +class _ProviderBatchUsageDetails(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + input_tokens_details: Mapping[str, JsonValue] | None = None + output_tokens_details: Mapping[str, JsonValue] | None = None + + +def _provider_batch_usage_details(usage: Usage) -> _ProviderBatchUsageDetails: + try: + return _ProviderBatchUsageDetails.model_validate(usage) + except ValidationError: + return _ProviderBatchUsageDetails() + + +def _has_unparsed_batch_token_counts(usage: Usage) -> bool: + provider_fields: Final = usage.model_extra or {} + sent_batch_counts: Final = "input_tokens" in provider_fields or "output_tokens" in provider_fields + return sent_batch_counts and _parse_batch_token_counts(provider_fields) is None + + +def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage: + provider_details: Final = _provider_batch_usage_details(usage) + cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None + reasoning_tokens: Final = ( + usage.completion_tokens_details.reasoning_tokens if usage.completion_tokens_details else None + ) + return BatchUsage( + input_tokens=usage.prompt_tokens, + input_tokens_details=InputTokensDetails.model_validate( + {**(provider_details.input_tokens_details or {}), "cached_tokens": cached_tokens or 0} + ), + output_tokens=usage.completion_tokens, + output_tokens_details=OutputTokensDetails.model_validate( + {**(provider_details.output_tokens_details or {}), "reasoning_tokens": reasoning_tokens or 0} + ), + total_tokens=usage.total_tokens, + ) + + class LiteLLMBatch(Batch): _hidden_params: dict = {} @@ -4798,6 +4906,40 @@ class LiteLLMBatch(Batch): usage: Usage | None = None + @field_validator("usage", mode="before") + @classmethod + def _usage_from_batch_token_counts(cls, value: object) -> object: + counts: Final = _parse_batch_token_counts(value) + return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts) + + @field_serializer("usage") + def _serialize_usage_with_batch_fields( + self, usage: Usage | None, info: FieldSerializationInfo + ) -> Mapping[str, object] | None: + if not isinstance(usage, Usage): + return usage + chat_fields: Final = usage.model_dump( + mode=info.mode, + include=info.include, + exclude=info.exclude, + exclude_unset=info.exclude_unset, + exclude_defaults=info.exclude_defaults, + exclude_none=info.exclude_none, + ) + if _has_unparsed_batch_token_counts(usage): + return chat_fields + return { + **chat_fields, + **_batch_usage_from_chat_usage(usage).model_dump( + mode=info.mode, + include=info.include, + exclude=info.exclude, + exclude_unset=info.exclude_unset, + exclude_defaults=info.exclude_defaults, + exclude_none=info.exclude_none, + ), + } + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) @@ -4818,6 +4960,15 @@ class LiteLLMBatch(Batch): return self.dict() +_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields) + + +def batch_chat_usage(batch: LiteLLMBatch) -> Usage | None: + if batch.usage is None: + return None + return Usage.model_validate(batch.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS))) + + class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase): # Events are already well-formed provider dicts. Validating them against the # OpenAIRealtimeEvents union makes Pydantic try every member per event, which diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index b02b09e557b..598593c1fcd 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -15,7 +15,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Final, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from e2e_metadata import step from proxy_client import ProxyClient @@ -77,6 +77,16 @@ class BatchErrorList(BaseModel): data: list[BatchErrorItem] = [] +class BatchUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None + input_tokens: int | None = None + output_tokens: int | None = None + + class BatchObject(BaseModel): id: str object: str | None = None @@ -90,6 +100,7 @@ class BatchObject(BaseModel): completion_window: str | None = None created_at: int | None = None model: str | None = None + usage: BatchUsage | None = None class BatchList(BaseModel): diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 54bcb80ed24..b7a52da5902 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1893,6 +1893,13 @@ class TestBatchTerminalState: f"listed-completed marker retrieved as {fetched.status!r}" ) assert fetched.output_file_id, "completed batch has no output_file_id" + usage: Final = fetched.usage + assert usage is not None and usage.input_tokens and usage.output_tokens, ( + f"completed batch has no batch token counts: {usage!r}" + ) + assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), ( + f"chat token counts disagree with the batch token counts: {usage!r}" + ) downloaded = client.proxy.transport.download( f"/v1/files/{fetched.output_file_id}/content", diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index a6d99a5ba55..c320cb36d73 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -1629,22 +1629,33 @@ class TestRetrieveBatchPricesOnlyFinalBatches: litellm_call_id="batch-call-2", function_id="f", ) - obj.custom_llm_provider = "openai" + obj.update_environment_variables( + model="gpt-5.6-luna", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) return obj @staticmethod def _batch(status: str, output_file_id: str | None): from litellm.types.utils import LiteLLMBatch - return LiteLLMBatch( - id="batch_6a9c99e185588190877d391f8b9d7f8a", - completion_window="24h", - created_at=1, - endpoint="/v1/chat/completions", - input_file_id="file-in", - object="batch", - status="validating", - output_file_id=output_file_id, + return LiteLLMBatch.model_validate( + { + "id": "batch_6a9c99e185588190877d391f8b9d7f8a", + "completion_window": "24h", + "created_at": 1, + "endpoint": "/v1/chat/completions", + "input_file_id": "file-in", + "object": "batch", + "status": "validating", + "output_file_id": output_file_id, + "usage": { + "input_tokens": 29, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 48, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 77, + }, + } ).model_copy(update={"status": status}) @pytest.mark.asyncio @@ -1658,11 +1669,12 @@ class TestRetrieveBatchPricesOnlyFinalBatches: handle_completed_batch = AsyncMock() monkeypatch.setattr(logging_module, "handle_completed_batch", handle_completed_batch) batch = self._batch(status, output_file_id) + logging_obj = self._logging_obj() - await self._logging_obj()._async_success_handler_body(result=batch, start_time=None, end_time=None) + await logging_obj._async_success_handler_body(result=batch, start_time=None, end_time=None) handle_completed_batch.assert_not_awaited() - assert "response_cost" not in batch._hidden_params + assert logging_obj.model_call_details["response_cost"] == 0.0 @pytest.mark.asyncio async def test_completed_batch_with_output_is_priced(self, monkeypatch) -> None: @@ -1689,6 +1701,80 @@ class TestRetrieveBatchPricesOnlyFinalBatches: assert batch.usage is not None assert batch.usage.total_tokens == 35 + @pytest.mark.parametrize("batch_ignore_default_logging", [True, False]) + def test_sync_logger_gets_a_free_event_for_the_cost_poller_retrieve(self, batch_ignore_default_logging: bool) -> None: + sync_events: list[object] = [] + + class SyncOnlyLogger(CustomLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: dt_object, + end_time: dt_object, + ) -> None: + sync_events.append(kwargs["response_cost"]) + + logging_obj = LitellmLogging( + model="gpt-5.6-luna", + messages=[{"role": "user", "content": "Hey"}], + stream=False, + call_type="aretrieve_batch", + start_time=time.time(), + litellm_call_id="batch-call-3", + function_id="f", + dynamic_success_callbacks=[SyncOnlyLogger()], + ) + logging_obj.update_environment_variables( + model="gpt-5.6-luna", + user=None, + optional_params={}, + litellm_params={"litellm_metadata": {"batch_ignore_default_logging": batch_ignore_default_logging}}, + custom_llm_provider="openai", + ) + + logging_obj.success_handler( + result=self._batch("completed", "file-out"), + start_time=dt_object.now(), + end_time=dt_object.now(), + for_async_request=True, + ) + + assert len(sync_events) == 1 + assert isinstance(sync_events[0], float) + assert (sync_events[0] == 0.0) is batch_ignore_default_logging + + def test_logged_usage_object_keeps_the_batch_chat_usage(self) -> None: + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + from litellm.types.utils import Usage + + aggregate: Final = Usage( + prompt_tokens=29, completion_tokens=48, total_tokens=77, reasoning_tokens=3, cache_creation_input_tokens=300 + ) + billed: Final = self._batch("completed", "file-out").model_copy(update={"usage": aggregate}) + provider_read: Final = self._batch("completed", "file-out") + now: Final = dt_object(2026, 1, 1, 12, 0, 0) + + usage_objects: Final = tuple( + payload["metadata"]["usage_object"] + for payload in ( + get_standard_logging_object_payload( + kwargs={"litellm_params": {}}, + init_response_obj=batch, + start_time=now, + end_time=now, + logging_obj=self._logging_obj(), + status="success", + ) + for batch in (billed, provider_read) + ) + if payload is not None + ) + + assert usage_objects[0] == aggregate.model_dump() + assert usage_objects[1]["prompt_tokens"] == 29 + assert "input_tokens" not in usage_objects[1] + class TestAnthropicPassthroughCustomPricing: """Verify the Anthropic pass-through handler forwards custom pricing.""" diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 86fc069f73b..4861b2687de 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -80,10 +80,11 @@ def _batch_cost_result( """Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns, for mocking it in tests that only care about cost/usage/models.""" from litellm.batches.batch_utils import BatchCostUsageResult + from litellm.types.utils import Usage return BatchCostUsageResult( cost=cost, - usage=usage, + usage=Usage(**usage), models=models, successful_requests=successful_requests, failed_requests=failed_requests, @@ -3261,7 +3262,7 @@ class TestMultiPodBatchCostClaim: patch( "litellm.batches.batch_utils.calculate_batch_cost_and_usage", new_callable=AsyncMock, - return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]), + return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]), ), patch( "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", @@ -3676,7 +3677,17 @@ class TestMultiPodBatchCostClaim: assert journal == ["fetch", "claim", "bill"] assert row.status == "complete" - assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5} + assert json.loads(row.file_object)["usage"] == { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": None, + "completion_tokens_details": None, + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 0}, + } logging_obj.async_success_handler.assert_awaited_once() assert row.batch_processed is True diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 740334252d2..1ea445fda0d 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -69,6 +69,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + LiteLLMBatch, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, @@ -222,6 +223,64 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123 +_PROVIDER_READ_BATCH: Final = LiteLLMBatch.model_validate( + { + "id": "batch_spend_row", + "completion_window": "24h", + "created_at": 1, + "endpoint": "/v1/chat/completions", + "input_file_id": "file-in", + "object": "batch", + "status": "completed", + "output_file_id": "file-out", + "usage": { + "input_tokens": 29, + "input_tokens_details": {"cached_tokens": 8}, + "output_tokens": 46, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 75, + }, + } +) + + +@pytest.mark.parametrize( + "batch", + [ + _PROVIDER_READ_BATCH, + _PROVIDER_READ_BATCH.model_copy( + update={ + "usage": litellm.Usage( + prompt_tokens=29, + completion_tokens=46, + total_tokens=75, + prompt_tokens_details={"cached_tokens": 8}, + ) + } + ), + ], + ids=["provider_read", "poller_billed"], +) +def test_batch_spend_row_stores_the_same_usage_extras_as_a_chat_call_with_its_counts(batch: LiteLLMBatch) -> None: + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "call_type": "aretrieve_batch", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=batch, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (29, 46, 75) + assert json.loads(payload["metadata"])["additional_usage_values"] == _get_additional_usage_values_for_usage( + litellm.Usage( + prompt_tokens=29, completion_tokens=46, total_tokens=75, prompt_tokens_details={"cached_tokens": 8} + ) + ) + + class _HashingCache(litellm.Cache): def __init__(self) -> None: pass diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index d85e48540f9..fe7b5a9470f 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -25,11 +25,13 @@ from litellm.types.utils import ( HiddenParams, ImageObject, ImageResponse, + LiteLLMBatch, ModelResponse, ModelResponseStream, TextCompletionResponse, all_litellm_params, is_decisions_model_mode, + Usage, text_tokens_without_nested_reasoning, ) @@ -1007,3 +1009,214 @@ def test_text_tokens_without_nested_reasoning_clamps( ) == expected_text_tokens ) + + +_BATCH_FIELDS: Final = { + "id": "batch_abc", + "object": "batch", + "endpoint": "/v1/responses", + "input_file_id": "file-abc", + "completion_window": "24h", + "status": "completed", + "created_at": 1, +} +_PROVIDER_BATCH_USAGE: Final = { + "input_tokens": 21, + "input_tokens_details": {"cached_tokens": 5}, + "output_tokens": 543, + "output_tokens_details": {"reasoning_tokens": 448}, + "total_tokens": 564, +} + + +def _batch_token_counts(batch: LiteLLMBatch) -> tuple[int, int, int, int | None, int | None]: + assert batch.usage is not None + assert batch.usage.prompt_tokens_details is not None + assert batch.usage.completion_tokens_details is not None + return ( + batch.usage.prompt_tokens, + batch.usage.completion_tokens, + batch.usage.total_tokens, + batch.usage.prompt_tokens_details.cached_tokens, + batch.usage.completion_tokens_details.reasoning_tokens, + ) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"), + pytest.param( + {"prompt_tokens": 0, "completion_tokens": 0, **_PROVIDER_BATCH_USAGE}, + id="stored_row_with_zeroed_chat_fields", + ), + ), +) +def test_batch_usage_reads_token_counts_from_the_provider_batch_usage(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + + assert _batch_token_counts(batch) == (21, 543, 564, 5, 448) + assert batch.usage is not None + assert (batch.usage.input_tokens, batch.usage.output_tokens) == (21, 543) + + +def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None: + batch: Final = LiteLLMBatch.model_validate( + {**_BATCH_FIELDS, "usage": {"prompt_tokens": 28, "completion_tokens": 8, "total_tokens": 36}} + ) + + assert batch.usage is not None + assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param({"input_tokens": 10, "output_tokens": 5}, id="no_details_or_total"), + pytest.param( + {"input_tokens": 10, "input_tokens_details": None, "output_tokens": 5, "output_tokens_details": None}, + id="null_details", + ), + pytest.param( + { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": None}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": None}, + "total_tokens": 15, + }, + id="null_detail_counts", + ), + ), +) +def test_batch_usage_reads_token_counts_from_partial_provider_batch_usage(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + dumped_usage: Final = json.loads(batch.model_dump_json())["usage"] + + assert batch.usage is not None + assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (10, 5, 15) + assert (dumped_usage["input_tokens"], dumped_usage["output_tokens"], dumped_usage["total_tokens"]) == (10, 5, 15) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"), + pytest.param( + Usage( + prompt_tokens=21, + completion_tokens=543, + total_tokens=564, + prompt_tokens_details={"cached_tokens": 5}, + completion_tokens_details={"reasoning_tokens": 448}, + ), + id="billed_chat_usage", + ), + ), +) +def test_batch_dump_carries_the_chat_and_provider_batch_usage_fields(usage: dict[str, object] | Usage) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + dumped: Final = batch.model_dump_json() + dumped_usage: Final = json.loads(dumped)["usage"] + + assert {key: dumped_usage[key] for key in _PROVIDER_BATCH_USAGE} == _PROVIDER_BATCH_USAGE + assert (dumped_usage["prompt_tokens"], dumped_usage["completion_tokens"]) == (21, 543) + assert batch.model_dump()["usage"] == dumped_usage + assert _batch_token_counts(LiteLLMBatch.model_validate_json(dumped)) == (21, 543, 564, 5, 448) + + +def test_batch_dump_keeps_cache_write_tokens_from_billed_usage() -> None: + batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy( + update={"usage": Usage(prompt_tokens=30, completion_tokens=4, total_tokens=34, cache_creation_input_tokens=7)} + ) + dumped: Final = batch.model_dump_json() + dumped_usage: Final = json.loads(dumped)["usage"] + + assert (dumped_usage["prompt_tokens"], dumped_usage["input_tokens"]) == (30, 30) + assert (dumped_usage["completion_tokens"], dumped_usage["output_tokens"]) == (4, 4) + assert dumped_usage["prompt_tokens_details"]["cache_write_tokens"] == 7 + assert None not in batch.model_dump(exclude_none=True)["usage"]["prompt_tokens_details"].values() + assert json.loads(LiteLLMBatch.model_validate_json(dumped).model_dump_json())["usage"] == dumped_usage + + +def test_batch_json_keeps_usage_that_skipped_validation() -> None: + batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(update={"usage": {"prompt_tokens": 3}}) + + assert json.loads(batch.model_dump_json(warnings=False))["usage"] == {"prompt_tokens": 3} + + +def test_batch_dump_applies_nested_include_and_exclude_to_both_usage_shapes() -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": _PROVIDER_BATCH_USAGE}) + + excluded: Final = batch.model_dump(exclude={"usage": {"prompt_tokens_details", "input_tokens_details"}})["usage"] + included: Final = batch.model_dump(include={"usage": {"prompt_tokens", "input_tokens"}})["usage"] + + assert {"prompt_tokens_details", "input_tokens_details"}.isdisjoint(excluded) + assert (excluded["prompt_tokens"], excluded["input_tokens"]) == (21, 21) + assert included == {"prompt_tokens": 21, "input_tokens": 21} + + +def test_batch_dump_applies_exclude_defaults_to_the_chat_usage_keys() -> None: + batch: Final = LiteLLMBatch.model_validate( + {**_BATCH_FIELDS, "usage": Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)} + ) + + python_usage: Final = batch.model_dump(exclude_defaults=True)["usage"] + json_usage: Final = json.loads(batch.model_dump_json(exclude_defaults=True, warnings=False))["usage"] + + assert python_usage == json_usage + assert {"prompt_tokens_details", "completion_tokens_details"}.isdisjoint(python_usage) + assert (python_usage["prompt_tokens"], python_usage["input_tokens"]) == (10, 10) + + +def test_batch_dump_forwards_every_provider_usage_detail_field() -> None: + batch: Final = LiteLLMBatch.model_validate( + { + **_BATCH_FIELDS, + "usage": { + **_PROVIDER_BATCH_USAGE, + "input_tokens_details": {"cached_tokens": 5, "audio_tokens": 3}, + "output_tokens_details": {"reasoning_tokens": 448, "audio_tokens": 2}, + }, + } + ) + + python_usage: Final = batch.model_dump()["usage"] + json_usage: Final = json.loads(batch.model_dump_json(warnings=False))["usage"] + + assert python_usage == json_usage + assert python_usage["input_tokens_details"] == {"cached_tokens": 5, "audio_tokens": 3} + assert python_usage["output_tokens_details"] == {"reasoning_tokens": 448, "audio_tokens": 2} + + +@pytest.mark.parametrize( + "usage", + [ + {"input_tokens": 29, "output_tokens": None, "total_tokens": 29}, + {"input_tokens": 29, "input_tokens_details": {"cached_tokens": 8}, "total_tokens": 29}, + ], +) +def test_batch_dump_serves_unparsed_provider_usage_unchanged(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + + python_usage: Final = batch.model_dump()["usage"] + json_usage: Final = json.loads(batch.model_dump_json(warnings=False))["usage"] + + assert python_usage == json_usage + assert {key: python_usage[key] for key in usage} == usage + + +def test_batch_dump_applies_exclude_none_to_provider_usage_detail_fields() -> None: + batch: Final = LiteLLMBatch.model_validate( + { + **_BATCH_FIELDS, + "usage": {**_PROVIDER_BATCH_USAGE, "input_tokens_details": {"cached_tokens": 5, "audio_tokens": None}}, + } + ) + + python_usage: Final = batch.model_dump(exclude_none=True)["usage"] + json_usage: Final = json.loads(batch.model_dump_json(exclude_none=True, warnings=False))["usage"] + + assert python_usage == json_usage + assert python_usage["input_tokens_details"] == {"cached_tokens": 5} + assert (python_usage["prompt_tokens"], python_usage["input_tokens"]) == (21, 21) From d25cde47924743fc78c22ee81bc26af176b4baee Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:30:07 -0700 Subject: [PATCH 08/34] chore(cost-map): add fireworks ultramarine 1 flash us and date deepseek v4 flash (#49146) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 19 +++++++++++++++++++ model_prices_and_context_window.json | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e8f6766ed25..82b1ad9bc5d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27794,6 +27794,7 @@ }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -28242,6 +28243,7 @@ }, "fireworks_ai/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -80613,5 +80615,22 @@ "/v1/messages" ], "supports_native_streaming": true + }, + "fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": { + "cache_read_input_token_cost": 4.5e-08, + "cache_read_input_token_cost_priority": 5.625e-08, + "input_cost_per_token": 2.25e-07, + "input_cost_per_token_priority": 2.8125e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "output_cost_per_token_priority": 9.375e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": false, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e8f6766ed25..82b1ad9bc5d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27794,6 +27794,7 @@ }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -28242,6 +28243,7 @@ }, "fireworks_ai/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -80613,5 +80615,22 @@ "/v1/messages" ], "supports_native_streaming": true + }, + "fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": { + "cache_read_input_token_cost": 4.5e-08, + "cache_read_input_token_cost_priority": 5.625e-08, + "input_cost_per_token": 2.25e-07, + "input_cost_per_token_priority": 2.8125e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "output_cost_per_token_priority": 9.375e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": false, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true } } From 53ce41625fdd64afd26a70b77179018a0760bccc Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:30:38 -0700 Subject: [PATCH 09/34] chore(databricks): retirement dates for DeepSeek V4 Flash (0731), V4 Pro (0813) and Inkling (#49124) * chore(databricks): retirement dates for DeepSeek V4 Pro (0813) and Inkling * chore(databricks): retirement dates for DeepSeek V4 Flash (0731), V4 Pro (0813) and Inkling --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 43 ++++++++++--------- model_prices_and_context_window.json | 43 ++++++++++--------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 82b1ad9bc5d..72509bd3c69 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21435,7 +21435,7 @@ "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, "deprecation_date": "2026-10-09", - "input_cost_per_token": 2.9999900000000002e-06, + "input_cost_per_token": 2.99999e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -21444,7 +21444,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_dbu_cost_per_token": 0.000214286, "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -21557,6 +21557,7 @@ "databricks/databricks-deepseek-v4-flash-0731": { "cache_creation_input_token_cost": 1.4e-07, "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-11-05", "input_cost_per_token": 1.4e-07, "input_dbu_cost_per_token": 2e-06, "litellm_provider": "databricks", @@ -21569,7 +21570,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "output_dbu_cost_per_token": 4e-06, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21590,6 +21591,7 @@ "databricks/databricks-deepseek-v4-pro-0813": { "cache_creation_input_token_cost": 1.31999e-06, "cache_read_input_token_cost": 1.3202e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.31999e-06, "input_dbu_cost_per_token": 1.8857e-05, "litellm_provider": "databricks", @@ -21601,7 +21603,7 @@ "mode": "chat", "output_cost_per_token": 3.95997e-06, "output_dbu_cost_per_token": 5.6571e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21788,7 +21790,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22047,7 +22049,7 @@ "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 128000, @@ -22170,7 +22172,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22193,7 +22195,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22264,7 +22266,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22539,7 +22541,7 @@ "databricks/databricks-gpt-5-mini": { "cache_creation_input_token_cost": 2.4997e-07, "cache_read_input_token_cost": 2.499e-08, - "input_cost_per_token": 2.4997000000000006e-07, + "input_cost_per_token": 2.4997e-07, "input_dbu_cost_per_token": 3.571e-06, "litellm_provider": "databricks", "max_input_tokens": 272000, @@ -22548,7 +22550,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.9999700000000004e-06, + "output_cost_per_token": 1.99997e-06, "output_dbu_cost_per_token": 2.8571e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22571,7 +22573,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.9998000000000007e-07, + "output_cost_per_token": 3.9998e-07, "output_dbu_cost_per_token": 5.714000000000001e-06, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22581,7 +22583,7 @@ "databricks/databricks-gpt-oss-120b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 131072, @@ -22606,7 +22608,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.0001999999999996e-07, + "output_cost_per_token": 3.0002e-07, "output_dbu_cost_per_token": 4.285999999999999e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22639,7 +22641,7 @@ "databricks/databricks-gte-large-en": { "cache_creation_input_token_cost": 1.2999e-07, "cache_read_input_token_cost": 1.2999e-07, - "input_cost_per_token": 1.2999000000000001e-07, + "input_cost_per_token": 1.2999e-07, "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, @@ -22655,6 +22657,7 @@ "databricks/databricks-inkling": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.7003e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -22665,7 +22668,7 @@ "mode": "chat", "output_cost_per_token": 4.04999e-06, "output_dbu_cost_per_token": 5.7857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image" @@ -22730,7 +22733,7 @@ "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true @@ -22738,7 +22741,7 @@ "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -22747,7 +22750,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 4.5003000000000007e-07, + "output_cost_per_token": 4.5003e-07, "output_dbu_cost_per_token": 6.429000000000001e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22763,7 +22766,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 82b1ad9bc5d..72509bd3c69 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21435,7 +21435,7 @@ "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, "deprecation_date": "2026-10-09", - "input_cost_per_token": 2.9999900000000002e-06, + "input_cost_per_token": 2.99999e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -21444,7 +21444,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_dbu_cost_per_token": 0.000214286, "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -21557,6 +21557,7 @@ "databricks/databricks-deepseek-v4-flash-0731": { "cache_creation_input_token_cost": 1.4e-07, "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-11-05", "input_cost_per_token": 1.4e-07, "input_dbu_cost_per_token": 2e-06, "litellm_provider": "databricks", @@ -21569,7 +21570,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "output_dbu_cost_per_token": 4e-06, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21590,6 +21591,7 @@ "databricks/databricks-deepseek-v4-pro-0813": { "cache_creation_input_token_cost": 1.31999e-06, "cache_read_input_token_cost": 1.3202e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.31999e-06, "input_dbu_cost_per_token": 1.8857e-05, "litellm_provider": "databricks", @@ -21601,7 +21603,7 @@ "mode": "chat", "output_cost_per_token": 3.95997e-06, "output_dbu_cost_per_token": 5.6571e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21788,7 +21790,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22047,7 +22049,7 @@ "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 128000, @@ -22170,7 +22172,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22193,7 +22195,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22264,7 +22266,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22539,7 +22541,7 @@ "databricks/databricks-gpt-5-mini": { "cache_creation_input_token_cost": 2.4997e-07, "cache_read_input_token_cost": 2.499e-08, - "input_cost_per_token": 2.4997000000000006e-07, + "input_cost_per_token": 2.4997e-07, "input_dbu_cost_per_token": 3.571e-06, "litellm_provider": "databricks", "max_input_tokens": 272000, @@ -22548,7 +22550,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.9999700000000004e-06, + "output_cost_per_token": 1.99997e-06, "output_dbu_cost_per_token": 2.8571e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22571,7 +22573,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.9998000000000007e-07, + "output_cost_per_token": 3.9998e-07, "output_dbu_cost_per_token": 5.714000000000001e-06, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22581,7 +22583,7 @@ "databricks/databricks-gpt-oss-120b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 131072, @@ -22606,7 +22608,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.0001999999999996e-07, + "output_cost_per_token": 3.0002e-07, "output_dbu_cost_per_token": 4.285999999999999e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22639,7 +22641,7 @@ "databricks/databricks-gte-large-en": { "cache_creation_input_token_cost": 1.2999e-07, "cache_read_input_token_cost": 1.2999e-07, - "input_cost_per_token": 1.2999000000000001e-07, + "input_cost_per_token": 1.2999e-07, "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, @@ -22655,6 +22657,7 @@ "databricks/databricks-inkling": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.7003e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -22665,7 +22668,7 @@ "mode": "chat", "output_cost_per_token": 4.04999e-06, "output_dbu_cost_per_token": 5.7857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image" @@ -22730,7 +22733,7 @@ "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true @@ -22738,7 +22741,7 @@ "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -22747,7 +22750,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 4.5003000000000007e-07, + "output_cost_per_token": 4.5003e-07, "output_dbu_cost_per_token": 6.429000000000001e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22763,7 +22766,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true From 19e538ab11aa072b30a78877b13a02cfb1298449 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:40:34 -0700 Subject: [PATCH 10/34] fix(cost-map): set together_ai Qwen3.7-Max back to the published price (#49125) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 72509bd3c69..8f514a9dfb7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48618,13 +48618,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 7.5e-06, + "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 72509bd3c69..8f514a9dfb7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48618,13 +48618,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 7.5e-06, + "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, From e0778bc9dda7e38e523ef4ab481b2bcc26a6db9b Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:43:24 -0700 Subject: [PATCH 11/34] refactor(types): replace Any with proven types in 21 files (#48921) * refactor(types): replace Any with proven types in 35 files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): keep anthropic chat transformation json path untyped Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): accept null gemini vector store fields and load encryption TypedDicts on python 3.10 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): validate Python client user responses and type delete_user as the count it returns * fix(types): keep loose payloads working under the sweep's validators An off-schema Anthropic stream event is now skipped instead of ending the guardrail's text and stop_reason scan, and MAI image usage accepts the loose numbers it did before. New dict and list adapters hide their input from validation errors, RerankResponse indexing keeps its field types, and the delete_user test mocks the int /user/delete returns. * fix(types): make the changed DynamoAI fields ReadOnly and skip non-object A2A stream items * refactor(types): tighten round 2 parse sites and keep payloads out of errors Parse Anthropic SSE events with validate_json and ignore extra fields, validate admin team and key rows per row with hidden inputs, type rerank get and __getitem__ per key, and move the Phoenix prompt version response to a frozen BaseModel * fix(types): keep MAI image calls alive on unreadable usage counts An unreadable MAI usage count now counts as absent instead of failing a billed image call, and the provider's usage fields pass through as sent. The Anthropic stream scan warnings no longer log raw model output, and the off-schema skip has tests that fail on the old scan. * fix(types): count non-object MAI usage as absent and parse Anthropic SSE with json.loads --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/caching/caching.py | 7 +- litellm/images/utils.py | 2 +- .../arize/arize_phoenix_client.py | 11 +- litellm/integrations/opentelemetry.py | 6 +- .../a2a/chat/guardrail_translation/handler.py | 7 +- .../chat/guardrail_translation/handler.py | 64 ++++++---- .../image_generation/mai_transformation.py | 118 ++++++++++++++---- litellm/llms/bedrock/embed/embedding.py | 10 +- .../audio_transcription/transformation.py | 5 +- .../image_generation/gpt_transformation.py | 5 +- litellm/proxy/auth/oauth2_check.py | 5 +- litellm/proxy/client/cli/commands/models.py | 12 +- litellm/proxy/client/users.py | 14 ++- .../team_metadata_validator_e2e.py | 16 ++- .../vigil_guard/vigil_guard.py | 4 +- .../hooks/max_budget_per_session_limiter.py | 55 +++++--- .../internal_user_endpoints.py | 26 +++- litellm/proxy/realtime_endpoints/endpoints.py | 25 +++- .../guardrails/guardrail_hooks/dynamoai.py | 11 +- .../proxy/guardrails/guardrail_hooks/noma.py | 2 +- litellm/types/rerank.py | 33 ++++- .../integrations/arize/test_arize_phoenix.py | 17 +++ .../test_a2a_guardrail_handler.py | 4 + .../test_anthropic_guardrail_handler.py | 51 ++++++++ .../test_mai_image_generation.py | 70 +++++++++++ tests/unit/proxy/client/test_users.py | 5 +- .../test_internal_user_endpoints.py | 28 +++++ 27 files changed, 496 insertions(+), 117 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index d875ac09d40..b58f40af155 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -18,7 +18,7 @@ from contextlib import contextmanager from enum import Enum from typing import Any, Final, Literal, cast -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter import litellm from litellm._internal_context import current_service_target, service_target @@ -43,6 +43,9 @@ from .s3_cache import S3Cache RESPONSE_CACHE_TARGET: Final = "llm_response" _ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"}) _DECODED_LITERAL: Final = TypeAdapter(object) +_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter( + list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True) +) @contextmanager @@ -207,7 +210,7 @@ class Cache: if not redis_startup_nodes: _env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES") if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str): - redis_startup_nodes = json.loads(_env_cluster_nodes) + redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes)) if redis_startup_nodes: # Only pass GCP parameters if they are provided diff --git a/litellm/images/utils.py b/litellm/images/utils.py index 4e0cde24a68..d1a2dd76af0 100644 --- a/litellm/images/utils.py +++ b/litellm/images/utils.py @@ -20,7 +20,7 @@ class ImageEditRequestUtils: image_edit_optional_params: ImageEditOptionalRequestParams, drop_params: bool | None = None, additional_drop_params: Collection[str] | None = None, - ) -> dict: + ) -> dict[str, object]: """ Get optional parameters for the image edit API. diff --git a/litellm/integrations/arize/arize_phoenix_client.py b/litellm/integrations/arize/arize_phoenix_client.py index 6e8ebb99a9e..4d50389faf1 100644 --- a/litellm/integrations/arize/arize_phoenix_client.py +++ b/litellm/integrations/arize/arize_phoenix_client.py @@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix. import urllib.parse from typing import Final +from pydantic import BaseModel, ConfigDict + from litellm.llms.custom_httpx.http_handler import HTTPHandler +class _PromptVersionResponse(BaseModel): + model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True) + + data: dict[str, object] | None = None + + def _sanitize_id(identifier: str) -> str: """Reject path traversal characters and URL-encode the identifier.""" if any(c in identifier for c in ("/", "\\", "#", "?")): @@ -71,8 +79,7 @@ class ArizePhoenixClient: response: Final = self.http_handler.get(url, headers=self.headers) response.raise_for_status() - data: Final = response.json() - return data.get("data") + return _PromptVersionResponse.model_validate(response.json()).data except Exception as e: # Check if it's an HTTP error diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0d0e309034a..a9bd93e6420 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -72,7 +72,7 @@ if TYPE_CHECKING: ) from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth - Span = _Span | Any + Span = _Span Tracer = _Tracer Context = _Context SpanExporter = _SpanExporter @@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span = None # Only set attributes if the span is still recording (not closed) # Note: parent_span is guaranteed to be not None here - if hasattr(parent_span, "set_status"): + if parent_span is not None and hasattr(parent_span, "set_status"): parent_span.set_status(Status(StatusCode.OK)) self.set_attributes(parent_span, kwargs, response_obj) # Raw-request as direct child of parent_span @@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # record error on parent span (keeps hierarchy shallow) # Only set attributes if the span is still recording (not closed) # Note: parent_otel_span is guaranteed to be not None here - if parent_otel_span.is_recording(): + if parent_otel_span is not None and parent_otel_span.is_recording(): self.set_attributes(parent_otel_span, kwargs, response_obj) parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message)) diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index afd9ba84c96..5dc9679fe78 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -14,6 +14,7 @@ import json from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, Optional +from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -28,6 +29,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class _A2ATextPart(TypedDict, total=False): """The subset of an A2A message part this handler reads text from.""" @@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation): obj = item elif isinstance(item, str): try: - obj = json.loads(item.strip()) - except (json.JSONDecodeError, TypeError): + obj = _JSON_OBJECT.validate_python(json.loads(item.strip())) + except (json.JSONDecodeError, TypeError, ValidationError): continue else: continue diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 31b0f06251d..e351542da01 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -18,9 +18,9 @@ from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger @@ -349,14 +349,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ... ) -class _AnthropicSSEDelta(TypedDict, total=False): - type: ReadOnly[str] +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSETextDelta(TypedDict, total=False): + type: ReadOnly[object] text: ReadOnly[str] - stop_reason: ReadOnly[str | None] -class _AnthropicSSEEvent(TypedDict, total=False): - delta: ReadOnly[_AnthropicSSEDelta] +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSETextDeltaEvent(TypedDict, total=False): + delta: ReadOnly[_SSETextDelta] + + +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSEStopDelta(TypedDict, total=False): + stop_reason: ReadOnly[object] + + +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSEStopDeltaEvent(TypedDict, total=False): + delta: ReadOnly[_SSEStopDelta] + + +_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent) +_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent) +_SSEEventT = TypeVar("_SSEEventT") + + +def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None: + try: + return adapter.validate_python(json.loads(data_line)) + except json.JSONDecodeError: + verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type) + except ValidationError: + verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type) + return None class AnthropicMessagesHandler(BaseTranslation): @@ -1806,13 +1832,10 @@ class AnthropicMessagesHandler(BaseTranslation): # Only process content_block_delta events if event_type == "content_block_delta" and data_line: - try: - data: _AnthropicSSEEvent = json.loads(data_line) - delta: _AnthropicSSEDelta = data.get("delta", {}) - if delta.get("type") == "text_delta": - text += delta.get("text", "") - except json.JSONDecodeError: - verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) + data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line) + delta: _SSETextDelta = {} if data is None else data.get("delta", {}) + if delta.get("type") == "text_delta": + text += delta.get("text", "") except Exception as e: verbose_proxy_logger.error("Error extracting text from SSE: %s", e) @@ -1869,14 +1892,11 @@ class AnthropicMessagesHandler(BaseTranslation): # Check for message_delta event with stop_reason if event_type == "message_delta" and data_line: - try: - data: _AnthropicSSEEvent = json.loads(data_line) - delta: _AnthropicSSEDelta = data.get("delta", {}) - stop_reason: str | None = delta.get("stop_reason") - if stop_reason is not None: - return True - except json.JSONDecodeError: - verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) + data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line) + delta: _SSEStopDelta = {} if data is None else data.get("delta", {}) + stop_reason: object = delta.get("stop_reason") + if stop_reason is not None: + return True except Exception as e: verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e) diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 8bb5d3e6b8d..421898e5765 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -1,7 +1,12 @@ -from typing import TYPE_CHECKING, Any, Final +import math +from collections.abc import Mapping +from typing import TYPE_CHECKING, Annotated, Final, TypeAlias import httpx +from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( @@ -18,6 +23,67 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float) + + +def _token_count_or_none(value: object) -> int | float | None: + if value is None: + return None + try: + count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value) + except ValidationError: + verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent") + return None + if not math.isfinite(count): + verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent") + return None + return count + + +_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)] + + +@with_config(ConfigDict(extra="allow")) +class _MAIImageUsageDetails(TypedDict, total=False): + text_tokens: ReadOnly[NotRequired[_TokenCount]] + image_tokens: ReadOnly[NotRequired[_TokenCount]] + + +@with_config(ConfigDict(extra="allow")) +class _MAIImageUsage(TypedDict, total=False): + input_tokens_details: ReadOnly[NotRequired[object]] + num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]] + num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]] + input_tokens: ReadOnly[NotRequired[_TokenCount]] + output_tokens: ReadOnly[NotRequired[_TokenCount]] + num_output_tokens: ReadOnly[NotRequired[_TokenCount]] + output_image_tokens: ReadOnly[NotRequired[_TokenCount]] + total_tokens: ReadOnly[NotRequired[_TokenCount]] + + +@with_config(ConfigDict(extra="allow")) +class _MAIImageResponse(TypedDict, total=False): + usage: ReadOnly[NotRequired[object]] + + +_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails) +_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage) +_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None: + if usage is None: + return None + try: + return _MAI_RAW_USAGE_ADAPTER.validate_python(usage) + except ValidationError: + verbose_logger.warning("MAI image usage is not an object, counting it as absent") + return None + + +_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse) + + class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5).""" @@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): return "maiimage" in model_normalized @staticmethod - def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]: + def normalize_mai_image_usage(usage: object) -> Mapping[str, object]: """Map Azure MAI usage fields to OpenAI ImageUsage schema.""" - if usage is None: + raw_usage: Final = _raw_usage_or_none(usage) + if raw_usage is None: return { "input_tokens": 0, "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, @@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): "total_tokens": 0, } - normalized_usage: Final = dict(usage) - input_tokens_details = normalized_usage.get("input_tokens_details") - if not isinstance(input_tokens_details, dict): - input_tokens_details = {} + normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage) + input_tokens_details_value: Final = normalized_usage.get("input_tokens_details") + input_tokens_details: Final[_MAIImageUsageDetails] = ( + _MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value) + if isinstance(input_tokens_details_value, dict) + else {} + ) text_tokens = normalized_usage.get("num_input_text_tokens") if text_tokens is None: @@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if total_tokens is None: total_tokens = input_tokens + output_tokens - normalized_usage.update( - { - "input_tokens": input_tokens, - "input_tokens_details": { - "image_tokens": image_tokens, - "text_tokens": text_tokens, - }, - "output_tokens": output_tokens, - "total_tokens": total_tokens, - } - ) - return normalized_usage + return { + **raw_usage, + "input_tokens": input_tokens, + "input_tokens_details": { + "image_tokens": image_tokens, + "text_tokens": text_tokens, + }, + "output_tokens": output_tokens, + "total_tokens": total_tokens, + } def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: return ["n", "size"] @@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): json_mode: bool | None = None, ) -> ImageResponse: try: - response: Final = raw_response.json() + response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json()) except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) - if "usage" in response: - response["usage"] = self.normalize_mai_image_usage(response.get("usage")) + response: Final = ( + { + **response_json, + "usage": self.normalize_mai_image_usage(response_json["usage"]), + } + if "usage" in response_json + else dict(response_json) + ) logging_obj.post_call( input=request_data.get("prompt", ""), diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 2f0ed4d0e82..f15a512077a 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx -from pydantic import TypeAdapter +from pydantic import ConfigDict, TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL @@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config from .cohere_transformation import BedrockCohereEmbeddingConfig from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return response.json() + return _JSON_OBJECT.validate_python(response.json()) async def _make_async_call( self, @@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return response.json() + return _JSON_OBJECT.validate_python(response.json()) def _transform_response( self, @@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM): # Parse response if response.status_code == 200: - return response.json() + return _JSON_OBJECT.validate_python(response.json()) else: raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}") diff --git a/litellm/llms/edenai/audio_transcription/transformation.py b/litellm/llms/edenai/audio_transcription/transformation.py index ee574a4f406..9ff530389e2 100644 --- a/litellm/llms/edenai/audio_transcription/transformation.py +++ b/litellm/llms/edenai/audio_transcription/transformation.py @@ -9,6 +9,7 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params @@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]: # mutable-ok: httpx form data """LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a @@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse: if "application/json" not in raw_response.headers.get("content-type", ""): return TranscriptionResponse(text=raw_response.text) - body: Final = raw_response.json() + body: Final = _JSON_OBJECT.validate_python(raw_response.json()) response: Final[TranscriptionResponse] = convert_to_model_response_object( response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription" ) diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index c3a826616ed..c29fe6a7d4f 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Final import httpx +from pydantic import ConfigDict, TypeAdapter from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -9,6 +10,8 @@ from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + if TYPE_CHECKING: from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj @@ -66,7 +69,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: - response: Final = raw_response.json() + response: Final = _JSON_OBJECT.validate_python(raw_response.json()) stringified_response: Final = response ## LOGGING diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py index 970f046c3d6..dc724e7afbe 100644 --- a/litellm/proxy/auth/oauth2_check.py +++ b/litellm/proxy/auth/oauth2_check.py @@ -3,6 +3,7 @@ import os from typing import Final, cast import httpx +from pydantic import ConfigDict, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.llms.custom_httpx.http_handler import ( @@ -11,6 +12,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +_TOKEN_INFO_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class OAuth2TokenValidationError(ValueError): """The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it @@ -184,7 +187,7 @@ class Oauth2Handler: response.raise_for_status() # If we get here, the request was successful - data = response.json() + data = _TOKEN_INFO_RESPONSE.validate_python(response.json()) verbose_proxy_logger.debug( "Oauth2 token validation for token=%s, response from endpoint=%s", diff --git a/litellm/proxy/client/cli/commands/models.py b/litellm/proxy/client/cli/commands/models.py index ac7d5ee45ea..538b42a8c83 100644 --- a/litellm/proxy/client/cli/commands/models.py +++ b/litellm/proxy/client/cli/commands/models.py @@ -1,7 +1,7 @@ # stdlib imports import re from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal @@ -61,6 +61,10 @@ def _get_model_info_obj_from_yaml(model: dict[str, Any]) -> ModelYamlInfo: ) +def _str_field(model: Mapping[str, object], key: str, default: str) -> str: + return str(model.get(key, default)) + + def format_iso_datetime_str(iso_datetime_str: str | None) -> str: """Format an ISO format datetime string to human-readable date with minute resolution.""" if not iso_datetime_str: @@ -142,10 +146,10 @@ def list_models(ctx: click.Context, output_format: Literal["table", "json"]) -> created = int(created) table.add_row( - str(model.get("id", "")), - str(model.get("object", "model")), + _str_field(model, "id", ""), + _str_field(model, "object", "model"), (format_timestamp(created) if isinstance(created, int) else format_iso_datetime_str(created)), - str(model.get("owned_by", "")), + _str_field(model, "owned_by", ""), ) rich.print(table) diff --git a/litellm/proxy/client/users.py b/litellm/proxy/client/users.py index 7b4f36dbc2f..11a32896a99 100644 --- a/litellm/proxy/client/users.py +++ b/litellm/proxy/client/users.py @@ -2,9 +2,13 @@ from collections.abc import Iterable, Mapping from typing import Any, Final import requests +from pydantic import ConfigDict, TypeAdapter from .exceptions import NotFoundError, UnauthorizedError +_JSON_VALUE: Final = TypeAdapter(object, config=ConfigDict(strict=True, hide_input_in_errors=True)) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class UsersManagementClient: def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30): @@ -39,7 +43,7 @@ class UsersManagementClient: if response.status_code == 404: raise NotFoundError(response.text) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) def get_user_v2(self, user_id: str | None = None) -> dict[str, object]: """Get user info v2 - lightweight, returns only user object (GET /v2/user/info)""" @@ -51,7 +55,7 @@ class UsersManagementClient: if response.status_code == 404: raise NotFoundError(response.text) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) def create_user(self, user_data: Mapping[str, object]) -> dict[str, object]: """Create a new user (POST /user/new)""" @@ -60,9 +64,9 @@ class UsersManagementClient: if response.status_code == 401: raise UnauthorizedError(response.text) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) - def delete_user(self, user_ids: list[str]) -> dict[str, object]: + def delete_user(self, user_ids: list[str]) -> object: """Delete users (POST /user/delete)""" url: Final = f"{self.base_url}/user/delete" response: Final = requests.post( @@ -71,4 +75,4 @@ class UsersManagementClient: if response.status_code == 401: raise UnauthorizedError(response.text) response.raise_for_status() - return response.json() + return _JSON_VALUE.validate_python(response.json()) diff --git a/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py b/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py index af4a59efb06..fe61f8c5b01 100644 --- a/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py +++ b/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py @@ -16,6 +16,10 @@ unknown impl value raises, which the proxy converts to the fail-closed 503. """ import os +from typing import Final + +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy.management_helpers.team_metadata_validation import ( @@ -23,12 +27,20 @@ from litellm.proxy.management_helpers.team_metadata_validation import ( TeamMetadataValidationResult, ) from litellm.types.llms.custom_http import httpxSpecialProvider -from typing import Final ALLOWED_COST_CENTERS: Final = frozenset({"CC-1001", "CC-1002"}) CLOSED_PORT_URL: Final = "http://127.0.0.1:9/validate" +@with_config(ConfigDict(extra="allow", strict=True)) +class _CostCenterReply(TypedDict): + ok: NotRequired[ReadOnly[object]] + reason: NotRequired[ReadOnly[str | None]] + + +_COST_CENTER_REPLY: Final = TypeAdapter(_CostCenterReply) + + async def _validate_allowlist(payload: TeamMetadataValidationPayload) -> TeamMetadataValidationResult: cost_center: Final = payload.metadata.get("cost_center") if cost_center is None: @@ -54,7 +66,7 @@ async def _validate_via_http(payload: TeamMetadataValidationPayload, service_url }, timeout=2.0, ) - body: Final = response.json() + body: Final = _COST_CENTER_REPLY.validate_python(response.json()) if body.get("ok") is True: return TeamMetadataValidationResult(valid=True) return TeamMetadataValidationResult( diff --git a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py index 611738ede8a..701b7bbb3c6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py @@ -300,7 +300,7 @@ class VigilGuardGuardrail(CustomGuardrail): def _build_output( inputs: GenericGuardrailAPIInputs, final_texts: list[str], - final_tool_calls: Any, + final_tool_calls: _ToolCalls | None, ) -> GenericGuardrailAPIInputs: # When nothing was changed, return the input shape verbatim so the guardrail # logs "allow" rather than "mask". When a text or a tool-call argument was @@ -315,7 +315,7 @@ class VigilGuardGuardrail(CustomGuardrail): guardrailed["images"] = inputs["images"] if "tools" in inputs: guardrailed["tools"] = inputs["tools"] - if tool_calls_changed: + if tool_calls_changed and final_tool_calls is not None: guardrailed["tool_calls"] = final_tool_calls return guardrailed diff --git a/litellm/proxy/hooks/max_budget_per_session_limiter.py b/litellm/proxy/hooks/max_budget_per_session_limiter.py index cf3025d8c7b..002aeb145a5 100644 --- a/litellm/proxy/hooks/max_budget_per_session_limiter.py +++ b/litellm/proxy/hooks/max_budget_per_session_limiter.py @@ -18,6 +18,8 @@ import logging import os from typing import TYPE_CHECKING, Any, Final +from pydantic import ConfigDict, TypeAdapter + from litellm import DualCache from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -36,6 +38,14 @@ else: InternalUsageCache = Any +_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapter( + str | bytes | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True) +) +_OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter( + str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + # Redis Lua script for atomic float increment with TTL. # INCRBYFLOAT returns the new value as a string. # Only sets EXPIRE on first call (when prior value was nil). @@ -216,9 +226,11 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): """Read current accumulated spend for a session.""" if self.internal_usage_cache.dual_cache.redis_cache is not None: try: - result = await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) - if result is not None: - return float(result) + redis_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( + await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) + ) + if redis_result is not None: + return float(redis_result) return 0.0 except Exception as e: log_redis_failure( @@ -228,24 +240,28 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): e, ) - result = await self.internal_usage_cache.async_get_cache( - key=cache_key, - litellm_parent_otel_span=None, - local_only=True, + cache_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( + await self.internal_usage_cache.async_get_cache( + key=cache_key, + litellm_parent_otel_span=None, + local_only=True, + ) ) - if result is not None: - return float(result) + if cache_result is not None: + return float(cache_result) return 0.0 async def _increment_spend(self, cache_key: str, amount: float) -> float: """Atomically increment the session spend and return the new value.""" if self.increment_script is not None: try: - result: Final = await self.increment_script( - keys=[cache_key], - args=[str(amount), self.ttl], + script_result: Final[str | bytes | int | float] = _CACHE_AMOUNT_ADAPTER.validate_python( + await self.increment_script( + keys=[cache_key], + args=[str(amount), self.ttl], + ) ) - return float(result) + return float(script_result) except Exception as e: log_redis_failure( verbose_proxy_logger, @@ -257,12 +273,15 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): return await self._in_memory_increment_spend(cache_key, amount) async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float: - current: Final = await self.internal_usage_cache.async_get_cache( - key=cache_key, - litellm_parent_otel_span=None, - local_only=True, + current: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( + await self.internal_usage_cache.async_get_cache( + key=cache_key, + litellm_parent_otel_span=None, + local_only=True, + ) ) - new_value: Final = (float(current) if current is not None else 0.0) + amount + current_spend: Final = float(current) if current is not None else 0.0 + new_value: Final = current_spend + amount await self.internal_usage_cache.async_set_cache( key=cache_key, value=new_value, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index bb901f22649..68287432492 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -22,7 +22,7 @@ from typing import Any, Final, Literal, Protocol, cast, overload import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, TypedDict import litellm @@ -124,6 +124,20 @@ _USER_MODEL_BUDGET_ADAPTER: Final = TypeAdapter(dict[str, float | BudgetConfig]) _USER_CACHE_INVALIDATION_BATCH_SIZE: Final = 50 +@with_config(ConfigDict(extra="allow", strict=True)) +class _AdminTeamAndKeyRows(TypedDict): + keys: ReadOnly[Sequence[object] | None] + teams: ReadOnly[Sequence[object] | None] + + +_ADMIN_TEAM_AND_KEY_ROWS_ADAPTER: Final = TypeAdapter( + list[_AdminTeamAndKeyRows], config=ConfigDict(hide_input_in_errors=True) +) +_ADMIN_KEY_ROW_ADAPTER: Final = TypeAdapter( + dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + def _user_table( prisma_client: "PrismaClient | None", ) -> "TableActions[prisma_models.LiteLLM_UserTable]": @@ -1170,8 +1184,8 @@ async def user_info_v2( async def _fetch_admin_teams_and_keys_rows( prisma_client: "PrismaClient", sql_query: str -) -> Sequence[Mapping[str, Sequence[Mapping[str, object]] | None]]: - return await prisma_client.db.query_raw(sql_query) +) -> Sequence[_AdminTeamAndKeyRows]: + return _ADMIN_TEAM_AND_KEY_ROWS_ADAPTER.validate_python(await prisma_client.db.query_raw(sql_query)) async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth): @@ -1201,17 +1215,17 @@ async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth): verbose_proxy_logger.debug("results_keys: %s", results) - _keys_in_db: Final[Sequence[Mapping[str, object]]] = results[0]["keys"] or [] + _keys_in_db: Final[Sequence[object]] = results[0]["keys"] or [] # cast all keys to LiteLLM_VerificationToken keys_in_db: Final = [] for key in _keys_in_db: - key_payload = dict[str, object](key) + key_payload = _ADMIN_KEY_ROW_ADAPTER.validate_python(key) if key_payload.get("models") is None: key_payload["models"] = [] keys_in_db.append(LiteLLM_VerificationToken.model_validate(key_payload)) # cast all teams to LiteLLM_TeamTable - _teams_rows: Final[Sequence[Mapping[str, object]]] = results[0]["teams"] or [] + _teams_rows: Final[Sequence[object]] = results[0]["teams"] or [] _teams_in_db: Final = sorted( (LiteLLM_TeamTable.model_validate(team) for team in _teams_rows), key=lambda x: getattr(x, "team_alias", "") or "", diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 9c0f3c4b81c..9dbf6bcc43d 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -3,11 +3,13 @@ import json import time from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth @@ -44,6 +46,20 @@ _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" _ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +@with_config(ConfigDict(strict=True, extra="allow")) +class _RealtimeTokenPayload(TypedDict, total=False): + v: NotRequired[ReadOnly[object]] + ephemeral_key: NotRequired[ReadOnly[str]] + model_id: NotRequired[ReadOnly[str]] + user_id: NotRequired[ReadOnly[str | None]] + team_id: NotRequired[ReadOnly[str | None]] + expires_at: NotRequired[ReadOnly[object]] + session_type: NotRequired[ReadOnly[str | None]] + + +_REALTIME_TOKEN_PAYLOAD_ADAPTER: Final = TypeAdapter(_RealtimeTokenPayload) + + def _coerce_realtime_session_type(session_type: str | None) -> str: if session_type in _ALLOWED_SESSION_TYPES: return session_type @@ -201,17 +217,14 @@ def _header_value(headers: Mapping[str, str], name: str, default: str) -> str: def _decode_realtime_token_payload( decrypted_value: str, -) -> dict[str, Any] | None: +) -> _RealtimeTokenPayload | None: """ Decode realtime token payload; returns None for legacy/raw ephemeral tokens. """ try: - decoded: Final = json.loads(decrypted_value) + decoded: Final = _REALTIME_TOKEN_PAYLOAD_ADAPTER.validate_python(json.loads(decrypted_value)) except Exception: return None - - if not isinstance(decoded, dict): - return None if decoded.get("v") != _REALTIME_TOKEN_VERSION: return None if not isinstance(decoded.get("ephemeral_key"), str): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py b/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py index ae2df8069a9..70d83eeb840 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py @@ -2,9 +2,10 @@ import enum from collections.abc import Mapping -from typing import Any, Literal, TypedDict +from typing import Literal, TypedDict from pydantic import Field +from typing_extensions import ReadOnly from .base import GuardrailConfigModel @@ -42,7 +43,7 @@ class PolicyApplicableTo(str, enum.Enum): class DynamoAIRequest(TypedDict, total=False): """Request structure for DynamoAI /moderation/analyze endpoint""" - messages: list[dict[str, Any]] + messages: ReadOnly[list[dict[str, object]]] textType: DynamoTextType | None policyIds: list[str] modelId: str | None @@ -58,8 +59,8 @@ class PolicyInfo(TypedDict, total=False): description: str method: PolicyMethod action: Literal["BLOCK", "WARN", "REDACT", "SANITIZE", "NONE"] - methodParams: dict[str, Any] - decisionParams: dict[str, Any] + methodParams: ReadOnly[dict[str, object]] + decisionParams: ReadOnly[dict[str, object]] applicableTo: PolicyApplicableTo created_at: str creatorId: str @@ -76,7 +77,7 @@ class AppliedPolicyDto(TypedDict, total=False): """Applied policy details from DynamoAI response""" policy: PolicyInfo - outputs: dict[str, Any] | None + outputs: ReadOnly[dict[str, object] | None] action: str | None diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py index 992815bf454..30b0fb80970 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py @@ -89,7 +89,7 @@ class NomaV2StreamingParams(BaseModel): cls, value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo ) -> object: try: - validated: Final = handler(value) + validated: Final[object] = handler(value) except ValidationError: verbose_proxy_logger.warning( "Noma v2 guardrail %s: ignoring %s=%.100r, which is not a valid value; the default applies", diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index 6d1ab85012b..ffdcf53fd7e 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -4,7 +4,8 @@ https://docs.cohere.com/reference/rerank """ -from typing import Literal +from collections.abc import Sequence +from typing import Literal, overload from pydantic import ConfigDict, PrivateAttr from typing_extensions import ReadOnly, Required, TypedDict @@ -95,11 +96,35 @@ class RerankResponse(LiteLLMBaseModel): def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params - def __getitem__(self, key): + @overload + def __getitem__(self, key: Literal["id"]) -> str | None: ... + + @overload + def __getitem__(self, key: Literal["results"]) -> list[RerankResponseResult] | None: ... + + @overload + def __getitem__(self, key: Literal["meta"]) -> RerankResponseMeta | None: ... + + @overload + def __getitem__(self, key: str) -> str | Sequence[RerankResponseResult] | RerankResponseMeta | None: ... + + def __getitem__(self, key: str) -> str | Sequence[RerankResponseResult] | RerankResponseMeta | None: return self.__dict__[key] - def get(self, key, default=None): + @overload + def get(self, key: Literal["id"], default: None = None) -> str | None: ... + + @overload + def get(self, key: Literal["results"], default: None = None) -> list[RerankResponseResult] | None: ... + + @overload + def get(self, key: Literal["meta"], default: None = None) -> RerankResponseMeta | None: ... + + @overload + def get(self, key: str, default: object = None) -> object: ... + + def get(self, key: str, default: object = None) -> object: return self.__dict__.get(key, default) - def __contains__(self, key) -> bool: + def __contains__(self, key: str) -> bool: return key in self.__dict__ diff --git a/tests/unit/integrations/arize/test_arize_phoenix.py b/tests/unit/integrations/arize/test_arize_phoenix.py index eb9e1326ced..5edf6e6d379 100644 --- a/tests/unit/integrations/arize/test_arize_phoenix.py +++ b/tests/unit/integrations/arize/test_arize_phoenix.py @@ -971,6 +971,23 @@ def test_arize_phoenix_client_get_prompt_version_rejects_traversal(): client.get_prompt_version("../../projects") +def test_arize_phoenix_client_error_does_not_echo_prompt_content(): + import httpx + import respx + + from litellm.integrations.arize.arize_phoenix_client import ArizePhoenixClient + + client = ArizePhoenixClient(api_key="test-key", api_base="https://phoenix.test") + with respx.mock: + respx.get("https://phoenix.test/v1/prompt_versions/pv-1").mock( + return_value=httpx.Response(200, json={"data": "secret system prompt text"}) + ) + with pytest.raises(Exception, match="Error fetching prompt version 'pv-1'") as exc_info: + client.get_prompt_version("pv-1") + + assert "secret system prompt text" not in str(exc_info.value) + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) diff --git a/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py b/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py index dd7e8fadcd8..01eff7f6bbf 100644 --- a/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py +++ b/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py @@ -33,3 +33,7 @@ class TestA2AGuardrailHandlerStreamingScanKey: def test_unparseable_items_are_ignored(self): key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hi"), "not json", b"bytes"]) assert key.texts == ("hi",) + + def test_json_items_that_are_not_objects_are_ignored(self): + key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hi"), "[1, 2]", "5"]) + assert key.texts == ("hi",) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 9e88b59adc9..141ad8a0481 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -302,6 +302,57 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: texts.append(data["delta"]["text"]) return texts + @pytest.mark.parametrize( + "off_schema_delta", + [None, "stray", {"type": "text_delta", "text": 5}, {"type": "thinking_delta", "text": None}], + ) + def test_off_schema_delta_is_skipped_without_ending_the_text_scan(self, off_schema_delta): + events = [ + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "abc"}}, + {"type": "content_block_delta", "index": 1, "delta": off_schema_delta}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "def"}}, + ] + chunk = "".join(f"event: content_block_delta\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + assert AnthropicMessagesHandler().get_streaming_string_so_far([chunk]) == "abcdef" + + def test_malformed_delta_data_is_skipped_without_ending_the_text_scan(self): + text_delta = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "abc"}} + chunk = ( + f"event: content_block_delta\ndata: {json.dumps(text_delta)}\n\n" + "event: content_block_delta\ndata: {not json\n\n" + f"event: content_block_delta\ndata: {json.dumps(text_delta)}\n\n" + ).encode() + + assert AnthropicMessagesHandler().get_streaming_string_so_far([chunk]) == "abcabc" + + def test_text_delta_with_a_lone_surrogate_escape_is_scanned(self): + event = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pineapple \ud83d"}} + chunk = f"event: content_block_delta\ndata: {json.dumps(event)}\n\n".encode() + + assert AnthropicMessagesHandler().get_streaming_string_so_far([chunk]) == "pineapple \ud83d" + + def test_message_delta_with_a_lone_surrogate_escape_still_ends_the_stream(self): + event = {"type": "message_delta", "delta": {"stop_reason": "stop_sequence", "stop_sequence": "\ud83d"}} + chunk = f"event: message_delta\ndata: {json.dumps(event)}\n\n".encode() + + scan_key = AnthropicMessagesHandler().get_streaming_scan_key([chunk]) + + assert scan_key is not None + assert scan_key.stream_ended is True + + def test_off_schema_message_delta_does_not_hide_a_later_stop_reason(self): + events = [ + {"type": "message_delta", "delta": None}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}}, + ] + chunk = "".join(f"event: message_delta\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + scan_key = AnthropicMessagesHandler().get_streaming_scan_key([chunk]) + + assert scan_key is not None + assert scan_key.stream_ended is True + @pytest.mark.asyncio async def test_deliver_ended_stream_rewrites_writes_text_back_into_sse_chunks(self): handler = AnthropicMessagesHandler() diff --git a/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py index 27e78d35c69..e702abf196d 100644 --- a/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -386,6 +386,76 @@ class TestAzureMAIImageGeneration: assert normalized["input_tokens"] == 22 assert normalized["total_tokens"] == 22 + def test_normalize_mai_usage_accepts_a_bool_token_count(self): + normalized = AzureFoundryMAIImageGenerationConfig().normalize_mai_image_usage( + {"input_tokens": 22, "output_tokens": True} + ) + + assert normalized["output_tokens"] == 1 + assert normalized["total_tokens"] == 23 + + def test_normalize_mai_usage_reads_string_token_counts_as_numbers(self): + normalized = AzureFoundryMAIImageGenerationConfig().normalize_mai_image_usage( + {"input_tokens": "8", "output_tokens": "1024"} + ) + + assert normalized["input_tokens"] == 8 + assert normalized["total_tokens"] == 1032 + + @pytest.mark.parametrize("unreadable", ["n/a", "inf", "nan", float("inf")]) + def test_normalize_mai_usage_falls_back_past_an_unreadable_count(self, unreadable): + normalized = AzureFoundryMAIImageGenerationConfig().normalize_mai_image_usage( + {"input_tokens": 8, "output_tokens": unreadable, "num_output_tokens": 1024} + ) + + assert normalized["output_tokens"] == 1024 + assert normalized["total_tokens"] == 1032 + + @pytest.mark.parametrize("unreadable", ["n/a", {"n": 1}, [1024]]) + def test_transform_image_generation_response_keeps_the_image_when_a_count_is_unreadable(self, unreadable): + raw_response = MagicMock(spec=httpx.Response) + raw_response.json.return_value = { + "created": 1780897477, + "data": [{"b64_json": "abc123"}], + "usage": {"input_tokens": 8, "output_tokens": 1024, "total_tokens": 1032, "num_output_tokens": unreadable}, + } + + image_response = AzureFoundryMAIImageGenerationConfig().transform_image_generation_response( + model="MAI-Image-2.5", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A red fox"}, + optional_params={"width": 1024, "height": 1024}, + litellm_params={}, + encoding=None, + ) + + assert image_response.data[0].b64_json == "abc123" + assert image_response.usage.output_tokens == 1024 + assert image_response.usage.total_tokens == 1032 + + @pytest.mark.parametrize("usage", [[], "n/a", 5]) + def test_transform_image_generation_response_keeps_the_image_when_usage_is_not_an_object(self, usage): + raw_response = MagicMock(spec=httpx.Response) + raw_response.json.return_value = {"created": 1780897477, "data": [{"b64_json": "abc123"}], "usage": usage} + + image_response = AzureFoundryMAIImageGenerationConfig().transform_image_generation_response( + model="MAI-Image-2.5", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A red fox"}, + optional_params={"width": 1024, "height": 1024}, + litellm_params={}, + encoding=None, + ) + + assert image_response.data[0].b64_json == "abc123" + assert image_response.usage.input_tokens == 0 + assert image_response.usage.output_tokens == 0 + assert image_response.usage.total_tokens == 0 + def test_azure_sync_image_generation_uses_mai_response_transform(self): raw_response = MagicMock(spec=httpx.Response) raw_response.json.return_value = { diff --git a/tests/unit/proxy/client/test_users.py b/tests/unit/proxy/client/test_users.py index 5b4d89420ab..a936de77c3e 100644 --- a/tests/unit/proxy/client/test_users.py +++ b/tests/unit/proxy/client/test_users.py @@ -5,7 +5,6 @@ import pytest import requests - from litellm.proxy.client.users import ( NotFoundError, UnauthorizedError, @@ -72,9 +71,9 @@ def test_create_user_unauthorized(mock_post, client): @patch("requests.post") def test_delete_user_success(mock_post, client): mock_post.return_value.status_code = 200 - mock_post.return_value.json.return_value = {"deleted": 1} + mock_post.return_value.json.return_value = 1 result = client.delete_user(["u1"]) - assert result["deleted"] == 1 + assert result == 1 mock_post.assert_called_once() diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 78a0a765fe6..42162087c9b 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4292,6 +4292,34 @@ async def test_get_user_info_for_proxy_admin_validates_keys_and_teams(): assert returned_key["models"] == [] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "raw_rows", + [ + [{"teams": None, "keys": "sk-row-content-must-not-leak"}], + [{"teams": None, "keys": ["sk-row-content-must-not-leak"]}], + ], +) +async def test_get_user_info_for_proxy_admin_errors_do_not_echo_row_content(raw_rows): + from unittest.mock import AsyncMock, MagicMock, patch + + from pydantic import ValidationError + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _get_user_info_for_proxy_admin, + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.query_raw = AsyncMock(return_value=raw_rows) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + with pytest.raises(ValidationError) as exc_info: + await _get_user_info_for_proxy_admin(user_api_key_dict=UserAPIKeyAuth(user_id=None)) + + assert "sk-row-content-must-not-leak" not in str(exc_info.value) + + def _object_permission_mocks(mocker, existing_object_permission_id=None): """Prisma double whose user row optionally already links a permission row.""" mock_prisma_client = mocker.MagicMock() From fd594ab0cf743fb16d8aa8e50972fc788fadd44a Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 29 Oct 2026 19:00:04 -0700 Subject: [PATCH 12/34] fix: enforce LiteAdmin MCP route permissions (#48080) * fix(admin-mcp): expose and enforce credential route permissions * fix: restore beta header and telemetry test parity * fix(admin-mcp): align permissions with registered HTTP routes * test(admin-mcp): verify encoded paths through HTTP authorization * test(admin-mcp): simplify HTTP permission regression fixtures --- .../src/providers/anthropic/beta.rs | 4 +- litellm/proxy/_types.py | 7 + litellm/proxy/admin_mcp.py | 96 ++++++++- litellm/proxy/auth/route_checks.py | 19 +- .../internal_user_endpoints.py | 27 ++- pyproject.toml | 2 +- tests/unit/proxy/test_admin_mcp.py | 186 +++++++++++++++++- tests/unit/proxy/test_component_allowlists.py | 5 +- tests/unit/proxy/test_proxy_server.py | 5 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 ++ uv.lock | 8 +- 11 files changed, 344 insertions(+), 29 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 8429c98a88a..80095cac36c 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -238,7 +238,8 @@ impl AnthropicBeta { Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 - | Self::ThinkingBindingControls20260801 => &[], + | Self::ThinkingBindingControls20260801 + | Self::ThinkingDisplayUpdates20260818 => &[], Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse], Self::Effort20251124 => &[BetaProvider::VertexAi], Self::FastMode20260201 | Self::Oauth20250420 => &[ @@ -275,7 +276,6 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi], - Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks], Self::ToolSearchTool20251019 => &[ BetaProvider::Anthropic, BetaProvider::AzureAi, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4047fa3f0ad..0a09fe4a855 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3177,11 +3177,18 @@ from litellm.types.proxy.auth.user_api_key_auth import ( ) +class AdminMCPPermissions(LiteLLMPydanticObjectBase): + model_config = ConfigDict(frozen=True) + tools: tuple[str, ...] + action_allowed: bool | None = None + + class UserInfoResponse(LiteLLMPydanticObjectBase): user_id: str | None user_info: dict | BaseModel | None keys: list teams: list + admin_mcp_permissions: AdminMCPPermissions | None = None class UserInfoV2Response(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/admin_mcp.py b/litellm/proxy/admin_mcp.py index 6371863f45c..32576ea8113 100644 --- a/litellm/proxy/admin_mcp.py +++ b/litellm/proxy/admin_mcp.py @@ -1,20 +1,21 @@ import os import re -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, Iterator, Mapping from contextlib import asynccontextmanager from contextvars import ContextVar from typing import Final -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException, Request +from fastapi.routing import APIRoute from pydantic import TypeAdapter from starlette.datastructures import Headers -from starlette.requests import Request from starlette.routing import Mount from starlette.types import ASGIApp, Receive, Scope, Send from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS -from litellm.proxy._types import SpecialHeaders +from litellm.proxy._types import AdminMCPPermissions, SpecialHeaders, UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.middleware.admission_control_middleware import ADMISSION_LEASE_SCOPE_KEY _REQUEST_HEADERS: Final = frozenset( @@ -58,6 +59,91 @@ _RESERVED_KEY_HEADERS: Final = ( _SETTINGS: Final = TypeAdapter(Mapping[str, object]) _IDENTITY_MAPPINGS: Final = TypeAdapter(tuple[dict[str, object], ...] | dict[str, object] | None) _OAUTH_MAPPINGS: Final = TypeAdapter(dict[str, str]) +_ROUTE_GRANTS: Final = TypeAdapter(tuple[str, ...]) + + +def _admin_route_allowed(auth: UserAPIKeyAuth, request: Request, method: str, path: str) -> bool: + target: Final = Request({**request.scope, "method": method, "path": path, "raw_path": path.encode()}) + try: + RouteChecks.should_call_route(route=path, valid_token=auth, request=target) + except HTTPException: + return False + return True + + +def _admin_routes(request: Request) -> Iterator[tuple[tuple[str, str], APIRoute]]: + app: Final = request.app + if not isinstance(app, FastAPI): + return + for route in app.routes: + if not isinstance(route, APIRoute): + continue + for method in route.methods: + yield (method, route.path_format), route + + +def _admin_grant_paths(route: APIRoute, grant: str) -> Iterator[str]: + if route.path_regex.fullmatch(grant) is not None: + yield grant + prefix: Final = grant[:-1] if grant.endswith("*") else grant + "/" + representative: Final = re.sub(r"\{[^}]+\}", "_", route.path_format) + # A matching suffix supplies the rest of a concrete route for prefix grants, + # including grants ending inside a static segment or a path parameter. + for candidate in (prefix + representative[offset:] for offset in range(len(representative) + 1)): + if route.path_regex.fullmatch(candidate) is not None: + yield candidate + + +def _admin_grant_allowed(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute, grant: str) -> bool: + return any(_admin_route_allowed(auth, request, method, path) for path in _admin_grant_paths(route, grant)) + + +def _admin_tool_visible(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute) -> bool: + try: + RouteChecks.check_route_enabled(route.path) + except HTTPException: + return False + if _admin_route_allowed(auth, request, method, route.path): + return True + grants: Final = _ROUTE_GRANTS.validate_python(getattr(auth, "allowed_routes", None) or ()) + return bool(route.param_convertors) and any( + _admin_grant_allowed(auth, request, method, route, grant) for grant in grants if grant.startswith("/") + ) + + +def get_admin_mcp_permissions( + auth: UserAPIKeyAuth, request: Request, tool: str | None = None, path: str | None = None +) -> AdminMCPPermissions: + try: + from litellm_admin_mcp.catalog import ( # pyright: ignore[reportMissingTypeStubs] # connector has no py.typed marker + BY_NAME, + OPERATIONS, + ) + except ImportError as exc: + raise HTTPException( + status_code=503, detail="Admin MCP permissions require the admin-mcp dependency group (Python 3.12+)" + ) from exc + + routes: Final = dict(_admin_routes(request)) + allowed: Final = tuple( + operation.name + for operation in OPERATIONS + if (route := routes.get((operation.method, operation.path))) is not None + and _admin_tool_visible(auth, request, operation.method, route) + ) + if tool is None: + return AdminMCPPermissions(tools=allowed) + operation: Final = BY_NAME.get(tool) + if operation is None or tool not in allowed: + return AdminMCPPermissions(tools=allowed, action_allowed=False) + if path is None: + return AdminMCPPermissions(tools=allowed, action_allowed=True) + registered: Final = routes[(operation.method, operation.path)] + decoded_path: Final = unquote(path) + action_allowed: Final = registered.path_regex.fullmatch(decoded_path) is not None and _admin_route_allowed( + auth, request, operation.method, decoded_path + ) + return AdminMCPPermissions(tools=allowed, action_allowed=action_allowed) def _configured_key_header() -> bytes | None: diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 105acb237af..125bfab4edf 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -111,14 +111,7 @@ _TOKEN_COUNTING_ACTION: Final = "countTokens" class RouteChecks: @staticmethod - def should_call_route( - route: str, - valid_token: UserAPIKeyAuth, - request: Request | None = None, - ): - """ - Check if management route is disabled and raise exception - """ + def check_route_enabled(route: str) -> None: try: from litellm_enterprise.proxy.auth.route_checks import EnterpriseRouteChecks @@ -128,6 +121,16 @@ class RouteChecks: except Exception: pass + @staticmethod + def should_call_route( + route: str, + valid_token: UserAPIKeyAuth, + request: Request | None = None, + ): + """ + Check if management route is disabled and raise exception + """ + RouteChecks.check_route_enabled(route) # Check if Virtual Key is allowed to call the route - Applies to all Roles RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token, request=request) return True diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 68287432492..6ceab27befa 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -945,6 +945,14 @@ async def user_info( request: Request, user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + admin_mcp: bool = fastapi.Query( + default=False, description="Include the caller's current admin MCP route permissions" + ), + mcp_tool: str | None = fastapi.Query(default=None, description="Admin MCP tool to authorize for this caller"), + mcp_path: str | None = fastapi.Query( + default=None, + description="URL-encoded concrete management path used by the MCP tool, before query-string encoding", + ), ): """ [10/07/2024] @@ -965,12 +973,23 @@ async def user_info( user_id = _normalize_user_info_user_id(request=request, user_id=user_id) _enforce_user_info_access(user_id=user_id, user_api_key_dict=user_api_key_dict) + from litellm.proxy.admin_mcp import get_admin_mcp_permissions + + mcp_permissions: Final = ( + get_admin_mcp_permissions(user_api_key_dict, request, mcp_tool, mcp_path) if admin_mcp is True else None + ) + if prisma_client is None: raise Exception( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) if user_id is None and user_api_key_has_admin_view(user_api_key_dict): - return await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict) + admin_response: Final = await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict) + return ( + admin_response.model_copy(update={"admin_mcp_permissions": mcp_permissions}) + if mcp_permissions is not None + else admin_response + ) elif user_id is None: user_id = user_api_key_dict.user_id ## GET USER ROW ## @@ -1009,7 +1028,11 @@ async def user_info( ), ) - return response_data + return ( + response_data.model_copy(update={"admin_mcp_permissions": mcp_permissions}) + if mcp_permissions is not None + else response_data + ) except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info(): Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/pyproject.toml b/pyproject.toml index 6a21fd9b26b..6e9e75a5b7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -251,7 +251,7 @@ e2e-dev = [ "mcp>=2.3.0,<3", ] admin-mcp = [ - "litellm-admin-mcp @ https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz#sha256=86b930f6706fb2da10d53d1798ee7fd14e44fb0afdef0da122cf0e6bb88fa2c3 ; python_version >= '3.12'", + "litellm-admin-mcp @ https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz#sha256=5803a20a00a18177f4591706956f31be311879543049b8b38143ad90bb1cb0a6 ; python_version >= '3.12'", ] proxy-dev = [ { include-group = "admin-mcp" }, diff --git a/tests/unit/proxy/test_admin_mcp.py b/tests/unit/proxy/test_admin_mcp.py index dc956038525..4aa0226552f 100644 --- a/tests/unit/proxy/test_admin_mcp.py +++ b/tests/unit/proxy/test_admin_mcp.py @@ -1,6 +1,7 @@ import asyncio import json import sys +from types import SimpleNamespace from typing import Final import httpx2 @@ -12,11 +13,11 @@ from starlette.testclient import TestClient from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS from litellm.proxy import proxy_server -from litellm.proxy._types import LiteLLM_UserTable, ProxyException, SpecialHeaders -from litellm.proxy.admin_mcp import admin_mcp_lifespan -from litellm.proxy.auth.user_api_key_auth import get_api_key, get_api_key_from_custom_header +from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, ProxyException, SpecialHeaders, UserAPIKeyAuth +from litellm.proxy.admin_mcp import admin_mcp_lifespan, get_admin_mcp_permissions +from litellm.proxy.auth.user_api_key_auth import get_api_key, get_api_key_from_custom_header, user_api_key_auth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache -from litellm.proxy.management_endpoints import internal_user_endpoints +from litellm.proxy.management_endpoints import internal_user_endpoints, team_callback_endpoints from litellm.proxy.middleware.admission_control_middleware import ( AdmissionControlMiddleware, AdmissionControlSettings, @@ -66,12 +67,22 @@ def management_app(monkeypatch: pytest.MonkeyPatch) -> FastAPI: if credential not in {"admin-a", "admin-b", "admin-b-limited", "member", "viewer"}: raise HTTPException(status_code=401) user_id: Final = "admin-b" if credential == "admin-b-limited" else credential + role: Final = {"member": LitellmUserRoles.INTERNAL_USER, "viewer": LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY}.get( + user_id, LitellmUserRoles.PROXY_ADMIN + ) + auth: Final = UserAPIKeyAuth( + user_id=user_id, user_role=role, + allowed_routes=["/user/info"] if credential == "admin-b-limited" else None, + ) return { "user_id": user_id, "user_info": { "user_id": user_id, - "user_role": {"member": "internal_user", "viewer": "proxy_admin_viewer"}.get(user_id, "proxy_admin"), + "user_role": role.value, }, + "admin_mcp_permissions": get_admin_mcp_permissions( + auth, request, request.query_params.get("mcp_tool"), request.query_params.get("mcp_path") + ).model_dump(mode="json"), } @app.get("/key/list", operation_id="list_keys_key_list_get") @@ -565,3 +576,168 @@ async def test_oauth2_proxy_auth_preserves_native_identity_and_peer_trust( assert response.status_code == status, response.text if status == 200: assert "list_keys" in {tool["name"] for tool in response.json()["result"]["tools"]} + + +@pytest.fixture +def user_info_http_app(monkeypatch: pytest.MonkeyPatch, grant: str) -> FastAPI: + class EmptyDatabase: + async def query_raw(self, _: str) -> tuple[dict[str, None], ...]: + return ({"teams": None, "keys": None},) + + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=EmptyDatabase())) + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", grant]) + app: Final = FastAPI() + app.include_router(internal_user_endpoints.router) + app.include_router(team_callback_endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: auth + return app + + +@pytest.fixture +def permissions_request() -> Request: + return Request({"type": "http", "method": "GET", "path": "/user/info", "headers": [], "app": proxy_server.app}) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize( + "routes,session,can_write", + [ + (None, False, True), + ([], False, True), + (["management_routes"], False, True), + (["info_routes"], False, False), + (["llm_api_routes"], False, False), + (None, True, True), + (["info_routes"], True, False), + ], +) +def test_admin_mcp_uses_credential_routes_without_removing_full_access( + routes: list[str] | None, session: bool, can_write: bool, permissions_request: Request +) -> None: + auth: Final = UserAPIKeyAuth( + user_id="admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + allowed_routes=routes, + is_session_token=session, + ) + permissions: Final = get_admin_mcp_permissions(auth, permissions_request, "create_key", "/key/generate") + assert permissions.action_allowed is can_write + assert ("create_key" in permissions.tools) is can_write + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize( + "team_id,grant", + [ + ("team-a", "/team/team-a/callback"), + ("team-a", "/team/team-a/*"), + ("org/team-a", "/team/org/team-a/callback"), + ("org/team-a", "/team/org/team-a/*"), + ("org/team-a", "/team/org/team-a/call*"), + ], +) +def test_admin_mcp_checks_the_concrete_resource_even_when_the_tool_is_visible( + team_id: str, grant: str, permissions_request: Request +) -> None: + auth: Final = UserAPIKeyAuth( + user_id="admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + allowed_routes=["/user/info", grant], + metadata={"is_session_token": True, "user_role": "proxy_admin"}, + ) + own: Final = get_admin_mcp_permissions(auth, permissions_request, "get_team_callbacks", f"/team/{team_id}/callback") + other: Final = get_admin_mcp_permissions(auth, permissions_request, "get_team_callbacks", "/team/other/callback") + wrong_route: Final = get_admin_mcp_permissions(auth, permissions_request, "get_team_callbacks", "/key/generate") + assert "get_team_callbacks" in own.tools + assert own.action_allowed is True + assert other.action_allowed is False + assert wrong_route.action_allowed is False + + +@pytest.mark.asyncio +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize( + "grant,path,allowed", + [ + ("/team/team a/callback", "/team/team%20a/callback", True), + ("/team/org/team-a/callback", "/team/org%2Fteam-a/callback", True), + ("/team/team%2Fa/callback", "/team/team%252Fa/callback", True), + ("/team/team+a/callback", "/team/team+a/callback", True), + ("/team/team-é/callback", "/team/team-%C3%A9/callback", True), + ("/team/team/a/callback", "/team/team%252Fa/callback", False), + ("/team/team a/callback", "/team/team+a/callback", False), + ("/team/team%41/callback", "/team/team%2541/callback", True), + ], +) +async def test_user_info_decodes_mcp_path_after_query_parsing( + user_info_http_app: FastAPI, + path: str, + allowed: bool, +) -> None: + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=user_info_http_app), + base_url="http://testserver", + params={"admin_mcp": "true", "mcp_tool": "get_team_callbacks"}, + ) as client: + own: Final = await client.get("/user/info", params={"mcp_path": path}) + sibling: Final = await client.get("/user/info", params={"mcp_path": "/team/other/callback"}) + assert own.status_code == 200, own.text + assert sibling.status_code == 200, sibling.text + assert "get_team_callbacks" in own.json()["admin_mcp_permissions"]["tools"] + assert own.json()["admin_mcp_permissions"]["action_allowed"] is allowed + assert sibling.json()["admin_mcp_permissions"]["action_allowed"] is False + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize("grant", ["/te?m/*", "/[t]eam/*", "/te*m/*"]) +def test_admin_mcp_does_not_invent_glob_permissions(grant: str, permissions_request: Request) -> None: + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", grant]) + permissions: Final = get_admin_mcp_permissions( + auth, permissions_request, "get_team_callbacks", "/team/team-a/callback" + ) + assert "get_team_callbacks" not in permissions.tools + assert permissions.action_allowed is False + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +def test_admin_mcp_discovery_respects_disabled_management_routes( + permissions_request: Request, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setenv("DISABLE_ADMIN_ENDPOINTS", "true") + auth: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", "/key/key-a/reset_spend"] + ) + permissions: Final = get_admin_mcp_permissions( + auth, permissions_request, "reset_key_spend", "/key/key-a/reset_spend" + ) + assert permissions.tools == () + assert permissions.action_allowed is False + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +def test_admin_mcp_preserves_http_prefix_grants(permissions_request: Request) -> None: + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", "/key/list"]) + own: Final = get_admin_mcp_permissions(auth, permissions_request, "reset_key_spend", "/key/list/reset_spend") + other: Final = get_admin_mcp_permissions(auth, permissions_request, "reset_key_spend", "/key/other/reset_spend") + assert "reset_key_spend" in own.tools + assert own.action_allowed is True + assert other.action_allowed is False + + +@pytest.mark.asyncio +async def test_user_info_explains_missing_admin_mcp_dependency( + permissions_request: Request, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(sys.modules, "litellm_admin_mcp.catalog", None) + auth: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(ProxyException, match="admin-mcp dependency group") as exc: + await internal_user_endpoints.user_info( + request=permissions_request, + user_id=None, + user_api_key_dict=auth, + admin_mcp=True, + mcp_tool=None, + mcp_path=None, + ) + assert exc.value.code == "503" diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index 4c0b3ad913b..738fe9d53df 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -468,7 +468,10 @@ def test_admin_mcp_survives_only_management_component_lifespans( @application.get("/user/info") async def user_info() -> dict[str, object]: - return {"user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}} + return { + "user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}, + "admin_mcp_permissions": {"tools": ["list_teams"], "action_allowed": True}, + } if component_lifespan is not None: application.router.lifespan_context = partial(component_lifespan, lifespan=application.router.lifespan_context) diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index c835e1e27ea..139cfbbe58c 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -147,7 +147,10 @@ async def test_proxy_shutdown_drains_active_admin_tool_before_closing_connector( @admin_mcp_proxy.get("/user/info") async def user_info() -> dict[str, object]: - return {"user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}} + return { + "user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}, + "admin_mcp_permissions": {"tools": ["list_teams"], "action_allowed": True}, + } @admin_mcp_proxy.get("/team/list", operation_id="list_team_team_list_get") async def list_teams() -> dict[str, object]: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90a0597cea5..13a064517a4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25710,6 +25710,13 @@ export interface components { */ source: "admin"; }; + /** AdminMCPPermissions */ + AdminMCPPermissions: { + /** Action Allowed */ + action_allowed?: boolean | null; + /** Tools */ + tools: string[]; + }; /** * AgentCapabilities * @description Defines optional capabilities supported by an agent. @@ -49920,6 +49927,7 @@ export interface components { }; /** UserInfoResponse */ UserInfoResponse: { + admin_mcp_permissions?: components["schemas"]["AdminMCPPermissions"] | null; /** Keys */ keys: unknown[]; /** Teams */ @@ -76732,6 +76740,12 @@ export interface operations { query?: { /** @description User ID in the request parameters */ user_id?: string | null; + /** @description Include the caller's current admin MCP route permissions */ + admin_mcp?: boolean; + /** @description Admin MCP tool to authorize for this caller */ + mcp_tool?: string | null; + /** @description URL-encoded concrete management path used by the MCP tool, before query-string encoding */ + mcp_path?: string | null; }; header?: never; path?: never; diff --git a/uv.lock b/uv.lock index 87b8fc95486..8cd31eb0828 100644 --- a/uv.lock +++ b/uv.lock @@ -5334,7 +5334,7 @@ requires-dist = [ provides-extras = ["proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] -admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }] +admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" }] benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, @@ -5431,7 +5431,7 @@ proxy-dev = [ { name = "azure-identity", specifier = "==1.25.2" }, { name = "backports-zstd", marker = "python_full_version < '3.14'", specifier = "==1.8.0" }, { name = "hypercorn", specifier = "==0.17.3" }, - { name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }, + { name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" }, { name = "opentelemetry-api", specifier = "==1.45.1" }, { name = "opentelemetry-exporter-otlp", specifier = "==1.45.1" }, { name = "opentelemetry-instrumentation-fastapi", specifier = "==0.66b1" }, @@ -5444,14 +5444,14 @@ proxy-dev = [ [[package]] name = "litellm-admin-mcp" version = "0.1.0" -source = { url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" } +source = { url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" } dependencies = [ { name = "httpx2" }, { name = "jsonschema" }, { name = "mcp" }, { name = "uvicorn" }, ] -sdist = { hash = "sha256:86b930f6706fb2da10d53d1798ee7fd14e44fb0afdef0da122cf0e6bb88fa2c3" } +sdist = { hash = "sha256:5803a20a00a18177f4591706956f31be311879543049b8b38143ad90bb1cb0a6" } [package.metadata] requires-dist = [ From 7aa0393cb67dc94c1c71a58ebb192f6d34d08fbb Mon Sep 17 00:00:00 2001 From: bap1106 Date: Fri, 30 Oct 2026 09:05:19 +0700 Subject: [PATCH 13/34] feat(providers): add CLF AI Gateway as an OpenAI-compatible provider with ten models (#39324) * feat(providers): add CLF AI Gateway (clf_ai_gateway), OpenAI-compatible, 9 models OpenAI-compatible gateway at https://api.clfaigateway.dev/v1 serving open-weight models (GLM, Kimi, DeepSeek, Qwen) on Cloudflare Workers AI upstream. Follows the DeepInfra pattern: ClfAiGatewayConfig(OpenAIGPTConfig), CLF_AI_GATEWAY_API_KEY / _API_BASE, provider registered in enum, provider lists, endpoint inference, lazy-import registry, ProviderConfigManager, validate_environment. Supported params deliberately omit functions/function_call/logit_bias (gateway returns 400 for them). 9 cost-map entries (root + backup) with measured context/output limits and currently billed prices; endpoint-support entry; README row. Mocked tests only. Co-Authored-By: Claude Fable 5 * fix(clf_ai_gateway): declare each model's real reasoning_effort levels The per-level supports_*_reasoning_effort flags cannot express the sets this gateway actually takes, so all nine models advertised a wrong one: every model offered "minimal" and eight offered "none", neither of which the gateway accepts, five dropped "xhigh", which it does accept, and qwen3.8-27b offered "high", which it rejects. That set surfaces through /model_group/info and the model picker, so a user picking a level off it gets a 400 from the gateway. Declare reasoning_effort_levels instead, taken from the gateway's own GET /v1/public/models, which resolve_supported_reasoning_efforts reads first and uses whole. Declaring it also stops clf_ai_gateway/deepseek-v4-pro and clf_ai_gateway/deepseek-v4-flash inheriting effort metadata from the unprefixed deepseek entries of the same name. * test(clf_ai_gateway): name the gateway-rejected params instead of commenting them * fix(clf_ai_gateway): list the provider in the proxy Add Model form main now requires every LlmProviders value to have an entry in provider_create_fields.json (or be frozen as unlisted), so the Add Model dropdown cannot silently miss a backend provider. Add the clf_ai_gateway entry in the same shape as the SCX.ai registration: required API key, optional API base with https://api.clfaigateway.dev/v1 as placeholder, and a cost-map model as the default model placeholder. A provider test pins the entry. Co-Authored-By: Claude Opus 5 * fix(clf_ai_gateway): sync prices and reasoning_effort levels with the live catalog CLF AI Gateway raised its launch discount from 40% to 60% off list on 2026-09-26. It also re-measured every model's reasoning_effort levels against the current Workers AI behaviour. - input/output/cached per-token prices for all 9 models now match GET https://api.clfaigateway.dev/v1/public/models to the nano; qwen3.8-27b cached input also follows the upstream cached rate published on 2026-09-25 - reasoning_effort_levels: add "max" to deepseek-v4-flash, deepseek-v4-pro, glm-5.2 and glm-5.3-flash; add "none" to kimi-k2.6 (it really disables thinking there); drop "none" from glm-5.3 (upstream runs it as "max") Context windows, output limits and vision flags are unchanged and were checked against the same endpoint. Co-Authored-By: Claude Opus 5.5 * fix(clf_ai_gateway): drop a rebind-ok that no longer suppresses anything After rebasing onto main, the type-discipline gate's new LIT013 rule flags the `# rebind-ok` on the endpoint-inference `dynamic_api_key` assignment. The rebind rule no longer fires on that line, so the suppression is inert. The sibling endpoint branches carry no comment there either. Co-Authored-By: Claude Opus 5.5 * test(clf_ai_gateway): expect the re-measured reasoning_effort levels Follows the catalog sync: max on deepseek-v4-flash, deepseek-v4-pro, glm-5.2 and glm-5.3-flash, none on kimi-k2.6, and no none on glm-5.3. Co-Authored-By: Claude Opus 5.5 * test(clf_ai_gateway): move the provider tests to tests/unit/llms main moved every per-provider test from tests/test_litellm/llms// to tests/unit/llms// (DeepInfra lives at tests/unit/llms/deepinfra/). This branch still added the old path, so no CI shard claimed it. That failed assert-ci-coverage and assert-shard-coverage, and the file never ran (hence the low codecov/patch). Co-Authored-By: Claude Opus 5.5 * test(clf_ai_gateway): assert routing, billing, and dropped params instead of pinned gateway facts * fix(clf_ai_gateway): keep the type-discipline gate at its merge-base count * chore(clf_ai_gateway): regenerate provider endpoint support * ci(clf_ai_gateway): run the provider tests in the llms unit shard * fix(clf_ai_gateway): drop the retired max_tokens field from the cost map rows * fix(clf_ai_gateway): forward params the gateway accepts, drop undeclared reasoning levels, add deepseek-v4.1-flash * fix(clf_ai_gateway): drop the unreachable non-chat supported-params branch * fix(clf_ai_gateway): send a dict reasoning_effort as its level * fix(clf_ai_gateway): refuse a reasoning_effort dict without a level unless drop_params drops it --------- Co-authored-by: bap1106 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Claude Opus 5 --- .github/workflows/test-unit.yml | 1 + README.md | 1 + litellm/__init__.py | 9 + litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 3 + .../get_llm_provider_logic.py | 9 + .../clf_ai_gateway/chat/transformation.py | 95 +++++++ ...odel_prices_and_context_window_backup.json | 266 ++++++++++++++++++ .../provider_endpoints_support_backup.json | 26 ++ .../provider_create_fields.json | 28 ++ litellm/types/utils.py | 1 + litellm/utils.py | 6 + model_prices_and_context_window.json | 266 ++++++++++++++++++ provider_endpoints_support.json | 26 ++ tests/unit/llms/clf_ai_gateway/__init__.py | 0 ...test_clf_ai_gateway_chat_transformation.py | 251 +++++++++++++++++ 16 files changed, 993 insertions(+) create mode 100644 litellm/llms/clf_ai_gateway/chat/transformation.py create mode 100644 tests/unit/llms/clf_ai_gateway/__init__.py create mode 100644 tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 9ebb9bab018..9570b4d8cc8 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -245,6 +245,7 @@ jobs: tests/unit/llms/chatgpt tests/unit/llms/clarifai tests/unit/llms/claude_code + tests/unit/llms/clf_ai_gateway tests/unit/llms/cloudflare tests/unit/llms/codestral tests/unit/llms/codex diff --git a/README.md b/README.md index 531a50e1aca..3f1f21f903e 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | | | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | | | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | | +| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | | | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | | | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | | [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index 084a4c516ce..d59edcaff89 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -633,6 +633,7 @@ fireworks_ai_models: Set = set() baseten_models: Set[str] = set() fireworks_ai_embedding_models: Set = set() deepinfra_models: Set = set() +clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets perplexity_models: Set = set() watsonx_models: Set = set() gemini_models: Set = set() @@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje bedrock_converse_models.add(key) elif value.get("litellm_provider") == "deepinfra": deepinfra_models.add(key) + elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map + clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key elif value.get("litellm_provider") == "perplexity": perplexity_models.add(key) elif value.get("litellm_provider") == "watsonx": @@ -1110,6 +1113,7 @@ model_list = list( | set(ollama_models) | bedrock_models | deepinfra_models + | clf_ai_gateway_models | perplexity_models | set(maritalk_models) | runwayml_models @@ -1235,6 +1239,7 @@ def _build_models_by_provider() -> dict: "ollama": ollama_models, "ollama_chat": ollama_models, "deepinfra": deepinfra_models, + "clf_ai_gateway": clf_ai_gateway_models, "perplexity": perplexity_models, "maritalk": maritalk_models, "watsonx": watsonx_models, @@ -2111,6 +2116,9 @@ if TYPE_CHECKING: LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig, ) from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig + from .llms.clf_ai_gateway.chat.transformation import ( + ClfAiGatewayConfig as _ClfAiGatewayConfig, + ) from .llms.llamafile.chat.transformation import ( LlamafileChatConfig as _LlamafileChatConfig, ) @@ -2140,6 +2148,7 @@ if TYPE_CHECKING: IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig] LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig] DeepInfraConfig: Type[_DeepInfraConfig] + ClfAiGatewayConfig: type[_ClfAiGatewayConfig] LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 7e81602fe77..c1f484f8d64 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -245,6 +245,7 @@ LLM_CONFIG_NAMES: Final = ( "MistralEmbeddingConfig", "OpenAIImageVariationConfig", "DeepInfraConfig", + "ClfAiGatewayConfig", "DeepgramAudioTranscriptionConfig", "TopazImageVariationConfig", "OpenAITextCompletionConfig", @@ -975,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OpenAIImageVariationConfig", ), "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"), + "ClfAiGatewayConfig": ( + ".llms.clf_ai_gateway.chat.transformation", + "ClfAiGatewayConfig", + ), "DeepgramAudioTranscriptionConfig": ( ".llms.deepgram.audio_transcription.transformation", "DeepgramAudioTranscriptionConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 466942d35be..e07d727973d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -819,6 +819,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "ollama", "ollama_chat", "deepinfra", + "clf_ai_gateway", "perplexity", "mistral", "groq", @@ -1018,6 +1019,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "api.perplexity.ai", "api.endpoints.anyscale.com/v1", "api.deepinfra.com/v1/openai", + "api.clfaigateway.dev/v1", "api.mistral.ai/v1", "codestral.mistral.ai/v1/chat/completions", "codestral.mistral.ai/v1/fim/completions", @@ -1089,6 +1091,7 @@ openai_compatible_providers: Final[list[str]] = [ "deepseek", "tencent", "deepinfra", + "clf_ai_gateway", "perplexity", "xinference", "xai", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f8c8ff71f8e..38242291b71 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -292,6 +292,10 @@ def get_llm_provider( elif endpoint == "api.deepinfra.com/v1/openai": custom_llm_provider = "deepinfra" dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY") + elif endpoint == "api.clfaigateway.dev/v1": + custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch + clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY") + dynamic_api_key = api_key or clf_env_key elif endpoint == "api.mistral.ai/v1": custom_llm_provider = "mistral" dynamic_api_key = get_secret_str("MISTRAL_API_KEY") @@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "clf_ai_gateway": + ( + api_base, # rebind-ok: resolved base/key are what this function returns, like every branch + dynamic_api_key, # rebind-ok: see above + ) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "empower": api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1" dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY") diff --git a/litellm/llms/clf_ai_gateway/chat/transformation.py b/litellm/llms/clf_ai_gateway/chat/transformation.py new file mode 100644 index 00000000000..25e5d0f51f7 --- /dev/null +++ b/litellm/llms/clf_ai_gateway/chat/transformation.py @@ -0,0 +1,95 @@ +from typing import Final + +import litellm +from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model +from litellm.secret_managers.main import get_secret_str + +SUPPORTED_OPENAI_PARAMS: Final = ( + "stream", + "stream_options", + "frequency_penalty", + "presence_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "n", + "stop", + "temperature", + "top_p", + "seed", + "response_format", + "tools", + "tool_choice", + "parallel_tool_calls", + "user", + "store", + "service_tier", +) + + +class ClfAiGatewayConfig(OpenAIGPTConfig): + @property + def custom_llm_provider(self) -> str | None: + return "clf_ai_gateway" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: matches OpenAIGPTConfig's list return + supports_reasoning: Final = litellm.supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ) + extra: Final = ("reasoning_effort",) if supports_reasoning else () + return [*SUPPORTED_OPENAI_PARAMS, *extra] + + def map_openai_params( + self, + non_default_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature + optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature + requested: Final = non_default_params.get("reasoning_effort") + requested_mapping: Final = as_str_mapping(requested) + effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested + declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway") + drop_enabled: Final = drop_params or litellm.drop_params is True + if requested_mapping is not None and effort is None and not drop_enabled: + raise UnsupportedParamsError( + message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop " + "unsupported openai params from the call, set `litellm.drop_params = True`", + llm_provider="clf_ai_gateway", + model=model, + status_code=400, + ) + drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared + if drops_effort: + verbose_logger.warning( + "Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.", + effort, + model, + declared, + ) + others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"} + normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others + forwarded: Final = others if drops_effort else normalized + mapped: Final[dict[str, object]] = super().map_openai_params( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # OpenAIGPTConfig types its params and result as bare dicts # mutable-ok: parent returns a dict + non_default_params=forwarded, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + return mapped + + def get_openai_compatible_provider_info( + self, api_base: str | None, api_key: str | None + ) -> tuple[str | None, str | None]: + resolved_api_base: Final = ( + api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1" + ) + dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY") + return resolved_api_base, dynamic_api_key diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8f514a9dfb7..1eaac0fda24 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78638,6 +78638,272 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "clf_ai_gateway/glm-5.3": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.3-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 1.2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.2": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-4.7-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.4e-08, + "output_cost_per_token": 1.6e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-pro": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.28e-07, + "output_cost_per_token": 1.584e-06, + "cache_read_input_token_cost": 1.7e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.76e-07, + "output_cost_per_token": 5.28e-07, + "cache_read_input_token_cost": 5e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4.1-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 4.8e-07, + "cache_read_input_token_cost": 2e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.6": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 6.4e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.7-code": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 7.6e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/qwen3.8-27b": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.28e-06, + "cache_read_input_token_cost": 2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "xhigh" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, "kyma/claude-fable-5": { "max_input_tokens": 1000000, "max_output_tokens": 128000, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 18edb2cd7dc..d33e61026df 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -827,6 +827,32 @@ "generateContent": false } }, + "clf_ai_gateway": { + "display_name": "CLF AI Gateway (`clf_ai_gateway`)", + "url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "cloudflare": { "display_name": "Cloudflare AI Workers (`cloudflare`)", "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d8ff32c3819..df3af4ad487 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -931,6 +931,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "CLF_AI_GATEWAY", + "provider_display_name": "CLF AI Gateway", + "litellm_provider": "clf_ai_gateway", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.clfaigateway.dev/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "clf_ai_gateway/glm-5.3" + }, { "provider": "CLOUDFLARE", "provider_display_name": "Cloudflare", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 181ed5ea6db..5905aaa3380 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4485,6 +4485,7 @@ class LlmProviders(str, Enum): OLLAMA = "ollama" OLLAMA_CHAT = "ollama_chat" DEEPINFRA = "deepinfra" + CLF_AI_GATEWAY = "clf_ai_gateway" PERPLEXITY = "perplexity" TYPESAFE = "typesafe" NACE = "nace" diff --git a/litellm/utils.py b/litellm/utils.py index 549590e1cf5..d951b80d49b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7859,6 +7859,11 @@ def validate_environment( keys_in_environment = True else: missing_keys.append("DEEPINFRA_API_KEY") + elif custom_llm_provider == "clf_ai_gateway": + if "CLF_AI_GATEWAY_API_KEY" in os.environ: + keys_in_environment = True # rebind-ok: the result accumulates across the provider chain + else: + missing_keys.append("CLF_AI_GATEWAY_API_KEY") elif custom_llm_provider == "featherless_ai": if "FEATHERLESS_AI_API_KEY" in os.environ: keys_in_environment = True @@ -9537,6 +9542,7 @@ class ProviderConfigManager: LlmProviders.OOBABOOGA: (lambda: litellm.OobaboogaConfig(), False), LlmProviders.OLLAMA_CHAT: (lambda: litellm.OllamaChatConfig(), False), LlmProviders.DEEPINFRA: (lambda: litellm.DeepInfraConfig(), False), + LlmProviders.CLF_AI_GATEWAY: (lambda: litellm.ClfAiGatewayConfig(), False), LlmProviders.PERPLEXITY: (lambda: litellm.PerplexityChatConfig(), False), LlmProviders.MISTRAL: (lambda: litellm.MistralConfig(), False), LlmProviders.CODESTRAL: (lambda: litellm.MistralConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8f514a9dfb7..1eaac0fda24 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78638,6 +78638,272 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "clf_ai_gateway/glm-5.3": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.3-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 1.2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.2": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-4.7-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.4e-08, + "output_cost_per_token": 1.6e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-pro": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.28e-07, + "output_cost_per_token": 1.584e-06, + "cache_read_input_token_cost": 1.7e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.76e-07, + "output_cost_per_token": 5.28e-07, + "cache_read_input_token_cost": 5e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4.1-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 4.8e-07, + "cache_read_input_token_cost": 2e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.6": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 6.4e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.7-code": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 7.6e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/qwen3.8-27b": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.28e-06, + "cache_read_input_token_cost": 2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "xhigh" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, "kyma/claude-fable-5": { "max_input_tokens": 1000000, "max_output_tokens": 128000, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 18edb2cd7dc..d33e61026df 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -827,6 +827,32 @@ "generateContent": false } }, + "clf_ai_gateway": { + "display_name": "CLF AI Gateway (`clf_ai_gateway`)", + "url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "cloudflare": { "display_name": "Cloudflare AI Workers (`cloudflare`)", "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", diff --git a/tests/unit/llms/clf_ai_gateway/__init__.py b/tests/unit/llms/clf_ai_gateway/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py b/tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py new file mode 100644 index 00000000000..ce1a1c1b870 --- /dev/null +++ b/tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py @@ -0,0 +1,251 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.clf_ai_gateway.chat.transformation import ClfAiGatewayConfig +from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + +DEFAULT_API_BASE: Final = "https://api.clfaigateway.dev/v1" +CHAT_URL: Final = f"{DEFAULT_API_BASE}/chat/completions" +REPO_ROOT: Final = Path(litellm.__file__).parent.parent + + +def _chat_response(usage: dict[str, object]) -> dict[str, object]: + return { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "glm-5.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}], + "usage": usage, + } + + +def _clf_rows(path: Path) -> dict[str, dict[str, object]]: + rows: Final = json.loads(path.read_text(encoding="utf-8")) + return {key: value for key, value in rows.items() if value.get("litellm_provider") == "clf_ai_gateway"} + + +def test_resolves_environment_and_explicit_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLF_AI_GATEWAY_API_KEY", "fake-environment-key") + monkeypatch.delenv("CLF_AI_GATEWAY_API_BASE", raising=False) + assert get_llm_provider(model="clf_ai_gateway/glm-5.3") == ( + "glm-5.3", + "clf_ai_gateway", + "fake-environment-key", + DEFAULT_API_BASE, + ) + assert get_llm_provider( + model="clf_ai_gateway/glm-5.3", api_key="fake-explicit-key", api_base="https://gateway.example/v1" + ) == ("glm-5.3", "clf_ai_gateway", "fake-explicit-key", "https://gateway.example/v1") + + monkeypatch.setenv("CLF_AI_GATEWAY_API_BASE", "https://env-gateway.example/v1") + assert get_llm_provider(model="clf_ai_gateway/glm-5.3")[3] == "https://env-gateway.example/v1" + + +def test_api_base_alone_selects_the_provider_and_keeps_an_explicit_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLF_AI_GATEWAY_API_KEY", "fake-environment-key") + _, provider, key, _ = get_llm_provider(model="glm-5.3", api_base=DEFAULT_API_BASE) + assert (provider, key) == ("clf_ai_gateway", "fake-environment-key") + + _, provider_with_key, explicit_key, _ = get_llm_provider( + model="glm-5.3", api_base=DEFAULT_API_BASE, api_key="fake-explicit-key" + ) + assert (provider_with_key, explicit_key) == ("clf_ai_gateway", "fake-explicit-key") + + +def test_chat_forwards_reasoning_effort_and_bills_cached_tokens(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, + json=_chat_response( + { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + "prompt_tokens_details": {"cached_tokens": 80}, + } + ), + ) + response: Final = litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "Say hello"}], + reasoning_effort="low", + ) + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer fake-test-key" + assert body["model"] == "glm-5.3" + assert body["reasoning_effort"] == "low" + assert response.choices[0].message.content == "hello" + rates: Final = litellm.model_cost["clf_ai_gateway/glm-5.3"] + expected: Final = ( + 20 * rates["input_cost_per_token"] + + 80 * rates["cache_read_input_token_cost"] + + 5 * rates["output_cost_per_token"] + ) + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("param", "value"), + [ + ("functions", [{"name": "lookup", "parameters": {"type": "object", "properties": {}}}]), + ("function_call", "auto"), + ], +) +def test_legacy_fields_the_gateway_rejects_are_refused_or_dropped( + local_model_cost_map: None, param: str, value: object +) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + **{param: value}, + ) + + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + drop_params=True, + **{param: value}, + ) + assert param not in json.loads(route.calls.last.request.content) + + +@pytest.mark.parametrize( + "params", + [ + {"logit_bias": {"50256": -100}}, + {"logprobs": True, "top_logprobs": 2}, + {"store": False}, + {"service_tier": "auto"}, + ], +) +def test_params_the_gateway_accepts_are_forwarded_without_drop_params( + local_model_cost_map: None, params: dict[str, object] +) -> None: + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + **params, + ) + body: Final = json.loads(route.calls.last.request.content) + assert {key: body.get(key) for key in params} == params + + +@pytest.mark.parametrize( + ("effort", "drop_params", "forwarded"), + [ + ("high", True, None), + ("high", False, "high"), + ("xhigh", True, "xhigh"), + ({"effort": "low", "summary": "auto"}, True, "low"), + ({"effort": "low", "summary": "auto"}, False, "low"), + ({"effort": "high", "summary": "auto"}, True, None), + ], +) +def test_drop_params_drops_a_reasoning_effort_level_the_model_does_not_declare( + local_model_cost_map: None, effort: str | dict[str, str], drop_params: bool, forwarded: str | None +) -> None: + assert "high" not in litellm.model_cost["clf_ai_gateway/qwen3.8-27b"]["reasoning_effort_levels"] + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + litellm.completion( + model="clf_ai_gateway/qwen3.8-27b", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort=effort, + drop_params=drop_params, + ) + assert json.loads(route.calls.last.request.content).get("reasoning_effort") == forwarded + + +def test_a_reasoning_effort_dict_without_a_level_is_refused_unless_dropped(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort={"summary": "auto"}, + ) + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort={"summary": "auto"}, + drop_params=True, + ) + assert route.call_count == 1 + assert "reasoning_effort" not in json.loads(route.calls.last.request.content) + + +def test_reasoning_effort_is_advertised_only_for_reasoning_models(local_model_cost_map: None) -> None: + config: Final = ClfAiGatewayConfig() + assert "reasoning_effort" in config.get_supported_openai_params(model="glm-5.3") + assert "reasoning_effort" not in config.get_supported_openai_params(model="not-in-the-cost-map") + assert litellm.get_supported_openai_params( + model="glm-5.3", custom_llm_provider="clf_ai_gateway" + ) == config.get_supported_openai_params(model="glm-5.3") + + +def test_cost_map_rows_match_the_backup_and_resolve_their_effort_levels() -> None: + rows: Final = _clf_rows(REPO_ROOT / "model_prices_and_context_window.json") + assert rows + assert rows == _clf_rows(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") + for key, row in rows.items(): + assert row["input_cost_per_token"] > 0, key + assert row["output_cost_per_token"] > 0, key + assert row["max_output_tokens"] <= row["max_input_tokens"], key + assert resolve_supported_reasoning_efforts({**row, "key": key}, deployment_is_mapped=True) == tuple( + row["reasoning_effort_levels"] + ), key + + +def test_validate_environment_reports_the_env_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CLF_AI_GATEWAY_API_KEY", raising=False) + missing: Final = litellm.validate_environment(model="clf_ai_gateway/glm-5.3") + assert missing["keys_in_environment"] is False + assert missing["missing_keys"] == ["CLF_AI_GATEWAY_API_KEY"] + + monkeypatch.setenv("CLF_AI_GATEWAY_API_KEY", "fake-environment-key") + present: Final = litellm.validate_environment(model="clf_ai_gateway/glm-5.3") + assert present == {"keys_in_environment": True, "missing_keys": []} + + +def test_add_model_form_entry_points_at_a_mapped_model_and_the_default_base(local_model_cost_map: None) -> None: + fields_path: Final = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + entries: Final = [ + entry + for entry in json.loads(fields_path.read_text(encoding="utf-8")) + if entry["litellm_provider"] == "clf_ai_gateway" + ] + assert len(entries) == 1 + assert entries[0]["default_model_placeholder"] in litellm.model_cost + fields: Final = {field["key"]: field for field in entries[0]["credential_fields"]} + assert fields["api_key"]["required"] is True + assert fields["api_base"]["required"] is False + assert fields["api_base"]["placeholder"] == DEFAULT_API_BASE From a7030a5a2cab8b1ff525e1e47f8e7bb741609d50 Mon Sep 17 00:00:00 2001 From: Gopal Kamineni <72813883+Gopal3746@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:12:30 -0400 Subject: [PATCH 14/34] fix(vertex_ai): batch Claude models in Anthropic's native row shape (#45715) * fix(vertex_ai): batch Claude models in Anthropic's native row shape Claude deployments on Vertex were filed under publishers/google/models/ when the batch input was uploaded, so create_batch asked Vertex for a Google model that doesn't exist and got a bare 404 on batchPredictionJobs. The rows were also written as Gemini GenerateContentRequests, which Vertex rejects for Claude even with the right model path Claude uploads now go under publishers/anthropic/models/ and each line becomes {"custom_id", "request": } using the existing Vertex Claude transform. Output rows in the Anthropic message shape are converted back to OpenAI batch rows (custom_id from the row, errors as failed rows), native rows are costed from Anthropic usage, and create_batch fails fast with a 400 when a Claude job targets the global location, which Vertex doesn't serve for partner model batches Fixes #45671 * test(vertex_ai): cover malformed Claude output rows and rows without custom_id * fix(vertex_ai): build Claude batch rows with chat's param mapping and surface Vertex batch create errors Claude batch rows now go through get_optional_params like Vertex Claude chat, so unsupported params, top_k, response_format and extra_body follow drop_params. Tool names Claude would rename get a 400 at upload. A refused batch create returns Vertex's own status and message. * fix(vertex_ai): price Claude batch output rows without a request echo, keep @ version ids, refuse a json_tool_call user tool, keep Vertex error headers * fix(vertex_ai): keep httpx.HTTPStatusError on Vertex batch create errors and apply deployment chat params to Claude batch rows * fix(vertex_ai): keep Vertex's status code on batch create errors through the proxy * fix(vertex_ai): keep Vertex's error body on batch create errors and require custom_id on Claude lines * fix(vertex_ai): keep the old batch create error text and Google object names A failed Vertex batch create keeps httpx's error line first and adds Vertex's body after it, and logs the status and body again. Only Claude object names keep the @ version, so Google models are stored under the same names as before --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/vertex_ai/batches/handler.py | 34 +- .../llms/vertex_ai/batches/transformation.py | 67 ++- .../llms/vertex_ai/files/transformation.py | 259 +++++++++-- tests/unit/batches/test_batch_utils.py | 30 ++ .../llms/vertex_ai/batches/test_handler.py | 47 +- .../vertex_ai/batches/test_transformation.py | 60 +++ .../test_vertex_ai_files_transformation.py | 431 +++++++++++++++++- 7 files changed, 858 insertions(+), 70 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 7c4238e3498..58f589fea61 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -15,8 +15,10 @@ from litellm.litellm_core_utils.url_utils import ( ) from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, + MaskedHTTPStatusError, get_async_httpx_client, get_httpx_client, + mask_sensitive_info, ) from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM @@ -81,6 +83,15 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str return response.json() +def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> MaskedHTTPStatusError: + body: Final = mask_sensitive_info(error.response.text) + return MaskedHTTPStatusError( + httpx.HTTPStatusError(f"{error}\n{body}", request=error.request, response=error.response), + message=body, + text=body, + ) + + class VertexAIBatchPrediction(VertexLLM): def __init__(self, gcs_bucket_name: str, *args, **kwargs): super().__init__(*args, **kwargs) @@ -166,11 +177,19 @@ class VertexAIBatchPrediction(VertexLLM): headers=headers, ) - response: Final = sync_handler.post( - url=api_base, - headers=headers, - data=json.dumps(vertex_batch_request), - ) + try: + response: Final = sync_handler.post( + url=api_base, + headers=headers, + data=json.dumps(vertex_batch_request), + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch create failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise _vertex_batch_create_error(e) from e payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()} _json_response: Final = payload_view["payload"] @@ -266,13 +285,12 @@ class VertexAIBatchPrediction(VertexLLM): data=json.dumps(vertex_batch_request), ) except httpx.HTTPStatusError as e: - error_body: Final = e.response.text litellm.verbose_logger.error( "Vertex AI batch create failed: status=%s, body=%s", e.response.status_code, - error_body[:1000], + e.response.text[:1000], ) - raise + raise _vertex_batch_create_error(e) from e payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()} _json_response: Final = payload_view["payload"] diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index 99ef750b5fc..99d78bcdca6 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -9,14 +9,41 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.llms.vertex_ai.common_utils import ( VertexAIError, + VertexAIModelRoute, convert_vertex_datetime_to_openai_datetime, + get_vertex_ai_model_route, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest from litellm.types.llms.vertex_ai import * -from litellm.types.llms.vertex_ai import GenerateContentResponseBody +from litellm.types.llms.vertex_ai import GenerateContentResponseBody, VertexPartnerProvider from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage _NATIVE_VERTEX_RESPONSE: Final = TypeAdapter(GenerateContentResponseBody) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +VERTEX_ANTHROPIC_VERSION: Final = "vertex-2023-10-16" +_ANTHROPIC_PUBLISHER_MODEL_PATH: Final = "publishers/anthropic/models/" + + +def is_vertex_anthropic_batch_model(model: str) -> bool: + """ + Whether a Vertex batch model (bare, `vertex_ai/`-prefixed or a publisher path) is a Claude model, decided the + way Vertex chat routes Claude. Vertex batches Claude only as `publishers/anthropic/models/` with rows in + Anthropic's Messages shape + """ + bare_model: Final = model.rsplit("/", 1)[-1] + return _ANTHROPIC_PUBLISHER_MODEL_PATH in model or ( + get_vertex_ai_model_route(bare_model) is VertexAIModelRoute.PARTNER_MODELS + and VertexPartnerProvider.claude.value in bare_model + ) + + +def as_json_object(value: object) -> Mapping[str, object] | None: + return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None + + +def is_vertex_anthropic_batch_output_response(response_body: Mapping[str, object]) -> bool: + return response_body.get("type") == "message" and isinstance(response_body.get("usage"), dict) def _int_field(mapping: Mapping[str, object], key: str) -> int: @@ -43,8 +70,22 @@ def is_vertex_embedding_batch_output_response(response_body: Mapping[str, object return isinstance(response_body.get("embedding"), dict) +def is_vertex_anthropic_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool: + """ + Whether a Vertex batch output row came from a Claude (Anthropic Messages) request. Vertex echoes only the + request's top-level `custom_id` on these rows, with an Anthropic `message` response on success and an empty + response plus a `status` on failure, so a Claude input file read back is never mistaken for output + """ + if not ("custom_id" in vertex_output_row and "processed_time" in vertex_output_row): + return False + response: Final = as_json_object(vertex_output_row.get("response")) + return (response is not None and is_vertex_anthropic_batch_output_response(response)) or bool( + vertex_output_row.get("status") + ) + + def is_native_vertex_batch_output_row(row: Mapping[str, object]) -> bool: - return isinstance(row.get("request"), dict) + return isinstance(row.get("request"), dict) or is_vertex_anthropic_batch_output_row(row) class NativeVertexBatchCostCalculator(Protocol): @@ -72,6 +113,9 @@ def _native_vertex_row_usage( response_body: Mapping[str, object], calculate_usage: Callable[[GenerateContentResponseBody], Usage], ) -> Usage | None: + anthropic_usage: Final = as_json_object(response_body.get("usage")) + if is_vertex_anthropic_batch_output_response(response_body) and anthropic_usage is not None: + return VertexAIAnthropicConfig().calculate_usage(usage_object=anthropic_usage, reasoning_content=None) if "usageMetadata" not in response_body: if not is_vertex_embedding_batch_output_response(response_body): return None @@ -97,21 +141,24 @@ def native_vertex_batch_row_stats( """ Usage and cost of one native Vertex predictions.jsonl row, a `{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}, "modelVersion": ...}}` - generateContent object or a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}` - embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`). - `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` - does, else the wildcard name so explicit deployment prices still apply; a row without a response, a - generateContent row without `response.usageMetadata`, and a row whose response fails validation are + generateContent object, a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}` + embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`) or a + `{"custom_id": ..., "request": ..., "response": {"type": "message", "model": ..., "usage": {...}}}` Claude object. + `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` (Claude + rows: `model`) does, else the wildcard name so explicit deployment prices still apply; a row without a response, + a generateContent row without `response.usageMetadata`, and a row whose response fails validation are None (failed). """ - response_body: Final = row.get("response") - if not isinstance(response_body, dict): + response_body: Final = as_json_object(row.get("response")) + if response_body is None: return None usage: Final = _native_vertex_row_usage(response_body, calculate_usage) if usage is None: return None total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens) - model_version: Final = response_body.get("modelVersion") + model_version: Final = response_body.get( + "model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion" + ) deployment_model: Final = model_name if model_name and "*" not in model_name else None model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name) if model is None: diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 410f29ef88a..9159c9a4070 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._uuid import uuid +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, RESPONSE_FORMAT_TOOL_NAME from litellm.files.types import FileContentStreamingResult from litellm.files.utils import FilesAPIUtils from litellm.litellm_core_utils.cloud_storage_security import ( @@ -32,13 +33,14 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, extract_file_metadata, ) from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body @@ -47,7 +49,14 @@ from litellm.llms.base_llm.files.transformation import ( BaseFileUploadStream, LiteLLMLoggingObj, ) -from litellm.llms.vertex_ai.batches.transformation import vertex_embedding_prompt_token_count +from litellm.llms.vertex_ai.batches.transformation import ( + VERTEX_ANTHROPIC_VERSION, + as_json_object, + is_vertex_anthropic_batch_model, + is_vertex_anthropic_batch_output_response, + is_vertex_anthropic_batch_output_row, + vertex_embedding_prompt_token_count, +) from litellm.llms.vertex_ai.common_utils import ( convert_vertex_datetime_to_openai_datetime, get_vertex_ai_fine_tuned_endpoint_id, @@ -59,6 +68,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( transform_openai_input_gemini_embed_content, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.files import StreamingMediaUploadConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -79,6 +89,7 @@ from litellm.types.utils import ( ModelResponse, Usage, ) +from litellm.utils import get_non_default_completion_params, get_optional_params from ..common_utils import VertexAIError from ..vertex_llm_base import VertexBase @@ -87,6 +98,7 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63 _CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_" _VERTEX_BATCH_KEY_FIELD: Final = "key" _MANAGED_GCS_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/([^/?]+)") +_PUBLISHER_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/") _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = ( ("outputDimensionality", "output_dimensionality"), ("taskType", "task_type"), @@ -225,10 +237,14 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, object]) -> str """ Resolve the OpenAI `custom_id` for a Vertex batch output row. - Embedding rows carry it in the top-level `key` field that Vertex echoes back; - `generateContent` rows have no such field, so it is smuggled through request - labels instead (see `_set_litellm_batch_custom_id_labels`). + Claude rows carry it in their own top-level `custom_id` field and embedding rows in + the top-level `key` field that Vertex echoes back; `generateContent` rows have no + such field, so it is smuggled through request labels instead (see + `_set_litellm_batch_custom_id_labels`). """ + anthropic_custom_id: Final = vertex_output_row.get("custom_id") + if anthropic_custom_id is not None: + return str(anthropic_custom_id) key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) if key is not None: return unquote(str(key)) @@ -289,6 +305,12 @@ def _is_vertex_generate_content_batch_output_row(vertex_output_row: Mapping[str, ) +def _is_vertex_chat_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool: + return _is_vertex_generate_content_batch_output_row(vertex_output_row) or is_vertex_anthropic_batch_output_row( + vertex_output_row + ) + + def _try_parse_vertex_batch_output_row(line: bytes) -> _VertexBatchRow | None: try: row: Final = _parse_vertex_batch_output_row(line.decode("utf-8")) @@ -410,6 +432,37 @@ def _openai_batch_output_row( } +def _vertex_anthropic_batch_output_row_to_openai( + custom_id: str, + response: object, + mock_httpx_response: httpx.Response, +) -> _OpenAIBatchOutputRow: + """ + One OpenAI batch output row from a Vertex Claude row's Anthropic `message` response + """ + anthropic_response: Final = as_json_object(response) + if anthropic_response is None or not is_vertex_anthropic_batch_output_response(anthropic_response): + return _openai_batch_output_row( + custom_id=custom_id, + error_code="vertex_ai_error", + error_message="Claude batch row has no message response", + ) + try: + model_response: Final = VertexAIAnthropicConfig().transform_parsed_response( + completion_response=dict(anthropic_response), + raw_response=mock_httpx_response, + model_response=ModelResponse(), + json_mode=True, + ) + except Exception as e: # noqa: BLE001 # one malformed row must fail alone, not the whole output file + return _openai_batch_output_row( + custom_id=custom_id, + error_code="transformation_error", + error_message=f"Failed to transform response: {e}", + ) + return _openai_batch_output_row(custom_id=custom_id, body=model_response.model_dump()) + + def _split_vertex_batch_key(vertex_output_row: Mapping[str, object]) -> tuple[str, int, int]: """ Resolve `(custom_id, index within that custom_id, group size)` for a Vertex batch @@ -684,21 +737,123 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset( + ("model", "messages", "custom_llm_provider", "drop_params", "stream", "stream_options") +) +_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"} +_CLAUDE_BATCH_DEPLOYMENT_PARAMS: Final = ( + _CLAUDE_BATCH_OPENAI_PARAMS - _CLAUDE_BATCH_ROW_FIXED_PARAMS - {"extra_headers"} +) | {"drop_params"} + + +def _claude_batch_deployment_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + {key: value for key, value in litellm_params.items() if key in _CLAUDE_BATCH_DEPLOYMENT_PARAMS} + ) + + +def _names_the_response_format_tool(tool: object) -> bool: + match tool: + case {"function": {"name": str(name)}}: + return name == RESPONSE_FORMAT_TOOL_NAME + case _: + return False + + +def _openai_batch_jsonl_entry_to_vertex_anthropic_row( + custom_id: object, + messages: list[AllMessageValues], + request_params: Mapping[str, object], + model: str, + deployment_params: Mapping[str, object], +) -> Mapping[str, object]: + """ + Transforms an OpenAI chat batch entry into the Anthropic Messages row Vertex batches Claude with, + `{"custom_id": ..., "request": {"messages": [...], "anthropic_version": "vertex-2023-10-16", "max_tokens": ...}}` + """ + if custom_id is None: + raise ValueError("custom_id is required on Claude lines, since Claude output rows are matched back only by it") + config: Final = VertexAIAnthropicConfig() + bare_model: Final = model.rsplit("/", 1)[-1] + params: Final = {**deployment_params, **request_params} + drop_params: Final = drops_unsupported_params(params.get("drop_params")) + chat_params: Final = { + **get_non_default_completion_params(params), + **{key: value for key, value in params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS}, + } + if any(map(_names_the_response_format_tool, iter_items(chat_params.get("tools") or ()))): + raise ValueError( + f"tool name {RESPONSE_FORMAT_TOOL_NAME!r} is reserved for response_format on Claude, " + "and batch output cannot tell a user tool by that name apart, so rename it" + ) + optional_params: Final[Mapping[str, object]] = get_optional_params( + model=bare_model, + custom_llm_provider="vertex_ai", + drop_params=drop_params, + messages=messages, + **{ # pyright: ignore[reportArgumentType] # batch JSONL bodies are untyped; get_optional_params checks each param as chat does + key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS + }, + ) + request_state: Final[dict[str, object]] = {"drop_params": drop_params} + anthropic_request: Final[Mapping[str, object]] = config.transform_request( + model=bare_model, + messages=messages, + optional_params={ + **{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")}, + "anthropic_version": VERTEX_ANTHROPIC_VERSION, + }, + litellm_params=request_state, + headers={}, + ) + renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)) + if renamed_tools: + raise ValueError( + f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, " + "and batch output cannot map a renamed tool back, so rename these tools" + ) + extra_body: Final = as_json_object(optional_params.get("extra_body")) + request: Final[Mapping[str, object]] = ( + anthropic_request + if extra_body is None + else { + **anthropic_request, + **config.transform_extra_body( + extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={} + ), + } + ) + return {"custom_id": str(custom_id), "request": request} + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + batch_model: str | None = None, + deployment_params: Mapping[str, object] = MappingProxyType({}), ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. + `batch_model` is the deployment model the batch runs against and wins over the entry's own `body.model`, + which picks the row shape: Anthropic Messages for Claude, `GenerateContentRequest` otherwise. + jsonl body for vertex is {"request": } Example Vertex jsonl {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}} """ + entry_body: Final = openai_entry.get("body") + model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "") + is_anthropic_model: Final = is_vertex_anthropic_batch_model(model) if _is_embeddings_batch_entry(openai_entry): + if is_anthropic_model: + raise VertexAIError( + status_code=400, + message=f"Vertex AI batch embeddings are not supported for Claude models ('{model}')", + ) return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry) - if not isinstance(openai_entry.get("body"), dict): + if not isinstance(entry_body, dict): raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object") openai_request_body: Final = openai_entry["body"] chat_request_body: Final = ( @@ -706,8 +861,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", [])) + if is_anthropic_model: + return ( + _openai_batch_jsonl_entry_to_vertex_anthropic_row( + custom_id=openai_entry.get("custom_id"), + messages=chat_messages, + request_params={ + key: value for key, value in chat_request_body.items() if key not in ("model", "messages") + }, + model=model, + deployment_params=deployment_params, + ), + ) vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), + messages=chat_messages, model=chat_request_body.get("model", ""), optional_params=map_openai_to_vertex_params(chat_request_body), custom_llm_provider="vertex_ai", @@ -859,15 +1027,24 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + batch_model: str | None = None, + deployment_params: Mapping[str, object] = MappingProxyType({}), ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._batch_model = batch_model + self._deployment_params = deployment_params def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, + self._map_openai_to_vertex_params, + batch_model=self._batch_model, + deployment_params=self._deployment_params, + ) except Exception as e: raise VertexAIError( status_code=400, @@ -923,16 +1100,31 @@ class _RawFileUploadStream(BaseFileUploadStream): return _iter_raw_file_chunks(self._file_content) +def _deployment_batch_model(litellm_params: Mapping[str, object]) -> str | None: + model: Final = litellm_params.get("model") + return model.removeprefix("vertex_ai/") if isinstance(model, str) and model else None + + +def _publisher_model_path(raw_model: str) -> str: + if _PUBLISHER_MODEL_PATH_PATTERN.search(raw_model): + return raw_model + publisher: Final = "anthropic" if is_vertex_anthropic_batch_model(raw_model) else "google" + return f"publishers/{publisher}/models/{raw_model}" + + def _managed_batch_object_name(raw_model: str, *, passthrough: bool) -> str: endpoint_id: Final = get_vertex_ai_fine_tuned_endpoint_id(raw_model) - model_path: Final = ( - f"endpoints/{endpoint_id}" - if endpoint_id is not None - else (raw_model if "publishers/google/models" in raw_model else f"publishers/google/models/{raw_model}") - ) - safe_model_path: Final = sanitize_cloud_object_path(model_path, fallback="model") + model_path: Final = f"endpoints/{endpoint_id}" if endpoint_id is not None else _publisher_model_path(raw_model) prefix: Final = _PASSTHROUGH_MANAGED_GCS_PREFIX if passthrough else VERTEX_AI_MANAGED_GCS_PREFIX - return f"{prefix}{safe_model_path}/{uuid.uuid4()}" + return f"{prefix}{_safe_managed_model_path(model_path)}/{uuid.uuid4()}" + + +def _safe_managed_model_path(model_path: str) -> str: + if not is_vertex_anthropic_batch_model(model_path): + return sanitize_cloud_object_path(model_path, fallback="model") + return "@".join( + sanitize_cloud_object_path(version_part, fallback="model") for version_part in model_path.split("@") + ) class VertexAIFilesConfig(VertexBase, BaseFilesConfig): @@ -1036,10 +1228,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): fallback_filename="file", ) - def _get_configured_bucket_name(self, litellm_params: dict) -> str: + def _get_configured_bucket_name(self, litellm_params: Mapping[str, object]) -> str: + configured_bucket_name: Final = litellm_params.get("gcs_bucket_name") or litellm_params.get("bucket_name") bucket_name: Final = ( - litellm_params.get("gcs_bucket_name") - or litellm_params.get("bucket_name") + (configured_bucket_name if isinstance(configured_bucket_name, str) else None) or os.getenv("GCS_BATCH_BUCKET_NAME") or os.getenv("GCS_BUCKET_NAME") ) @@ -1053,7 +1245,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], data: CreateFileRequest, ) -> str: """ @@ -1077,11 +1269,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_deployment_batch_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1138,7 +1329,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1170,6 +1361,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params), + batch_model=_deployment_batch_model(litellm_params), + deployment_params=_claude_batch_deployment_params(litellm_params), ), content_type="application/json", ) @@ -1220,7 +1413,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException: return VertexAIError(status_code=status_code, message=error_message, headers=headers) - def _parse_gcs_uri(self, file_id: str, litellm_params: dict | None = None) -> tuple[str, str]: + def _parse_gcs_uri(self, file_id: str, litellm_params: Mapping[str, object] | None = None) -> tuple[str, str]: """ Validate a managed GCS file_id and return (bucket, url-encoded-object-path). """ @@ -1238,7 +1431,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_id: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}" @@ -1268,7 +1461,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_id: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}" @@ -1311,7 +1504,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_content_request, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: file_id: Final = file_content_request.get("file_id", "") bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) @@ -1408,7 +1601,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): headers=MappingProxyType({**headers, "content-length": str(len(transformed_content))}), ) - if not _is_vertex_generate_content_batch_output_row(first_row): + if not _is_vertex_chat_batch_output_row(first_row): return FileContentStreamingResult(stream_iterator=replayed_stream, headers=headers) return FileContentStreamingResult( @@ -1492,10 +1685,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): # first line is not valid UTF-8/JSON) raises and falls through to the # passthrough below, leaving the content untouched. first_row: Final = _parse_vertex_batch_output_row(first_line) - if not ( - _is_vertex_embeddings_batch_output_row(first_row) - or _is_vertex_generate_content_batch_output_row(first_row) - ): + if not (_is_vertex_embeddings_batch_output_row(first_row) or _is_vertex_chat_batch_output_row(first_row)): return content context: Final = _new_vertex_batch_output_row_transform_context() @@ -1560,6 +1750,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): error_message=status, ) + if is_vertex_anthropic_batch_output_row(vertex_output): + return _vertex_anthropic_batch_output_row_to_openai( + custom_id=custom_id, + response=vertex_output.get("response"), + mock_httpx_response=mock_httpx_response, + ) + # Transform successful response using existing transformation vertex_response: Final = vertex_output.get("response", {}) diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index f2bd5855d15..d154086aec9 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -2310,6 +2310,36 @@ async def test_native_vertex_rows_route_to_vertex_cost_path_without_flag(monkeyp assert {(call["model"], call["custom_llm_provider"]) for call in calls} == {("gemini-2.5-flash", "vertex_ai")} +@pytest.mark.asyncio +async def test_raw_vertex_claude_rows_are_priced_like_with_the_native_output_flag(monkeypatch): + claude_message: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "Blue."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 5}, + } + rows: Final = [ + {"custom_id": "ok", "processed_time": "2026-10-29T03:00:00Z", "response": claude_message, "status": ""}, + {"custom_id": "bad", "processed_time": "2026-10-29T03:00:00Z", "response": {}, "status": "Bad Request"}, + ] + + async def cost_with_flag(flag: bool) -> bu.BatchCostUsageResult: + monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", flag, raising=False) + return await bu.calculate_batch_cost_and_usage( + file_content_dictionary=rows, custom_llm_provider="vertex_ai", model_name="vertex_ai/claude-sonnet-4-6" + ) + + unflagged: Final = await cost_with_flag(False) + flagged: Final = await cost_with_flag(True) + + assert unflagged.cost > 0 + assert (unflagged.successful_requests, unflagged.failed_requests) == (1, 1) + assert (unflagged.cost, unflagged.usage.total_tokens) == (flagged.cost, flagged.usage.total_tokens) + + @pytest.mark.asyncio async def test_openai_shaped_vertex_rows_keep_the_generic_path_without_flag(monkeypatch): monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", False, raising=False) diff --git a/tests/unit/llms/vertex_ai/batches/test_handler.py b/tests/unit/llms/vertex_ai/batches/test_handler.py index 91a2adb9d19..2c246e065a8 100644 --- a/tests/unit/llms/vertex_ai/batches/test_handler.py +++ b/tests/unit/llms/vertex_ai/batches/test_handler.py @@ -30,6 +30,7 @@ from __future__ import annotations import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -41,9 +42,11 @@ from litellm.llms.vertex_ai.batches.handler import ( # noqa: E402 VertexAIBatchPrediction, ) from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 +from litellm.proxy.utils import handle_exception_on_proxy # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 HMOD = "litellm.llms.vertex_ai.batches.handler" +_CLIENT_ERROR_429: Final = "Client error '429 Too Many Requests' for url 'https://x/batchPredictionJobs'" TOKEN = "ya29.fake-access-token" PROJECT = "my-project" LOCATION = "us-central1" @@ -370,19 +373,19 @@ def test_create_batch_sync_endpoint_without_deployed_model_raises_400(): client.post.assert_not_called() -def test_create_batch_sync_httpstatuserror_propagates(): - """``HTTPHandler.post`` raises for non-2xx via ``raise_for_status``; the - sync create path must surface that error, not swallow it.""" +def test_create_batch_sync_httpstatuserror_carries_vertex_status_body_and_headers(): h = _make_handler() client = MagicMock() request = httpx.Request("POST", "https://x/batchPredictionJobs") - err_response = httpx.Response(status_code=500, request=request, text="boom") + err_response: Final = httpx.Response( + status_code=429, request=request, text="boom", headers={"retry-after": "30"} + ) client.post.side_effect = httpx.HTTPStatusError( - "boom", request=request, response=err_response + _CLIENT_ERROR_429, request=request, response=err_response ) with patch(f"{HMOD}.get_httpx_client", return_value=client): - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(httpx.HTTPStatusError) as exc_info: h.create_batch( _is_async=False, create_batch_data=CREATE_DATA, @@ -393,6 +396,11 @@ def test_create_batch_sync_httpstatuserror_propagates(): timeout=600.0, max_retries=None, ) + assert exc_info.value.response.status_code == 429 + assert exc_info.value.response.headers["retry-after"] == "30" + assert handle_exception_on_proxy(exc_info.value).code == "429" + assert (getattr(exc_info.value, "message", None), getattr(exc_info.value, "text", None)) == ("boom", "boom") + assert str(exc_info.value).splitlines() == [_CLIENT_ERROR_429, "boom"] def test_create_batch_input_file_id_without_model_raises_400_before_post(): @@ -800,14 +808,16 @@ def test_cancel_batch_sync_httpstatuserror_logged_and_reraised(): client.get.assert_not_called() -def test_create_batch_async_httpstatuserror_logged_and_reraised(): +def test_create_batch_async_httpstatuserror_carries_vertex_status_body_and_headers(): h = _make_handler() async_client = MagicMock() request = httpx.Request("POST", "https://x/batchPredictionJobs") - err_response = httpx.Response(status_code=500, request=request, text="boom") + err_response: Final = httpx.Response( + status_code=429, request=request, text="boom", headers={"retry-after": "30"} + ) async_client.post = AsyncMock( side_effect=httpx.HTTPStatusError( - "boom", request=request, response=err_response + _CLIENT_ERROR_429, request=request, response=err_response ) ) @@ -825,8 +835,13 @@ def test_create_batch_async_httpstatuserror_logged_and_reraised(): timeout=600.0, max_retries=None, ) - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(httpx.HTTPStatusError) as exc_info: _run(coro) + assert exc_info.value.response.status_code == 429 + assert exc_info.value.response.headers["retry-after"] == "30" + assert handle_exception_on_proxy(exc_info.value).code == "429" + assert (getattr(exc_info.value, "message", None), getattr(exc_info.value, "text", None)) == ("boom", "boom") + assert str(exc_info.value).splitlines() == [_CLIENT_ERROR_429, "boom"] def test_async_retrieve_batch_non_200_raises(): @@ -1043,15 +1058,7 @@ async def test_vertex_list_batches(monkeypatch): @pytest.mark.asyncio -async def test_vertex_async_create_batch_logs_error_body_on_http_error(): - """ - When Vertex AI returns an HTTP error (e.g. 400), _async_create_batch should - re-raise httpx.HTTPStatusError (not swallow it) and log the response body. - - Before the fix the error body was lost because AsyncHTTPHandler.post() - calls raise_for_status() internally, raising before the handler's own - status-code check could log the body. - """ +async def test_vertex_async_create_batch_httpstatuserror_carries_vertex_error_body(): from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") @@ -1081,4 +1088,4 @@ async def test_vertex_async_create_batch_logs_error_body_on_http_error(): ) assert exc_info.value.response.status_code == 400 - assert "gemini-2.0-flash" in exc_info.value.response.text + assert "Do not support publisher model gemini-2.0-flash" in str(exc_info.value) diff --git a/tests/unit/llms/vertex_ai/batches/test_transformation.py b/tests/unit/llms/vertex_ai/batches/test_transformation.py index 064845a7266..1440a57a15f 100644 --- a/tests/unit/llms/vertex_ai/batches/test_transformation.py +++ b/tests/unit/llms/vertex_ai/batches/test_transformation.py @@ -21,6 +21,8 @@ import pytest from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402 VertexAIBatchTransformation, + is_vertex_anthropic_batch_model, + native_vertex_batch_row_stats, ) from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 @@ -493,3 +495,61 @@ def test_get_gcs_uri_prefix_keeps_passthrough_segment_so_output_lands_beside_inp T._get_gcs_uri_prefix_from_file(PASSTHROUGH_INPUT_FILE) == "gs://litellm-testing-bucket/litellm-vertex-files/passthrough/publishers/google/models/gemini-2.5-flash" ) + + +CLAUDE_INPUT_FILE = ( + "gs://litellm-testing-bucket/litellm-vertex-files/publishers/anthropic/models/claude-sonnet-4-6/uuid-1" +) + + +def test_claude_batch_job_targets_the_anthropic_publisher_model(): + job: Final = T.transform_openai_batch_request_to_vertex_ai_batch_request( + {"input_file_id": CLAUDE_INPUT_FILE}, vertex_project="p", vertex_location="us-east5" + ) + assert job["model"] == "publishers/anthropic/models/claude-sonnet-4-6" + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("claude-sonnet-4-6", True), + ("vertex_ai/claude-opus-4-1@20250805", True), + ("publishers/anthropic/models/claude-haiku-4-5", True), + ("gemini-2.5-flash", False), + ("publishers/google/models/gemini-2.5-flash", False), + ], +) +def test_is_vertex_anthropic_batch_model(model: str, expected: bool): + assert is_vertex_anthropic_batch_model(model) is expected + + +def test_claude_output_row_is_costed_from_anthropic_usage_and_its_own_model(): + def cost_calculator(usage, model, custom_llm_provider=None, model_info=None, vertex_location=None) -> tuple[float, float]: + if model != "claude-sonnet-4-6": + return 0.0, 0.0 + return usage.prompt_tokens * 0.001, usage.completion_tokens * 0.002 + + stats: Final = native_vertex_batch_row_stats( + { + "custom_id": "r1", + "request": {"messages": [{"role": "user", "content": "hi"}], "anthropic_version": "vertex-2023-10-16"}, + "response": { + "id": "msg_vrtx_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 5}, + }, + }, + None, + model_info=None, + calculate_usage=lambda _: pytest.fail("Claude rows must not go through the Gemini usage parser"), + cost_calculator=cost_calculator, + ) + + assert stats is not None, "a successful Claude row must be costed, not counted as failed" + assert (stats.usage.prompt_tokens, stats.usage.completion_tokens, stats.total_tokens) == (12, 5, 17) + assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((0.012, 0.010)) + assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((12 * 0.001, 5 * 0.002)) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index cf4a7c4cbc8..98e204dd44e 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final from types import MappingProxyType from urllib.parse import parse_qs, urlparse @@ -16,6 +16,7 @@ from pydantic import TypeAdapter from unittest.mock import MagicMock import litellm +from litellm.llms.base_llm.files.transformation import BaseFileUploadStream from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -168,6 +169,17 @@ class TestBatchObjectNaming: object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini-2.5-flash"}}]) assert object_name.startswith("litellm-vertex-files/publishers/google/models/gemini-2.5-flash/") + @pytest.mark.parametrize( + ("model", "stored_model"), + [ + ("multimodalembedding@001", "publishers/google/models/multimodalembedding_001/"), + ("claude-haiku-4-5@20251001", "publishers/anthropic/models/claude-haiku-4-5@20251001/"), + ], + ) + def test_only_claude_models_keep_the_at_version_in_their_object_name(self, config, model, stored_model): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": model}}]) + assert object_name.startswith(f"litellm-vertex-files/{stored_model}") + def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config): """A numeric endpoint id must not be filed under publishers/google/models/gemini/, which the batch transformation later mangles into a nonexistent publisher model (LIT-6899).""" @@ -1992,6 +2004,423 @@ class TestVertexEmbeddingsBatchOutputTranslation: assert config._try_transform_vertex_batch_output_to_openai(content) == content +CLAUDE_DEPLOYMENT = {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/claude-sonnet-4-6"} + + +def _claude_chat_entry(custom_id: str | None, content: str, **body: object) -> dict[str, object]: + entry: Final[dict[str, object]] = { + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "claude-sonnet-4-6", "messages": [{"role": "user", "content": content}], **body}, + } + return entry if custom_id is None else {"custom_id": custom_id, **entry} + + +def _uploaded_rows( + config: VertexAIFilesConfig, litellm_params: Mapping[str, object], entries: Sequence[Mapping[str, object]] +) -> list: + upload: Final = config.transform_create_file_request( + model="", + create_file_data={ + "file": ("batch.jsonl", "\n".join(json.dumps(entry) for entry in entries).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={}, + litellm_params=dict(litellm_params), + ) + match upload: + case {"streaming_media_upload": {"body_stream": BaseFileUploadStream() as stream}}: + return [json.loads(line) for line in b"".join(stream.iter_bytes()).splitlines()] + case _: + pytest.fail(f"expected a streamed upload, got {upload!r}") + + +def _claude_output_row(custom_id: str, text: str, content: object = None) -> dict[str, object]: + return { + "custom_id": custom_id, + "status": "", + "processed_time": "2026-10-28T20:37:47.634+00:00", + "response": { + "id": "msg_vrtx_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": text}] if content is None else content, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 5}, + }, + } + + +class TestVertexClaudeBatch: + """Claude on Vertex batches only as `publishers/anthropic/models/` with Anthropic Messages rows + (https://github.com/BerriAI/litellm/issues/45671)""" + + @pytest.mark.parametrize("claude_model", ["claude-sonnet-4-6", "claude-haiku-4-5@20251001"]) + def test_claude_upload_creates_a_batch_job_for_the_anthropic_publisher_model(self, config, claude_model): + from litellm.llms.vertex_ai.batches.transformation import VertexAIBatchTransformation + + url: Final = config.get_complete_file_url( + api_base=None, + api_key=None, + model="", + optional_params={}, + litellm_params={**CLAUDE_DEPLOYMENT, "model": f"vertex_ai/{claude_model}"}, + data={"file": ("batch.jsonl", json.dumps(_claude_chat_entry("r1", "hi")).encode()), "purpose": "batch"}, + ) + object_name: Final = parse_qs(urlparse(url).query)["name"][0] + + job: Final = VertexAIBatchTransformation.transform_openai_batch_request_to_vertex_ai_batch_request( + {"input_file_id": f"gs://my-bucket/{object_name}"}, vertex_project="p", vertex_location="us-east5" + ) + + assert job["model"] == f"publishers/anthropic/models/{claude_model}", object_name + + def test_claude_body_model_without_deployment_is_filed_under_anthropic_publisher(self, config): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "claude-haiku-4-5"}}]) + assert object_name.startswith("litellm-vertex-files/publishers/anthropic/models/claude-haiku-4-5/") + + def test_publisher_path_model_is_not_nested_under_google(self, config): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl( + [{"body": {}}], deployment_model="vertex_ai/publishers/anthropic/models/claude-sonnet-4-6" + ) + assert object_name.startswith("litellm-vertex-files/publishers/anthropic/models/claude-sonnet-4-6/") + + def test_claude_rows_are_uploaded_in_anthropic_messages_shape(self, config): + rows: Final = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [_claude_chat_entry("request-1", "Hello!", max_tokens=50, temperature=0.2)], + ) + + assert rows == [ + { + "custom_id": "request-1", + "request": { + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}], + "max_tokens": 50, + "temperature": 0.2, + "anthropic_version": "vertex-2023-10-16", + }, + } + ] + + def test_claude_system_message_becomes_the_anthropic_system_field(self, config): + entry: Final = _claude_chat_entry( + "r1", + "unused", + max_tokens=10, + messages=[{"role": "system", "content": "be brief"}, {"role": "user", "content": "Hello!"}], + ) + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert row["request"]["system"] == [{"type": "text", "text": "be brief"}] + assert [message["role"] for message in row["request"]["messages"]] == ["user"] + + def test_claude_row_drops_an_unsupported_param_under_deployment_drop_params(self, config, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + (row,) = _uploaded_rows( + config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9", "drop_params": True}, [entry] + ) + + assert row["request"]["max_tokens"] == 10 + assert "top_p" not in row["request"] + + def test_claude_row_drops_a_param_the_deployment_lists_in_additional_drop_params(self, config, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + (row,) = _uploaded_rows( + config, + {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9", "additional_drop_params": ["top_p"]}, + [entry], + ) + + assert row["request"]["max_tokens"] == 10 + assert "top_p" not in row["request"] + + def test_claude_rows_take_deployment_params_as_defaults_the_row_overrides(self, config): + rows: Final = _uploaded_rows( + config, + {**CLAUDE_DEPLOYMENT, "max_tokens": 64, "temperature": 0.3}, + [_claude_chat_entry("r1", "hi"), _claude_chat_entry("r2", "hi", max_tokens=10)], + ) + + assert [(row["request"]["max_tokens"], row["request"]["temperature"]) for row in rows] == [(64, 0.3), (10, 0.3)] + + def test_claude_row_with_an_unsupported_param_is_refused_without_drop_params(self, config, monkeypatch): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + with pytest.raises(VertexAIError, match=r"line 1 .*top_p=0.5.*drop_params") as exc_info: + _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"}, [entry]) + assert exc_info.value.status_code == 400 + + def test_claude_deployment_wins_over_a_gemini_body_model(self, config): + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, model="gemini-2.5-flash") + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert row["request"]["anthropic_version"] == "vertex-2023-10-16" + assert "contents" not in row["request"] + + def test_gemini_deployment_still_uploads_generate_content_rows(self, config): + (row,) = _uploaded_rows( + config, + {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/gemini-2.5-flash"}, + [_claude_chat_entry("r1", "hi")], + ) + assert row["request"]["contents"] == [{"role": "user", "parts": [{"text": "hi"}]}] + assert "custom_id" not in row + + def test_claude_embeddings_batch_line_is_rejected_with_400(self): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + entry: Final = { + "custom_id": "e1", + "url": "/v1/embeddings", + "body": {"model": "claude-sonnet-4-6", "input": "hi"}, + } + with pytest.raises(VertexAIError) as exc_info: + _openai_batch_jsonl_entry_to_vertex_rows(entry, VertexAIFilesConfig()._map_openai_to_vertex_params) + assert exc_info.value.status_code == 400 + + def test_claude_output_rows_become_openai_batch_rows(self, config): + content: Final = b"\n".join( + json.dumps(row).encode() for row in (_claude_output_row("a", "first"), _claude_output_row("b", "second")) + ) + + rows: Final = [ + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ] + + assert [row["custom_id"] for row in rows] == ["a", "b"] + assert [row["error"] for row in rows] == [None, None] + bodies: Final = [row["response"]["body"] for row in rows] + assert [body["choices"][0]["message"]["content"] for body in bodies] == ["first", "second"] + assert [body["choices"][0]["finish_reason"] for body in bodies] == ["stop", "stop"] + assert [(body["usage"]["prompt_tokens"], body["usage"]["completion_tokens"]) for body in bodies] == [ + (12, 5), + (12, 5), + ] + assert {body["model"] for body in bodies} == {"claude-sonnet-4-6"} + + @pytest.mark.parametrize("failed_first", [False, True]) + def test_failed_claude_output_row_is_an_openai_error_row(self, config, failed_first): + failed_row: Final = { + "custom_id": "bad", + "status": 'Bad Request: {"type": "error", "error": {"type": "invalid_request_error", "message": "too long"}}', + "processed_time": "2026-10-28T20:37:48.413+00:00", + "response": {}, + } + rows_in: Final = (failed_row, _claude_output_row("ok", "fine")) + content: Final = b"\n".join(json.dumps(row).encode() for row in (rows_in if failed_first else rows_in[::-1])) + + rows_out: Final = { + row["custom_id"]: row + for row in ( + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ) + } + + assert rows_out["ok"]["response"]["body"]["choices"][0]["message"]["content"] == "fine" + assert rows_out["bad"]["response"] is None + assert rows_out["bad"]["error"]["code"] == "vertex_ai_error" + assert "too long" in rows_out["bad"]["error"]["message"] + + def test_claude_row_keeps_provider_only_params_and_leaves_out_litellm_ones(self, config): + (row,) = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [_claude_chat_entry("r1", "hi", max_tokens=10, top_k=5, drop_params=True, metadata={"team": "a"})], + ) + + assert row["request"]["top_k"] == 5 + assert not {"drop_params", "metadata", "extra_body"} & set(row["request"]) + + def test_claude_row_top_k_follows_drop_params_on_a_model_without_sampling_params(self, config, monkeypatch): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + deployment: Final = {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"} + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_k=5) + + with pytest.raises(VertexAIError, match=r"line 1 .*top_k") as exc_info: + _uploaded_rows(config, deployment, [entry]) + (row,) = _uploaded_rows(config, {**deployment, "drop_params": True}, [entry]) + + assert exc_info.value.status_code == 400 + assert "top_k" not in row["request"] + + @pytest.mark.parametrize("param", [{"n": 2}, {"seed": 7}, {"presence_penalty": 0.5}, {"logprobs": True}]) + def test_claude_row_param_claude_does_not_support_follows_drop_params(self, config, monkeypatch, param): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setattr(litellm, "drop_params", False) + (name,) = param + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, **param) + + with pytest.raises(VertexAIError, match=rf"line 1 .*{name}") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + (row,) = _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "drop_params": True}, [entry]) + + assert exc_info.value.status_code == 400 + assert name not in row["request"] + + def test_claude_row_refuses_a_tool_name_claude_would_rename(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + def tool(name: str) -> dict[str, object]: + return {"type": "function", "function": {"name": name, "parameters": {"type": "object", "properties": {}}}} + + renamed: Final = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool("get.weather")]) + kept: Final = _claude_chat_entry("r2", "hi", max_tokens=10, tools=[tool("get_weather")]) + + with pytest.raises(VertexAIError, match=r"line 1 .*get\.weather") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [renamed]) + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [kept]) + + assert exc_info.value.status_code == 400 + assert [t["name"] for t in row["request"]["tools"]] == ["get_weather"] + + def test_claude_row_refuses_a_user_tool_named_like_the_response_format_tool(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + tool: Final = {"type": "function", "function": {"name": "json_tool_call", "parameters": {"type": "object"}}} + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool]) + + with pytest.raises(VertexAIError, match=r"line 1 .*json_tool_call") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert exc_info.value.status_code == 400 + + def test_claude_row_leaves_out_stream(self, config): + entry: Final = _claude_chat_entry( + "r1", "hi", max_tokens=10, stream=True, stream_options={"include_usage": True} + ) + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert "stream" not in row["request"] + assert "stream_options" not in row["request"] + + def test_claude_row_merges_extra_body_into_the_request(self, config): + (row,) = _uploaded_rows( + config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10, extra_body={"top_k": 3})] + ) + + assert row["request"]["top_k"] == 3 + assert "extra_body" not in row["request"] + + def test_claude_response_format_row_round_trips_as_json_content(self, config): + schema: Final = {"type": "object", "properties": {"color": {"type": "string"}}, "required": ["color"]} + (row,) = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [ + _claude_chat_entry( + "r1", + "sky color?", + max_tokens=50, + response_format={"type": "json_schema", "json_schema": {"name": "sky", "schema": schema}}, + ) + ], + ) + assert "json_mode" not in row["request"] + (json_tool,) = row["request"]["tools"] + output_row: Final = _claude_output_row( + "r1", + "unused", + content=[{"type": "tool_use", "id": "toolu_1", "name": json_tool["name"], "input": {"color": "blue"}}], + ) + + (out,) = ( + json.loads(line) + for line in config._try_transform_vertex_batch_output_to_openai( + json.dumps(output_row).encode() + ).splitlines() + ) + + message: Final = out["response"]["body"]["choices"][0]["message"] + assert json.loads(message["content"]) == {"color": "blue"} + assert not message.get("tool_calls") + + def test_claude_output_keeps_a_user_tool_call_as_a_tool_call(self, config): + output_row: Final = _claude_output_row( + "r1", + "unused", + content=[{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}], + ) + + (out,) = ( + json.loads(line) + for line in config._try_transform_vertex_batch_output_to_openai( + json.dumps(output_row).encode() + ).splitlines() + ) + + (tool_call,) = out["response"]["body"]["choices"][0]["message"]["tool_calls"] + assert tool_call["function"]["name"] == "get_weather" + assert json.loads(tool_call["function"]["arguments"]) == {"city": "Paris"} + + def test_claude_input_file_read_back_is_returned_unchanged(self, config): + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10)]) + content: Final = json.dumps(row).encode() + + assert config._try_transform_vertex_batch_output_to_openai(content) == content + + def test_malformed_claude_message_fails_only_its_own_row(self, config): + malformed: Final = _claude_output_row("bad", "unused", content=5) + content: Final = json.dumps(_claude_output_row("ok", "fine")).encode() + b"\n" + json.dumps(malformed).encode() + + ok_row, bad_row = ( + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ) + + assert ok_row["response"]["body"]["choices"][0]["message"]["content"] == "fine" + assert bad_row["custom_id"] == "bad" + assert bad_row["response"] is None + assert bad_row["error"]["code"] == "transformation_error" + + def test_claude_line_without_custom_id_is_a_400_at_upload(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + with pytest.raises(VertexAIError, match=r"line 2 .*custom_id") as exc_info: + _uploaded_rows( + config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10), _claude_chat_entry(None, "hi")] + ) + + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_streamed_claude_output_is_transformed(self, config): + async def chunks(): + yield json.dumps(_claude_output_row("a", "first")).encode() + b"\n" + + result: Final = await config.transform_file_content_stream( + stream_iterator=chunks(), + headers={}, + request_url="https://storage.googleapis.com/storage/v1/b/my-bucket/o/x?alt=media", + logging_obj=MagicMock(), + litellm_params={}, + ) + body: Final = b"".join([chunk async for chunk in result.stream_iterator]) + + (row,) = (json.loads(line) for line in body.splitlines()) + assert row["custom_id"] == "a" + assert row["response"]["body"]["choices"][0]["message"]["content"] == "first" + + _SAMPLING_KNOBS: Final = MappingProxyType({"temperature": 0, "top_p": 0.5, "top_k": 3}) From 44cd858697c397decff308e2db3d382a26887f2f Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:19:36 -0700 Subject: [PATCH 15/34] fix(cost): bill the requested model's rates when the response names an unpriced model (#48900) * fix(cost): bill the requested model's rates when the response names an unpriced model Fireworks answers chat and /v1/messages calls with a display name such as "Qwen 3.8 Max" in the response's model field. The cost path priced that name, the Fireworks calculator fell back to its free default bucket without raising, and the call billed $0 even though the requested model has a row. When the response model resolves to no cost-map row or rule that declares rates and the requested model does, price from the requested model. A priced response model still wins, so dated snapshots keep their own rates, and an unpriced requested model such as a router alias leaves the result unchanged. * perf(cost): check the exact cost-map row before generalization rules when picking the priced model * fix(cost): apply the priced fallback to the provider response model streaming records * perf(cost): skip the served model's rule lookup when the requested model has no rates * fix(cost): price the requested model without a model arg and keep Fireworks size buckets behind the proxy When completion_cost runs with no model, the requested model now comes from the call's hidden litellm_model_name, so a display-name response still bills at the requested model's rates. A Fireworks model whose name carries a parameter size counts as priced through its size bucket. The Fireworks per-token lookup now treats a cost-map row that prices nothing, like the empty row the Router registers for an unlisted deployment, as missing, so an unlisted Fireworks model bills at its size bucket instead of $0 even when Fireworks echoes its slug. The "row prices anything" helpers move to llm_cost_calc/utils so the Fireworks lookup can reuse them without a circular import. * fix(fireworks): keep prices from a matching fallback rule for an unlisted model An unlisted Fireworks model that only a fallback generalization rule prices has no raw cost map row, so the empty-row check sent it to the size bucket and dropped the rule's rates. The Fireworks lookup now keeps the model info when the row or a matching rule declares a rate. * fix(cost): price a bare response model under the requested provider and drop the unlisted Fireworks bucket change The logging payload picks the pricing model without a provider, so a bare echo like mistral-large-latest failed its price check and the requested alias took over model_map_key. The served name is now priced under the requested model's provider, and an echo that is the requested name minus its provider prefix skips the checks. Treating the Router's empty price row as missing let unlisted Fireworks deployments bill from a name guess, which misprices dedicated deployment ids and routes some short names to another provider's rates. That change and its tests go back to the merge base, so unlisted Fireworks models keep billing $0 behind the proxy as on main. * perf(cost): read the response model's price row without building full model info An unpriced display name like Qwen 3.8 Max made the price check build full model info, and Fireworks' supported params lookup scans the whole cost map for an unknown name. The row check now resolves the cost map key through the cached model info helper, so that pick takes about 0.45 ms instead of several. A regional test covers a Bedrock call whose response names an unpriced display name with no model passed, which bills the requested model's regional row. The duplicate get_model_info wrapper goes, since its twin already does the same thing. * fix(cost): keep the served model's filled-in and size-bucket rates, and cache misses A served row priced only by a fill_missing_for_providers rule bills those rates on main, but the price check read only the raw row and billed the requested alias instead. It now merges the rule's fill-in keys through the helper model info lookup uses. A served Fireworks or Together AI id that names its parameter count keeps its size-bucket price, as on main, instead of switching to the requested row. The served model is checked first, cost map misses are cached and cleared with the model info cache, and the rules are read only for a name that resolved through one. A priced snapshot behind an Azure deployment name takes about 19 us of CPU time against 7 us on the merge base, down from about 300 us. * fix(cost): bill the requested model's listed rates over a served size-bucket guess A served Fireworks or Together AI name that is not in the cost map but names its parameter count, like a display name with a size in it, now bills at the requested model's listed rates instead of the provider's size-bucket price. The bucket price still applies when the requested model has no rates, as on main --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 90 +++-- litellm/utils.py | 23 +- .../test_litellm_logging.py | 43 +++ tests/unit/test_cost_calculator.py | 309 +++++++++++++++++- 4 files changed, 430 insertions(+), 35 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index defe1e63b86..6677b7327dc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -153,7 +153,9 @@ from litellm.utils import ( ProviderConfigManager, TextCompletionResponse, TranscriptionResponse, + cached_cost_map_key, cached_get_model_info_helper, + mapped_model_fill_missing, model_info_with_pricing_overrides, runtime_model_cost_registrations, token_counter, @@ -962,21 +964,17 @@ def select_model_name_for_cost_calc( """ 1. If custom pricing is true, return received model name 2. If base_model is set (e.g. for azure models), return that - 3. If completion response has model set return that + 3. If completion response has model set return that, unless only the requested model's cost-map row has rates 4. Check if model is passed in return that """ return_model: str | None = None custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider) - completion_response_model: str | None = None - if completion_response is not None: - if isinstance(completion_response, BaseModel): - completion_response_model = getattr(completion_response, "model", None) - elif isinstance(completion_response, dict): - completion_response_model = completion_response.get("model", None) + completion_response_model: Final = _get_response_model(completion_response) hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None) provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model") + requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name") explicit_pricing: Final = custom_pricing is True or base_model is not None configured_model: Final = model or ( completion_response_model if isinstance(completion_response_model, str) else None @@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc( else: return_model = model - elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None: - return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model + elif base_model is not None or bedrock_pricing_model is not None: + return_model = base_model if base_model is not None else bedrock_pricing_model + + elif provider_response_model is not None: + return _served_or_priced_requested_model( + provider_response_model, requested_model, custom_llm_provider, priced_region + ) elif completion_response_model is None and hidden_params is not None: if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0: return_model = hidden_params.get("model", model) if return_model is None and completion_response_model is not None: - return_model = completion_response_model + return _served_or_priced_requested_model( + completion_response_model, requested_model, custom_llm_provider, priced_region + ) if return_model is None and model is not None: return_model = model - if ( - return_model is not None - and custom_llm_provider is not None - and not _model_contains_known_llm_provider(return_model) - ): # add provider prefix if not already present, to match model_cost - provider_prefix: Final = ( - custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}" - ) - return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region) - - return return_model + if return_model is None: + return None + return _cost_map_model_name(return_model, custom_llm_provider, priced_region) _select_model_name_for_cost_calc = select_model_name_for_cost_calc +def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str: + if custom_llm_provider is None or _model_contains_known_llm_provider(model): + return model + provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}" + return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name) + + +def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool: + rule_info: Final = unmapped_model_generalization(model, custom_llm_provider) + return rule_info is not None and _cost_map_entry_prices_anything(rule_info) + + +def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool: + key: Final = cached_cost_map_key(model, custom_llm_provider) + if key is None: + return False + entry: Final = _raw_cost_map_entry(key) + if entry is None: + return _rule_declares_rates(model, custom_llm_provider) + fill_missing: Final = mapped_model_fill_missing(key, entry) + return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry}) + + +def _served_or_priced_requested_model( + served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None +) -> str: + served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region) + if requested_model is None: + return served + requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None) + if served in (requested, requested.partition("/")[2]): + return served + pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider) + if _resolves_to_rates(served, pricing_provider): + return served + return requested if _resolves_to_rates(requested, pricing_provider) else served + + def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str: """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already @@ -3516,7 +3551,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a session that should bill nothing fell through to the public rates instead. """ - resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider) + resolved: Final = _cost_map_model_info(model_name, custom_llm_provider) entries: Final = ( litellm.model_cost.get(resolved.get("key")) if resolved is not None else None, litellm.model_cost.get(model_name), @@ -3712,9 +3747,9 @@ def handle_realtime_transcription_cost_calculation( return 0.0 model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name - model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider) + model_info: Final = _cost_map_model_info(model_name, custom_llm_provider) override_info: Final = ( - _get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None + _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None ) total_cost = 0.0 @@ -3745,13 +3780,6 @@ def _get_transcription_model_name_from_results( return None -def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None: - try: - return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - except Exception: - return None - - _TranscriptionRateKey: TypeAlias = Literal[ "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token" ] diff --git a/litellm/utils.py b/litellm/utils.py index d951b80d49b..41a3fddf3a3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -279,7 +279,7 @@ from litellm.types.utils import ( ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} -_BACKFILL_MODES: Final = frozenset({"chat", "responses"}) +_BACKFILL_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"}) # +-----------------------------------------------+ # | | @@ -6530,6 +6530,7 @@ def _invalidate_model_cost_lowercase_map() -> None: # Clear LRU caches that depend on model_cost data _cached_get_model_info.cache_clear() cached_get_model_info_helper.cache_clear() + cached_cost_map_key.cache_clear() def _rebuild_model_cost_lowercase_map() -> dict[str, str]: @@ -6755,6 +6756,14 @@ def unmapped_model_generalization(model: str, custom_llm_provider: str | None) - return generalization[1] if generalization is not None else None +def mapped_model_fill_missing(key: str, model_info: Mapping[str, object]) -> Mapping[str, object] | None: + mode: Final = model_info.get("mode", "chat") + provider: Final = model_info.get("litellm_provider", "") + if not isinstance(mode, str) or mode not in _BACKFILL_MODES or not isinstance(provider, str): + return None + return match_fill_missing_generalizations(key, provider) + + def _strip_mantle_region_prefix(model: str) -> str: from litellm.llms.bedrock_mantle.common_utils import split_mantle_region_prefix @@ -6883,6 +6892,14 @@ def cached_get_model_info_helper( _cached_get_model_info_helper = cached_get_model_info_helper +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) +def cached_cost_map_key(model: str, custom_llm_provider: str | None) -> str | None: + try: + return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)["key"] + except Exception: + return None + + def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSpecificModelInfo | None: ## PROVIDER-SPECIFIC INFORMATION # if custom_llm_provider == "predibase": @@ -7040,8 +7057,8 @@ def get_model_info_helper( key: str | None = lookup[0] _model_info: dict[str, Any] | None = lookup[1] - if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", "")) + if _model_info is not None and key is not None: + fill_missing: Final = mapped_model_fill_missing(key, _model_info) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index c320cb36d73..b3ae3bf6240 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -12320,6 +12320,49 @@ def test_get_model_cost_information_custom_pricing_uses_base_model(): assert result["model_map_key"] != "invoke_test_claude" +@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +def test_get_model_cost_information_maps_requested_model_when_response_names_an_unmapped_display_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + response: Final = ModelResponse(model="Sol Display Max", choices=[]) + response._hidden_params["provider_response_model"] = "Sol Display Max" + response._hidden_params["litellm_model_name"] = requested_model + + result: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=None, custom_pricing=False, custom_llm_provider="fireworks_ai", init_response_obj=response + ) + + assert result["model_map_key"] == requested_model + assert result["model_map_value"] is not None + assert result["model_map_value"]["input_cost_per_token"] == litellm.model_cost[requested_model]["input_cost_per_token"] + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +def test_get_model_cost_information_keeps_a_priced_response_model_that_echoes_without_its_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + served_row: Final = {"litellm_provider": "mistral", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6} + requested_row: Final = {"litellm_provider": "mistral", "mode": "chat", "input_cost_per_token": 9e-6, "output_cost_per_token": 9e-6} + monkeypatch.setitem(litellm.model_cost, "mistral/sol-snapshot-test", served_row) + monkeypatch.setitem(litellm.model_cost, "mistral/sol-alias-test", requested_row) + response: Final = ModelResponse(model="sol-snapshot-test", choices=[]) + response._hidden_params["litellm_model_name"] = "mistral/sol-alias-test" + + result: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=None, custom_pricing=False, custom_llm_provider="mistral", init_response_obj=response + ) + + assert result["model_map_key"] == "sol-snapshot-test" + assert result["model_map_value"] is not None + assert result["model_map_value"]["input_cost_per_token"] == served_row["input_cost_per_token"] + + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_deployment_when_no_base_model(): """metadata["deployment"] is used for cost-map lookup when base_model is not set.""" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 84f6dab9eff..f10baccd1cb 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -37,7 +37,7 @@ from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject -from litellm.utils import supports_prompt_caching +from litellm.utils import _invalidate_model_cost_lowercase_map, supports_prompt_caching import os import threading from unittest.mock import MagicMock, patch @@ -424,6 +424,313 @@ def test_completion_cost_prices_rule_only_base_model_from_the_response_model( assert cost == pytest.approx(usage.prompt_tokens * 0.0000111 + usage.completion_tokens * 0.0000222) +def _token_rates_cost(row: Mapping[str, object], usage: Usage) -> float: + input_rate: Final = row["input_cost_per_token"] + output_rate: Final = row["output_cost_per_token"] + assert isinstance(input_rate, float) and isinstance(output_rate, float), row + return usage.prompt_tokens * input_rate + usage.completion_tokens * output_rate + + +@pytest.mark.parametrize( + "requested_model", + ["accounts/fireworks/models/sol-display-test", "fireworks_ai/accounts/fireworks/models/sol-display-test"], +) +def test_completion_cost_bills_requested_model_when_response_names_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, requested_model: str +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "fireworks_ai/accounts/fireworks/models/sol-display-test", + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="Sol Display Max", choices=[], usage=usage), + model=requested_model, + custom_llm_provider="fireworks_ai", + ) + + assert cost == pytest.approx( + _token_rates_cost(litellm.model_cost["fireworks_ai/accounts/fireworks/models/sol-display-test"], usage) + ) + + +def test_completion_cost_bills_requested_model_when_stream_reports_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="accounts/fireworks/models/sol-display-test", choices=[], usage=usage) + response._hidden_params["provider_response_model"] = "Sol Display Max" + + cost: Final = completion_cost( + completion_response=response, model=requested_model, custom_llm_provider="fireworks_ai" + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + +def test_completion_cost_bills_requested_model_when_response_row_has_no_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-requested-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-served-snapshot-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": None}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-served-snapshot-test", choices=[], usage=usage), + model="openai/sol-requested-test", + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test"], usage)) + + +def test_completion_cost_bills_priced_response_model_over_priced_requested_model( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-requested-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-requested-test-2099-01-01", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-requested-test-2099-01-01", choices=[], usage=usage), + model="openai/sol-requested-test", + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test-2099-01-01"], usage)) + + +def test_completion_cost_keeps_response_pricing_when_requested_alias_has_no_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + size_bucket: Final = get_base_model_for_pricing("Sol 70B Display") + monkeypatch.setitem( + litellm.model_cost, + size_bucket, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="Sol 70B Display", choices=[], usage=usage), + model="sol-team-alias", + custom_llm_provider="fireworks_ai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[size_bucket], usage)) + + +def test_completion_cost_bills_requested_model_priced_only_by_a_rule_when_response_row_has_no_rates( + _local_model_cost_map: None, _priced_claude_family_generalization: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "anthropic/sol-served-snapshot-test", + {"litellm_provider": "anthropic", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": None}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-served-snapshot-test", choices=[], usage=usage), + model="anthropic/claude-sol-9", + custom_llm_provider="anthropic", + ) + + assert cost == pytest.approx(usage.prompt_tokens * 0.0000321 + usage.completion_tokens * 0.0000765) + + +def test_completion_cost_bills_requested_model_from_hidden_params_when_no_model_is_passed( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="Sol Display Max", choices=[], usage=usage) + response._hidden_params["litellm_model_name"] = requested_model + + cost: Final = completion_cost(completion_response=response) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + +def test_completion_cost_bills_requested_model_regional_row_when_response_names_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "bedrock/sol-regional-test", + {"litellm_provider": "bedrock", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "bedrock/eu-central-1/sol-regional-test", + {"litellm_provider": "bedrock", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="Sol Regional Display", choices=[], usage=usage) + response._hidden_params.update( + {"litellm_model_name": "bedrock/sol-regional-test", "region_name": "eu-central-1", "custom_llm_provider": "bedrock"} + ) + + cost: Final = completion_cost(completion_response=response) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["bedrock/eu-central-1/sol-regional-test"], usage)) + + +def test_completion_cost_keeps_served_row_rates_filled_by_a_rule_over_the_requested_alias( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(litellm.model_cost, "sol-fill-snapshot-test", {"litellm_provider": "openai", "mode": "chat"}) + monkeypatch.setitem( + litellm.model_cost, + "sol-fill-alias-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + filled_rates: Final = {"input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6} + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="sol-fill-snapshot-test", choices=[], usage=usage) + response._hidden_params.update({"litellm_model_name": "openai/sol-fill-alias-test", "custom_llm_provider": "openai"}) + previous_rules: Final = list(get_fallback_generalization_rules()) + set_fallback_generalizations( + [ + { + "name": "sol-fill-snapshot-rates", + "pattern": r"^sol-fill-snapshot-test$", + "fill_missing_for_providers": ["openai"], + "model_info": filled_rates, + } + ] + ) + try: + cost: Final = completion_cost(completion_response=response) + finally: + set_fallback_generalizations(previous_rules) + + assert cost == pytest.approx(_token_rates_cost(filled_rates, usage)) + + +_SIZE_BUCKET_CASES: Final = ( + pytest.param( + "fireworks_ai", + "fireworks_ai/accounts/fireworks/models/sol-bucket-requested-test", + "Sol Bucket 405B Display", + "fireworks-ai-above-16b", + id="fireworks_ai", + ), + pytest.param( + "together_ai", + "together_ai/sol-bucket-requested-test", + "meta-llama/Sol-Bucket-8B-Test", + "together-ai-4.1b-8b", + id="together_ai", + ), +) + + +def _size_bucket_response(served_model: str, requested_model: str, custom_llm_provider: str) -> ModelResponse: + response: Final = ModelResponse(model=served_model, choices=[], usage=_SIZE_BUCKET_USAGE) + response._hidden_params.update({"litellm_model_name": requested_model, "custom_llm_provider": custom_llm_provider}) + return response + + +_SIZE_BUCKET_USAGE: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + +@pytest.mark.parametrize(("custom_llm_provider", "requested_model", "served_model", "bucket"), _SIZE_BUCKET_CASES) +def test_completion_cost_bills_requested_listed_rates_over_a_served_size_bucket( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + custom_llm_provider: str, + requested_model: str, + served_model: str, + bucket: str, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + requested_model, + { + "litellm_provider": custom_llm_provider, + "mode": "chat", + "input_cost_per_token": 3e-6, + "output_cost_per_token": 7e-6, + }, + ) + + cost: Final = completion_cost( + completion_response=_size_bucket_response(served_model, requested_model, custom_llm_provider) + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], _SIZE_BUCKET_USAGE)) + assert cost != pytest.approx(_token_rates_cost(litellm.model_cost[bucket], _SIZE_BUCKET_USAGE)) + + +@pytest.mark.parametrize(("custom_llm_provider", "requested_model", "served_model", "bucket"), _SIZE_BUCKET_CASES) +def test_completion_cost_keeps_served_size_bucket_rates_when_the_requested_model_is_unpriced( + _local_model_cost_map: None, + custom_llm_provider: str, + requested_model: str, + served_model: str, + bucket: str, +) -> None: + cost: Final = completion_cost( + completion_response=_size_bucket_response(served_model, requested_model, custom_llm_provider) + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[bucket], _SIZE_BUCKET_USAGE)) + + +def test_completion_cost_bills_a_requested_model_priced_after_an_unpriced_lookup( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-late-priced-test" + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + def display_name_response() -> ModelResponse: + response: Final = ModelResponse(model="Sol Late Display", choices=[], usage=usage) + response._hidden_params.update({"litellm_model_name": requested_model, "custom_llm_provider": "fireworks_ai"}) + return response + + completion_cost(completion_response=display_name_response()) + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + _invalidate_model_cost_lowercase_map() + + cost: Final = completion_cost(completion_response=display_name_response()) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + def test_completion_cost_orders_candidates_without_extra_huggingface_config_requests( _local_model_cost_map: None, ) -> None: From 85af2f65c9894bd0fa3a354dc6dec96df47d060d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 19:29:44 -0700 Subject: [PATCH 16/34] refactor(helm)!: connect both charts to a separately deployed Lens (#49169) Neither chart bundles Lens anymore. The Lens subchart, its lock entry, the vendored chart archive, the templates/lens resources and the bundled-Lens install smoke are gone. A connection-only lens block wires the gateway and backend to a Lens deployed from its own chart, and an enabled lensWorker value fails the render instead of being ignored. Supersedes #47869, which made the same change for helm/litellm only. --- .github/workflows/lens-install-smoke.yml | 152 ------ helm/litellm-helm/Chart.lock | 7 +- helm/litellm-helm/Chart.yaml | 3 - helm/litellm-helm/README.md | 4 +- helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz | Bin 9226 -> 0 bytes helm/litellm-helm/templates/_helpers.tpl | 97 +--- helm/litellm-helm/templates/ingress.yaml | 12 +- .../templates/lens/clickhouse.yaml | 3 - .../templates/lens/deployment.yaml | 3 - helm/litellm-helm/templates/lens/ingress.yaml | 3 - helm/litellm-helm/templates/lens/secrets.yaml | 3 - helm/litellm-helm/templates/lens/service.yaml | 3 - .../tests/lens_connection_tests.yaml | 128 +++++ .../tests/lens_endpoint_defaults_tests.yaml | 84 --- helm/litellm-helm/tests/lens_modes_tests.yaml | 187 ------- .../tests/lens_modes_validation_tests.yaml | 52 -- .../tests/lens_saved_secrets_tests.yaml | 59 --- .../tests/lens_service_tests.yaml | 167 ------ helm/litellm-helm/tests/lens_setup_tests.yaml | 162 ------ helm/litellm-helm/values.yaml | 69 +-- helm/litellm/Chart.lock | 6 - helm/litellm/Chart.yaml | 5 - helm/litellm/README.md | 6 +- helm/litellm/charts/lens-0.1.0-dev.0.tgz | Bin 9226 -> 0 bytes helm/litellm/templates/_helpers.tpl | 96 +--- helm/litellm/templates/ingress.yaml | 12 +- helm/litellm/templates/lens/clickhouse.yaml | 3 - helm/litellm/templates/lens/deployment.yaml | 3 - helm/litellm/templates/lens/ingress.yaml | 3 - helm/litellm/templates/lens/secrets.yaml | 3 - helm/litellm/templates/lens/service.yaml | 3 - helm/litellm/tests/component_hosts_tests.yaml | 13 +- helm/litellm/tests/lens_connection_tests.yaml | 166 ++++++ .../tests/lens_endpoint_defaults_tests.yaml | 86 ---- helm/litellm/tests/lens_modes_tests.yaml | 208 -------- .../tests/lens_modes_validation_tests.yaml | 54 -- .../tests/lens_saved_secrets_tests.yaml | 61 --- helm/litellm/tests/lens_service_tests.yaml | 196 ------- helm/litellm/tests/lens_setup_tests.yaml | 164 ------ helm/litellm/tests/lens_worker_tests.yaml | 79 --- helm/litellm/values.yaml | 69 +-- tests/e2e/migrations/lens_helm_smoke.sh | 477 ------------------ .../migrations/lens_release_qualification.sh | 284 ----------- .../test_lens_release_qualification.py | 60 --- 44 files changed, 356 insertions(+), 2899 deletions(-) delete mode 100644 .github/workflows/lens-install-smoke.yml delete mode 100644 helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz delete mode 100644 helm/litellm-helm/templates/lens/clickhouse.yaml delete mode 100644 helm/litellm-helm/templates/lens/deployment.yaml delete mode 100644 helm/litellm-helm/templates/lens/ingress.yaml delete mode 100644 helm/litellm-helm/templates/lens/secrets.yaml delete mode 100644 helm/litellm-helm/templates/lens/service.yaml create mode 100644 helm/litellm-helm/tests/lens_connection_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_modes_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_modes_validation_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_saved_secrets_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_service_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_setup_tests.yaml delete mode 100644 helm/litellm/Chart.lock delete mode 100644 helm/litellm/charts/lens-0.1.0-dev.0.tgz delete mode 100644 helm/litellm/templates/lens/clickhouse.yaml delete mode 100644 helm/litellm/templates/lens/deployment.yaml delete mode 100644 helm/litellm/templates/lens/ingress.yaml delete mode 100644 helm/litellm/templates/lens/secrets.yaml delete mode 100644 helm/litellm/templates/lens/service.yaml create mode 100644 helm/litellm/tests/lens_connection_tests.yaml delete mode 100644 helm/litellm/tests/lens_endpoint_defaults_tests.yaml delete mode 100644 helm/litellm/tests/lens_modes_tests.yaml delete mode 100644 helm/litellm/tests/lens_modes_validation_tests.yaml delete mode 100644 helm/litellm/tests/lens_saved_secrets_tests.yaml delete mode 100644 helm/litellm/tests/lens_service_tests.yaml delete mode 100644 helm/litellm/tests/lens_setup_tests.yaml delete mode 100644 helm/litellm/tests/lens_worker_tests.yaml delete mode 100644 tests/e2e/migrations/lens_helm_smoke.sh delete mode 100644 tests/e2e/migrations/lens_release_qualification.sh delete mode 100644 tests/e2e_harness/migrations/test_lens_release_qualification.py diff --git a/.github/workflows/lens-install-smoke.yml b/.github/workflows/lens-install-smoke.yml deleted file mode 100644 index 9832a69bd90..00000000000 --- a/.github/workflows/lens-install-smoke.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: Lens installation smoke - -on: - workflow_dispatch: - inputs: - lens_image: - description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256 - required: true - type: string - lens_upgrade_image: - description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256 - required: true - type: string - gateway_baseline_ref: - description: Previously qualified gateway source commit, full SHA - required: true - type: string - -permissions: - contents: read - -concurrency: - group: lens-install-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build-images: - runs-on: ubuntu-latest-16-cores - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - component: gateway - dockerfile: gateway/Dockerfile - - component: backend - dockerfile: backend/Dockerfile - - component: ui - dockerfile: ui/Dockerfile - - component: migrations - dockerfile: migrations/Dockerfile - - component: monolith - dockerfile: Dockerfile - - component: gateway - dockerfile: gateway/Dockerfile - baseline: true - - component: backend - dockerfile: backend/Dockerfile - baseline: true - - component: monolith - dockerfile: Dockerfile - baseline: true - env: - COMPONENT: ${{ matrix.component }} - DOCKERFILE: ${{ matrix.dockerfile }} - SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }} - IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }} - CURRENT_REF: ${{ github.sha }} - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - ref: ${{ env.SOURCE_REF }} - - name: Verify the selected source - shell: bash -euo pipefail {0} - run: | - [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]] - test "$(git rev-parse HEAD)" = "$SOURCE_REF" - if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then - test "$SOURCE_REF" != "$CURRENT_REF" - fi - - name: Build the matching release image - run: | - docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \ - -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" . - - name: Save the matching release image - run: | - docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \ - | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz" - - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }} - path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz - compression-level: 0 - retention-days: 3 - if-no-files-found: error - overwrite: true - - helm-install: - needs: build-images - runs-on: ubuntu-latest-16-cores - timeout-minutes: 25 - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - with: - pattern: lens-install-*-${{ github.sha }} - merge-multiple: true - path: ${{ runner.temp }}/lens-install-images - - name: Load the matching release images - run: | - for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do - gzip -dc "$archive" | docker load - rm "$archive" - done - - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 - with: - cosign-release: v2.4.3 - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ vars.LENS_READ_USER || github.actor }} - password: ${{ secrets.LENS_READ_TOKEN }} - - name: Verify independent Lens images before loading them - env: - LENS_IMAGE: ${{ inputs.lens_image }} - LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }} - LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }} - LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }} - shell: bash -euo pipefail {0} - run: | - [[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]] - printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub" - echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check - for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do - [[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]] - cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null - cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null - docker pull "$image" - done - docker tag "$LENS_IMAGE" lens-ci-worker:baseline - docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade - - name: Install pinned Kubernetes test tools - run: | - curl --fail --location --output "$RUNNER_TEMP/kind" \ - https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64 - echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check - chmod +x "$RUNNER_TEMP/kind" - curl --fail --location --output "$RUNNER_TEMP/kubectl" \ - https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl - echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check - chmod +x "$RUNNER_TEMP/kubectl" - curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \ - https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz - echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check - tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP" - echo "$RUNNER_TEMP" >> "$GITHUB_PATH" - echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH" - - name: Install, ingest, upgrade, and restart both charts - run: bash tests/e2e/migrations/lens_helm_smoke.sh diff --git a/helm/litellm-helm/Chart.lock b/helm/litellm-helm/Chart.lock index 220e379ff5a..1e63f4f9eca 100644 --- a/helm/litellm-helm/Chart.lock +++ b/helm/litellm-helm/Chart.lock @@ -5,8 +5,5 @@ dependencies: - name: redis repository: oci://registry-1.docker.io/bitnamicharts version: 18.19.1 -- name: lens - repository: oci://ghcr.io/berriai/charts - version: 0.1.0-dev.0 -digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b -generated: '2026-10-09T04:18:10.643228000Z' +digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009 +generated: '2026-10-30T02:13:11.505726000Z' diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index a3a94241057..a3cb388ffc6 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -39,6 +39,3 @@ dependencies: version: "18.19.1" repository: oci://registry-1.docker.io/bitnamicharts condition: redis.enabled - - name: lens - version: 0.1.0-dev.0 - repository: oci://ghcr.io/berriai/charts diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index a3488d6114c..694cca05b52 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -256,9 +256,7 @@ itself. ## Connect Lens -`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment - -Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published +This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render ## Shutdown accounting diff --git a/helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz b/helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz deleted file mode 100644 index 79021702c2ca9a8eb8f60bfb0ece3616903c7fdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9226 zcmV+lB=y@LiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhciT3$=zP|%z*2iwlCvyImLF+VH)~wSO?_%RzLt|tZ*H#z zky{eR6u|9niO0sq#(o2cA(7p^85JtHqVxD~5ApPR zz25fbCjQ^+^{W5(HrKcQw!YQh?)NvI_cosYt+(D^-(3G2={*QEl_!&miof+fyRYit z{zD#`BwSI&cp?WaLNqx9F;fu7^MJB$D!Aeyj|Zgvyxk&%i)p}9NJ5?{$`TMVU~@W! zK=Mq4Fd#FfQaR{$r%cVVaS-x(_a_j7?H+V>+bx>1w;&|r$$%{SEt;nJ&t9+|^g0nN zf?g{E848vv{IENPL=h!u2y%_EWC2pKsR4S8Ojr`JWGV?wA_5Br7Eh+66@ zg#n>y8nY0;wiZpje(A~h0uQc#jWm!C#sF?u|NZ{+es%qC_WEDf|6@Ec8w)C~24q5G z39U&M$9mnoUg$*|!GN^et?~<8P%M#(CSkdp0L>Fl!SDj<+6?(TrWq+hu9G~!*g;xV|FgfPx zM1X`unYK@|I37YMK*<4l_eYCHbC#U(3rNgc-M3*D*w_VJ4G5O$C|#&Er3xJKI(?p7T-Qv%V zh6co-za4xZYzNzamowVm+8)4o^SkFgcplQ{5IygO{jKi-#*=U|2G|ZK&%b}(TYo;; z9&c@KY=`~t``hF3cmf;IX4L;~^Sf;gNhU!INCXp_#Y*E+ae;vJd#_mWdoQL^8XP5A z93wS3I&2Z5fBu<)RQ4AkVVVuddapOPzvnPV!rNN!A2ECvvpFluyYKy1%){MFs_2+X zw=9Ye%PA`pv0f>V=-GHlujGKNZ#qKI6;dwLfZ(hV;>&09;abn504W=3F{NrYAl()r zVSx?o80oy|FSCWIwnoJFqC zI7^}!qCKx{Ky2Y}HNzu=A1hUgb~RtJvTFLWYaX2-2&r^Dwt&P5Y!`K-HN;8h7PQHsKufl zxQ1A2Y5A7N*<8zcHa~UZ)9QRU>dP4&@8JHt(gn`*3_DL&WwzFph9VW@wh+FR%^`qAWd$<(ZX zwW7fnFx7PX99G?k;=j{*OahMrzboqM5|Bj23khK+n7Z2IiGug4XiH?tt~}<+Dd#Fz zkG)TCP(NMot(RZF5PAIYB?SEVUmffbz8IOdlpf-P7KQ`nawjj3_i_!DX&rYRk>m?;Lb z+zKI);OT(8+dVw|!}|isqFm4oaPRP7@7G^mzZveIzZveIRvJTB#Wx%<5vAOkOAP zRY}Si3^efh8psBiBbt^G)x;7Og(`LCOh2bwR#gmAN3<{5387&KQXcULw2^)aH2O_2 z1+SA3{wPUgHX#W#Q9{*)LlPGIbWnHkG_&_wI}NP3Cis_7Z`EB?g5(AO{C=y7cdm2q zx#@afjoYAB_(FTP*@Zhfbl|13(FD(5VE zvA)wSeiQ{_K0UWsI@c`dMZ*g1hMbFtB{;xt%%?luS~aiBgr;)F)g7C4)#OgMxrL$! zzuvh7rC2hRFFd-~;i+{~om(z>K9?`nd%a$7r`!0=ul#4mMK(Xzt6#p*D}Se3dQ||@ zy#bOPK_}|m@PW*9TI$J8=#H5l7QNV+aj7(+y;$!*4|+i_Sl{X9?{>^w_cu0oy7;G? zQXiz{UG8)n>1gL)oSP3Xb`ll_Kqc5X^{i~1FMozoPj?mQJRVeelcSGGWO z<}}R#supZn?0PG~V-xPr&h=l>=UDasMUck)Y7U8dJpbSG&F3}$-}c5=|KDRgHUD2h z0LK5f+-vkQFWD@@$}-DE7Om&+bH!TO_fc=ld~KKP|4rWmRk)cvS?-4>O)ob(it1gv zDu)dqY@1f^=GD2~=yLDI|5uKe?X9hiexvK9{AJSo`rh3yzGCS7V`ZDB@Wqtp6yKXE zZEzsuICo)g-<^jUI7JNnfqXd5KG75MGmAl9Nd@!D)R6mi_8jgX56_QYzua&3H%LX3 zh{ncmzohobo1YF3_Rim&9xnCfXl0)Xh)DYo5=ru4I$<*Z*3Qvge7?ceb9ivJe|UIw zep@gK_iPcX_g%Ah?qB!2`|{}E`26hkulvUj;Ag5Z@JeEZi}Wmk(>&;p@*-s-ZA{Ze(eYJJAWb$ zyyaNOX0V?*u8M@H=J^EZt?{G!jAi z-B(GI?|=7f_ovZI;UHDOs;_K3|L%9K% zr|Rk@6Xnk45V34NAV-+_S9YKm35MQ^_A?L5rebMr`T2oGu~OFH=UAe({=D?VqLiPL zc(1*=Q{}!P(S!VE_gfl8=s6fa%r_z5H zll%Aol2w3;?MtgijD~Ez#lgxJpO(KUI`xUuY~bC^H&fOr=O$KA4q2GM&%Ni8uFN4_ z<~hHC@%k|%H_&grw`A}mi2%|)o@y-mUkXSg7J8JVNmg*KJcHBU*H(0Mt{a!2${n_6 z5YGdtZ3t7NlcHO`P1+X#X)|S^Dp+bp%3r8C^F!k4uwEufPRsG-YFMCY=e|@vI|95k1tM^s^e~hOtz6}91 z3vf#uUVOz*4@f{=a&f_uX>jo!F8VxI;K0;;_(TEHE`6**?tNUkH>Dm4x0UlR zx{{Vyc1V`Y{-q78*g6=LpU}|2>Ngu&6vsb%(XxdU>7FKQqX)pLL5D za7C`K4U3p~rmZIFH>O6LC(FV8d5 zY$lkgx3bp~0E%D3YoFdSAK;m`iLTl=MEsw*O!^8dzQexc&m2H4lI;L*yLff`nLj}u zyw<_7+6Qs!6A z{@+J=zOmc5jG>F($zLFzw_0a2CNUnW2!{CLSQ0e@67qQ(g91}w_vB!Wyg5KCp>!}S z5oYI%X%Y>xUo=m^lE30UOu`ew6Y!4sv1w-k*(HZe!QtVNJ1-Hdop7n90*3$VkWc~0 zvz*!9YW>Uw;do#yjmZq+luQJllPjJH67on}>~so=S|b{YcSBHwDUzjR!UdVbA5Vjz z8wA~mhq5a{Wobu4-oZM-JbD(81Iu^Rj4GludI~0{8NMjlTR)hCcPrBbWPAgXBm&qFT;&uU> zI=00pHX4sdbdpejR;?I~rItE~$s7gSwOX%JNXV%1JkxH*x}$)+ z9xKWcq_;dF#}N^U`IIFd$q{TKwzNB13f~lz^Z4PR3Ft5(9Zvo!d6H5!`|-&KbnFG* zA+gs_$VW1!61F!ta4#X(+mYv(A2dVjhu49kFCM+RLNir(VuP| z>KU1NMfOTVMMj;D9ogsyJw`UU!tqHqO}wt=G+`6K6d^OxNK27qN|a2X3eff!pe2Sm^~B^% zVb3@cN#_lq{Xi&*IgQAe#xw~*m<}S%oY^>2AoEjH2n_|7(r+W-hNivp5iuhx#Hyir z+{8>33o*xwYpiGymT)a;8&IalnkCX9yVWvIe8mNV6mk)flwN7rrPW*_cn_N9438iq zBmG!1JpwU>GXXF%z{$XAFZp2{;C74}#fNLT227AJ6GE#70v1e@q5J{oL6e7VVhL7A z#6ep1#++EM=Z;Irp$-VDXJeivkv7%vWP89BU2KUpbfCv)_AK!vfe^8*<$p!NYPI%I zED%b5&GU9_gvRY1afD4XHPZ|#M0Vvt|K{`%`_rOf)R<3b*MHN7rI8(!o;PjLfh?^e zubaY2L6EtbSx1_+lWq}wL{sc_o7GaHHET)gVU3!P1{wWOg5h1I4V>^?b2KV} zci>EM+Rih*zHue7WE#8kqnut9Ad?x@w^5CZ!k8~p5@b zUvXjtKQcjAW6e@EqG?VQ2u5S}FNny3!KGY7Wzen>2|^LUFRdh^u~ryubq%M#2v8r4 zfhJk1IeY;(ZdEh_c=zy-ygnv7nt7i)fL;XuJtCJ>l0P#Vvk8NU(5aSl%BKK#&UmXe zG~0X#OPDgCTSSL5m{Ww~QZR*!8%y}NKZ2&-?XDy}8=98|WN!xH#R_F4=2PF!u#&Dp+*^{;<=05a zg@905gRKJWRM13=ZoP;#hXHB|T8V4Qg&MAaIP~TmDFUS%zMKIvUoCB&Kv~834Rq2h zCvzkcw@VgKo-u2oD4z0|Pp=3a^M%!3oUrUTVQFc@G2EnyGzcGKr-&$xA3GAeAxjQg zt&cXk>&D3FG~rUqnUD5(E{8R}cyOXVwmy*+a(U_gd^4o^3>pONclg~F001y}M|mz~lA7{|ocs*^v;4@DX|KJQlHGS0SU zGoc;D$1J&kXc>dr;}MXJZB!;`EIBb3w2XlmA*hx>bH9iGHj#hSBQp?}dFU-_&Nb#+al zNO`yhumF+()>0Ktic`N<<0PeJjVvNE#A?|1DsD|awpu5?;$iuTi=^ba>X`c!k<5rM z0jRs^W~R980+CY2Iwv(#}EY#fP0+N#cqUC~z$DDEfV8!(m zMS?X4%M~@2r-e{sD0vlgYKhUl&~~g@q5|eB#VDJi8-P<=MA5HD@nbbhU-Yse76eas8VB;n=&}A+_owYFd(n2 zK}^sL!C6K{Jp~NU<#xOl+fv4uNJBa^cuzL^qktS}Q?@*=k!=N686M^2Q_2%bMhz!} zjD9dyacO`JEsd5Rx1r5wu4ax>v65A(2~|Yfp^9wuz1=4PIYZtOq?5|Sf9Q-11)4D zkA)+6>FhO7KNL)Dt_6^$UjwJv}Sz_);RMJse$s)ZM3Hn{h%2rvpE>SU_2$pEkvc3 zU3PheZgtsUpvQ)VE+BAGXpRu&hnY#=M=_^yOlU+?rPs1gQF>-aHZU*A$l6w#G95C_ znB7Sk)&|qOHoPjeWZB(_$V;Y|pj0f5$t4#TmeP=j^f2eCy0VMVx!f%}E{uZ?d_H7k zg`tPs!H;@*S@0soc||lWY138fPGn>b1vd0Ia@h2Mx2+6|n-E5X<*U8~CJM|`gLiD` z6;rQM*=ns?fMqwL9S}Mjjmxl@te7owf4xod49KpSan4Mu}OA;GLPJ=PSVae2p zZHM>@f6_`4LuNI zCXBYrLQ5Vy zjT=~%MZUaBLY7Q_AQ4BEG34_(Mrx46B1Z;Raw@JzgR8-u_~3lkvaA`jS|fkw<&Snd z>CTG%o-am3(F@SPL#P){2n0QiC0{kJ_`FXGGkTX4C!o#JoPJy>ZQ>UyGAZ__YPr7v znO8`;&q}idHEW1$Y|U@O)hIQSH}1DnG}CHnLgTo6jpLhNCLr|M&HZTsIpPZplok|R zP!>zjZ5Y=+B5Fk8K!))Rs+U6R9Uq^|%o;CIoK=9*{SA4;Umh&1y0$sYsF=q++sq9?M8L ziFu+KC%;Uj3kcY**U06J#rXslQ6$Y7Ol--xgYp$s4nLcrbvk;zbaD)h)=0?HD|Dt- zPm0j|x%xZb<6CnHmo}4*ipX}(13mvQnvB&g$Riio(i7vL#x!fi3jxjM^Q;T&~RKu(?5tO_XfkfL?&g72FA2_CYFjoH8x+LpP%QA{|3)gb0LG!h7D*g~Bvq@ybR z;Q5>*bhGG{E{e#Yw4NQ3a51N`nddRg=z?)!=iXvLL*eTQm}T*HPZ|pyJzi*U7$w96 zcHG7e$oo(ke%hj*KndT&7Kz2i=IlZrUt>s}_ zt7e5918V47DTP!7%zz3>HMGdUiJR( zt&QH-`~M#0!9_MGO9)TwSqETC`PYPWNI6ZoW$CtX5_z(tV{h)w;~MTtZg;yzhcGqB z*|IEi*_&81i85rRj&@sSi)g;3vwokka|xk!c8Rjt9cx{?)7W_?&G+hkXQj$Z#8;)i z$~81pO7PC`>lhW%-;Qdw)jIz$HtnDHX3j=jJ|vR+biI33O`8`iiPl^T#Zm6pH^CxP z2=lc0z&jAV4%dfOM~GOU*KTpKoEg&f1CPm(c#vb;oRsmdy?hw$KbFqM7np&^fz-P! z@@}C18xTkv0m2;}?|OgWa%$%T<0?+CbB2VWb6Ha(71w{V<%;tW~dbfeOPvauq#+i3Z536aQ7u;U^U}XIzz`F}Ae+l^VOQQZQ zp|gj2eBEmCAN+Xwzh?;EuL*F2{=d=Ntm*&Hzuy1zIFGOY`>aD`kyu>mpMwHeM}ZjX3btapnLw0@=w&5jnuw$!y8n1S=kdi68={U68i;Xi(wg&W5kg)4hkEj0?zP5Yz0I$~n)_dC<+g1H^gh-i@|mF>hPX+U9AglB$UXd~Bb`Npr-K3XMf*1MaskvH1(J9(2Y5Puc-{k`k` z5kRu=D&g@(czki?!n(Ek9d*|a*L~>(!&}_`w)9S<(v3(>x-MpW`Wqh~9;cNHxXRi@ zq3+(%&D#KmiMv&>G$7SYhq~WEFIW$Hod^~|@3Z@@T+vvApfpDhe`O>2;N8+#C_XM_ zk0zzufFh1=c|lcc&J$<`WHaao>!sHw`Z}MJLC36~2 zxR^ua;@l}qYIaTIRAh+~nN$!pBV~enGR{gD*_sE0;Nl3R#CuNf>p=c|WHuykHVVef zyI#$S4zqV7ndVzWouXbHcQx%U=QvJtFB#Lw`N}uVDn~sC;S3 zS1&|NBpHzY;94Nusdw3%>e!tKjo*4a^7Y z<&u`Uab?E}Hi${9H1~CQ&Jb3tqTBmA@-vv52;TW9?B*CujWHWU3}BrLupW? zyHewU2f>{b20X4^&&y1y6i-LjzgDY!_G-e~yL&!6-I{LY#~yH49DG;{k-J>EOF`2V z1g2^8%cRIS$L$Bzxw0Eeyg9n1EbJ8k;FjtlB{{yb`vr>py)rwz3y-VsCm0kv{yKKE z-&JGbUmbl@m5E#MOq0IguEDA+37-~f{|hk4I=(&Tfyn&}aL84&|M31F&;MujjeAf3 zyfOZ}(O<8f|Ji^3)&KVxk01Ya3TK`-9l`^7|Lmp%d&{iRQe{ZB*2@JEFz1Vvs=0-9 zKsI}?*wWA5%`8PK_#HTEEj#E=g1+eTEe>-prfh!d?yf$;C{x`z@Im9^rP;qfAg73^ zHAK)5@#Egz&e^pndUiI7aX6(A-r!}zfanpAKn}>eQ=rjrf+={Ngz!hH;ElL;PjS^T z`QUch56FkoR9@c;KliUez8zyXXJnK<`1c}%3cYiE?BN()v;QE7%tp`Ru%9w|vaxAo zRUzl5%M}ot6KxPF^W<@7yY$6zXuEhG@@xyFH0l63l7@j@hNBFopN^d5EZ@@7k4d zl~T)6suYN1K%VTxe0uH#jIREoyP%>Qv$20*QhuZs<4HH>)17Xu+LQYSO(ryzGp_Ck zpQ|Q!y3H*-S(4oDo%NDu+?VvpmHF$rOXUtZ=;b<euJ%^Y!`q{D(aMF8~1l|CR+&^Z>K~0B!#vcmMzZ diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index e4e6cdd4435..929fc415163 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP. value: {{ .Values.collector.drainTimeoutSeconds | quote }} {{- end -}} -{{- define "litellm.lensWorker.image" -}} -{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}} -{{- end -}} - {{- define "litellm.gateway.collectorSocketDir" -}} {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} @@ -402,95 +398,28 @@ shutdown drain window. {{- end }} {{- end -}} -{{- define "litellm.lensWorker.labels" -}} -{{- $labels := include "litellm.labels" . | fromYaml -}} -{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}} -{{- toYaml $labels -}} -{{- end -}} - -{{- define "litellm.lensWorker.serviceTokenSecretName" -}} -{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}} -{{- end -}} - -{{- define "litellm.lensWorker.gatewaySecretName" -}} -{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}} -{{- end -}} - -{{- define "litellm.lensWorker.bundledClickhouse" -}} -{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.publicUrl" -}} -{{- if .Values.lensWorker.publicUrl -}} -{{- .Values.lensWorker.publicUrl -}} -{{- else if eq (include "litellm.lens.mode" .) "external" -}} -{{- fail "lensWorker.publicUrl is required for external Lens" -}} -{{- else if .Values.lensWorker.ingress.enabled -}} -{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}} -{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}} -{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}} -{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}} -{{- else -}} -{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}} -{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.clickhouseName" -}} -{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}} -{{- end -}} - {{- define "litellm.lensConnectionEnv" -}} -{{- $mode := include "litellm.lens.mode" . -}} -{{- if ne $mode "disabled" }} -{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}} -{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}} +{{- with .Values.lensWorker }} +{{- if or .enabled (and .mode (ne .mode "disabled")) }} +{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }} {{- end }} -{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}} -{{- fail "lensWorker.gateway.secretName is required for external Lens" -}} -{{- end -}} +{{- end }} +{{- with .Values.lens }} +{{- if .enabled }} - name: LITELLM_LENS_URL - value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }} + value: {{ required "lens.url is required when lens.enabled is true" .url | quote }} - name: LITELLM_LENS_PUBLIC_URL - value: {{ include "litellm.lensWorker.publicUrl" . | quote }} + value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }} - name: LENS_GATEWAY_SECRET valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }} - key: {{ .Values.lensWorker.gateway.secretKey | quote }} + name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }} + key: {{ .gatewaySecret.key | quote }} - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} - key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} + name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }} + key: {{ .serviceTokenSecret.key | quote }} +{{- end }} {{- end }} {{- end -}} - -{{- define "litellm.lens.mode" -}} -{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}} -{{- if not (has $mode (list "bundled" "external" "disabled")) -}} -{{- fail "lensWorker.mode must be bundled, external or disabled" -}} -{{- end -}} -{{- $mode -}} -{{- end -}} - -{{- define "litellm.lens.render" -}} -{{- $root := .root -}} -{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}} -{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}} -{{- $_ := set $values "component" "lens-worker" -}} -{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}} -{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}} -{{- $_ := set $values.gateway "enabled" true -}} -{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}} -{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}} -{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}} -{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}} -{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}} -{{- $_ := set $values "ingestionUrl" $ingestion -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}} -{{- end -}} -{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}} -{{- end -}} diff --git a/helm/litellm-helm/templates/ingress.yaml b/helm/litellm-helm/templates/ingress.yaml index 97ebb054b04..3d6e3f4af7a 100644 --- a/helm/litellm-helm/templates/ingress.yaml +++ b/helm/litellm-helm/templates/ingress.yaml @@ -1,8 +1,6 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "litellm.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- $lensMode := include "litellm.lens.mode" . -}} -{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} @@ -46,18 +44,18 @@ spec: - host: {{ .host | quote }} http: paths: - {{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }} + {{- if and $.Values.lens.enabled $.Values.lens.service.name }} - path: /lens-ingest pathType: Prefix backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }} + name: {{ $.Values.lens.service.name }} port: - number: {{ $.Values.lensWorker.service.port }} + number: {{ $.Values.lens.service.port }} {{- else }} - serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }} - servicePort: {{ $.Values.lensWorker.service.port }} + serviceName: {{ $.Values.lens.service.name }} + servicePort: {{ $.Values.lens.service.port }} {{- end }} {{- end }} {{- range .paths }} diff --git a/helm/litellm-helm/templates/lens/clickhouse.yaml b/helm/litellm-helm/templates/lens/clickhouse.yaml deleted file mode 100644 index d9daafda6ba..00000000000 --- a/helm/litellm-helm/templates/lens/clickhouse.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/deployment.yaml b/helm/litellm-helm/templates/lens/deployment.yaml deleted file mode 100644 index 01a6acb1908..00000000000 --- a/helm/litellm-helm/templates/lens/deployment.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/ingress.yaml b/helm/litellm-helm/templates/lens/ingress.yaml deleted file mode 100644 index b5994b22d92..00000000000 --- a/helm/litellm-helm/templates/lens/ingress.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/secrets.yaml b/helm/litellm-helm/templates/lens/secrets.yaml deleted file mode 100644 index 76928c26e5f..00000000000 --- a/helm/litellm-helm/templates/lens/secrets.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/service.yaml b/helm/litellm-helm/templates/lens/service.yaml deleted file mode 100644 index 60c7e995134..00000000000 --- a/helm/litellm-helm/templates/lens/service.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "service") }} -{{- end }} diff --git a/helm/litellm-helm/tests/lens_connection_tests.yaml b/helm/litellm-helm/tests/lens_connection_tests.yaml new file mode 100644 index 00000000000..e01e49d0ffc --- /dev/null +++ b/helm/litellm-helm/tests/lens_connection_tests.yaml @@ -0,0 +1,128 @@ +suite: lens connection +templates: + - deployment.yaml + - configmap-litellm.yaml + - ingress.yaml +tests: + - it: connects the proxy to an external Lens + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_URL + value: http://lens.lens.svc:4318 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://lens.example.com/lens-ingest + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_GATEWAY_SECRET + valueFrom: + secretKeyRef: + name: lens-gateway + key: gateway-secret + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: lens-service + key: service-token + + - it: requires lens.url when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.url is required when lens.enabled is true + + - it: requires lens.publicUrl when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.publicUrl is required when lens.enabled is true + + - it: requires lens.gatewaySecret.name when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.gatewaySecret.name is required when lens.enabled is true + + - it: requires lens.serviceTokenSecret.name when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + asserts: + - failedTemplate: + errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true + + - it: routes /lens-ingest to the named Lens service ahead of the host paths + template: ingress.yaml + set: + ingress.enabled: true + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + lens.service.name: existing-lens + asserts: + - equal: + path: spec.rules[0].http.paths[0].path + value: /lens-ingest + - contains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + pathType: Prefix + backend: + service: + name: existing-lens + port: + number: 4318 + + - it: leaves /lens-ingest off the ingress without a Lens service name + template: ingress.yaml + set: + ingress.enabled: true + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - exists: + path: spec.rules[0].http.paths + - notContains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + any: true diff --git a/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml b/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml deleted file mode 100644 index 22dcc7e9f9c..00000000000 --- a/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml +++ /dev/null @@ -1,84 +0,0 @@ -suite: Lens endpoint defaults -templates: - - deployment.yaml - - configmap-litellm.yaml -set: - lensWorker.enabled: true - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - ingress.tls: - - hosts: [gateway.example] - secretName: tls -tests: - - it: derives the trace endpoint from the deployment hostname and TLS - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - it: preserves an explicitly configured public address - set: - lensWorker.publicUrl: https://custom.example/prefix - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://custom.example/prefix - - it: uses the dedicated Lens ingress when configured - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: - - hosts: [traces.example] - secretName: traces-tls - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTPS for a dedicated ALB ingress with certificate annotations - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.className: alb - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' - alb.ingress.kubernetes.io/ssl-redirect: "443" - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTP for a dedicated ingress without its own TLS - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: {} - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://traces.example - - it: uses HTTP when ingress has no TLS - set: - ingress.tls: [] - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://gateway.example/lens-ingest diff --git a/helm/litellm-helm/tests/lens_modes_tests.yaml b/helm/litellm-helm/tests/lens_modes_tests.yaml deleted file mode 100644 index e746db8f83e..00000000000 --- a/helm/litellm-helm/tests/lens_modes_tests.yaml +++ /dev/null @@ -1,187 +0,0 @@ ---- -suite: Lens deployment modes -templates: -- ingress.yaml -- configmap-litellm.yaml -- deployment.yaml -- lens/deployment.yaml -- lens/service.yaml -- lens/clickhouse.yaml -- lens/secrets.yaml -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: retains the existing deployment before transferring its release ownership - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.retainResources: true - asserts: - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep -- it: connects an external deployment without installing Lens resources - templates: - - lens/deployment.yaml - - lens/service.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - hasDocuments: - count: 0 -- it: sends delegated requests to the explicitly selected external Lens - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens.other-namespace:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: external-lens-signing - key: gateway-secret -- it: disables Lens explicitly even with a retained legacy enable flag - template: lens/deployment.yaml - set: - lensWorker.mode: disabled - lensWorker.enabled: true - asserts: - - hasDocuments: - count: 0 -- it: bundles Lens through explicit mode selection - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_MODE - value: standalone -- it: uses separate credentials for external identity and service access - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.serviceTokenSecret: {name: worker-service, key: token} - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: {name: worker-service, key: token} -- it: supplies the same separate signing secret to the bundled service - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} -- it: routes explicit bundled mode with the enable flag off - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: bundled - lensWorker.enabled: false - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: gateway-lens-worker - port: {number: 4318} -- it: routes external mode to its selected service - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "existing-lens" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: {number: 4318} -- it: omits ingestion when an external service has no ingress backend - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true -- it: omits ingestion for disabled mode with the enable flag on - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: disabled - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true diff --git a/helm/litellm-helm/tests/lens_modes_validation_tests.yaml b/helm/litellm-helm/tests/lens_modes_validation_tests.yaml deleted file mode 100644 index eaf7d58f6ac..00000000000 --- a/helm/litellm-helm/tests/lens_modes_validation_tests.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -suite: Lens deployment mode validation -templates: -- configmap-litellm.yaml -- deployment.yaml -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: requires a separate identity signing reference for external Lens - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: worker-service - asserts: - - failedTemplate: - errorMessage: lensWorker.gateway.secretName is required for external Lens -- it: requires an external service address - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - failedTemplate: - errorMessage: lensWorker.externalUrl is required for external Lens -- it: requires the shared server credential for an external service - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - asserts: - - failedTemplate: - errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens -- it: requires an agent reachable URL for external ingestion - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - lensWorker.publicUrl: '' - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required for external Lens -- it: rejects unsupported deployment modes - template: deployment.yaml - set: - lensWorker.mode: typo - asserts: - - failedTemplate: - errorMessage: lensWorker.mode must be bundled, external or disabled diff --git a/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml b/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml deleted file mode 100644 index c85f8d8c9e7..00000000000 --- a/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml +++ /dev/null @@ -1,59 +0,0 @@ -suite: Lens preserves credentials across Helm upgrades -templates: - - lens/secrets.yaml -set: - fullnameOverride: lens-test - lensWorker.enabled: true -release: - namespace: lens - name: lens-test - upgrade: true -kubernetesProvider: - scheme: - v1/Secret: - gvr: - version: v1 - resource: secrets - namespaced: true - objects: - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-gateway - namespace: lens - data: - gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk= - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-service - namespace: lens - data: - service-token: c2F2ZWQtc2VydmljZS10b2tlbg== - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-clickhouse - namespace: lens - data: - password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= -tests: - - it: preserves the separate signing credential across upgrades - documentSelector: {path: metadata.name, value: lens-test-lens-gateway} - asserts: - - equal: - path: data.gateway-secret - value: c2F2ZWQtc2lnbmluZy1rZXk= - - notExists: {path: data.service-token} - - it: reuses the service credential instead of breaking running services - documentIndex: 0 - asserts: - - equal: - path: data.service-token - value: c2F2ZWQtc2VydmljZS10b2tlbg== - - it: reuses the database password instead of locking out stored traces - documentIndex: 1 - asserts: - - equal: - path: data.password - value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= diff --git a/helm/litellm-helm/tests/lens_service_tests.yaml b/helm/litellm-helm/tests/lens_service_tests.yaml deleted file mode 100644 index 1beabca3064..00000000000 --- a/helm/litellm-helm/tests/lens_service_tests.yaml +++ /dev/null @@ -1,167 +0,0 @@ -suite: Lens service isolation and ingestion routing -templates: -- configmap-litellm.yaml -- deployment.yaml -- ingress.yaml -- lens/ingress.yaml -- lens/service.yaml -- lens/deployment.yaml -tests: -- it: connects deployment.yaml to the shared Lens service - template: deployment.yaml - set: &id001 - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens-test-lens-worker:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-test-lens-gateway - key: gateway-secret -- it: routes uploads directly to Lens instead of the gateway - template: ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - ingress.enabled: true - ingress.hosts: - - host: gateway.example - paths: - - path: / - pathType: Prefix - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - number: 4318 -- it: keeps internal routes out of a dedicated ingestion hostname - template: lens/ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - asserts: - - equal: - path: spec.rules[0].http.paths - value: - - path: /v1/ - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - name: otlp -- it: maps the Lens service to the ingestion listener - template: lens/service.yaml - set: *id001 - asserts: - - equal: - path: spec.selector - value: - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: lens-worker - - equal: - path: spec.ports - value: - - name: otlp - port: 4318 - targetPort: otlp -- it: gives only Lens the ClickHouse secret - template: lens/deployment.yaml - set: *id001 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_URL - valueFrom: - secretKeyRef: - name: lens-storage - key: url -- it: requires an agent reachable ingestion URL - template: deployment.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: '' - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname -- it: omits Lens connection settings when disabled in deployment.yaml - template: deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - any: true -- it: preserves an existing ClickHouse database and retention - template: lens/deployment.yaml - set: - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.clickhouseDatabase: existing_traces - lensWorker.retentionDays: 45 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_DATABASE - value: existing_traces - - contains: - path: spec.template.spec.containers[0].env - content: - name: AGENT_TRACING_RETENTION_DAYS - value: '45' -- it: keeps Lens pods outside the gateway autoscaling selector - template: lens/deployment.yaml - set: &id002 - nameOverride: inference - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/name"] - value: inference-lens-worker -- it: preserves the existing gateway deployment selector - template: deployment.yaml - set: *id002 - asserts: - - equal: - path: spec.selector.matchLabels["app.kubernetes.io/name"] - value: inference diff --git a/helm/litellm-helm/tests/lens_setup_tests.yaml b/helm/litellm-helm/tests/lens_setup_tests.yaml deleted file mode 100644 index 20cd8590153..00000000000 --- a/helm/litellm-helm/tests/lens_setup_tests.yaml +++ /dev/null @@ -1,162 +0,0 @@ -suite: Lens managed setup -set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example -release: - namespace: lens - name: lens-test -templates: - - lens/deployment.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml -tests: - - it: generates private login, gateway and storage credentials - template: lens/secrets.yaml - asserts: - - hasDocuments: - count: 4 - - matchRegex: - path: data.service-token - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 0 - - matchRegex: - path: data.password - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 1 - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - - it: connects Lens to its private bundled storage - template: lens/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: lens-test-lens-clickhouse - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: stores traces on a persistent volume with the chosen storage class - template: lens/clickhouse.yaml - documentIndex: 1 - set: - lensWorker.clickhouse.storage: 40Gi - lensWorker.clickhouse.storageClassName: fast - asserts: - - equal: - path: spec.volumeClaimTemplates[0].spec - value: - accessModes: [ReadWriteOnce] - storageClassName: fast - resources: - requests: - storage: 40Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: keeps an existing external database instead of creating a new one - template: lens/clickhouse.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - asserts: - - hasDocuments: - count: 0 - - it: leaves supplied secrets under their existing manager - template: lens/secrets.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - lensWorker.serviceTokenSecret.name: external-service - lensWorker.adminTokenSecret.name: external-admin - lensWorker.gateway.secretName: external-signing - asserts: - - hasDocuments: - count: 0 - - it: creates no credentials or database when Lens is disabled - templates: - - lens/secrets.yaml - - lens/clickhouse.yaml - set: - lensWorker.enabled: false - asserts: - - hasDocuments: - count: 0 - - it: requires an external database when bundled storage is explicitly disabled - template: lens/deployment.yaml - set: - lensWorker.clickhouse.enabled: false - asserts: - - failedTemplate: - errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled - - it: keeps storage names valid and uses the same name for the connection - set: - fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - asserts: - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 0 - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 0 - - equal: - path: spec.serviceName - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/secrets.yaml - documentIndex: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/deployment.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - key: password - template: lens/deployment.yaml - - - it: allows cold storage startup and inherits registry credentials - template: lens/clickhouse.yaml - documentIndex: 1 - set: - imagePullSecrets: [{name: registry-auth}] - asserts: - - equal: - path: spec.template.spec.imagePullSecrets - value: [{name: registry-auth}] - - equal: - path: spec.template.spec.containers[0].startupProbe - value: - httpGet: {path: /ping, port: http} - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 60 diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 8450c5f2840..f51f8bdd173 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -668,70 +668,15 @@ serviceMonitor: # - test-namespace lens: - library: true - -lensWorker: - retainResources: false - mode: "" - externalUrl: "" - externalServiceName: "" - gateway: - secretName: "" - secretKey: gateway-secret - standaloneUrl: "" - adminTokenSecret: - name: "" - key: admin-token - extraEnv: [] - extraEnvFrom: [] enabled: false - replicaCount: 1 - image: - repository: ghcr.io/berriai/lens - tag: "0.1.0-dev.0" - digest: "" - pullPolicy: IfNotPresent - tokenSecret: + url: "" + publicUrl: "" + service: name: "" - key: token + port: 4318 + gatewaySecret: + name: "" + key: gateway-secret serviceTokenSecret: name: "" key: service-token - clickhouse: - enabled: true - image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e - storage: 20Gi - storageClassName: null - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 2Gi - clickhouseDatabase: litellm - retentionDays: 14 - clickhouseSecret: - name: "" - key: url - publicUrl: "" - service: - port: 4318 - annotations: {} - ingress: - enabled: false - path: /v1/ - className: "" - host: "" - annotations: {} - tls: [] - url: "" - tmpSizeLimit: 1Gi - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 2Gi - nodeSelector: {} - tolerations: [] - affinity: {} diff --git a/helm/litellm/Chart.lock b/helm/litellm/Chart.lock deleted file mode 100644 index d9f261935ca..00000000000 --- a/helm/litellm/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: lens - repository: oci://ghcr.io/berriai/charts - version: 0.1.0-dev.0 -digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf -generated: '2026-10-09T04:18:10.642706000Z' diff --git a/helm/litellm/Chart.yaml b/helm/litellm/Chart.yaml index 36780b808dc..e67f5790c7e 100644 --- a/helm/litellm/Chart.yaml +++ b/helm/litellm/Chart.yaml @@ -6,8 +6,3 @@ version: 0.1.0 appVersion: "0.1.0" annotations: org.opencontainers.image.source: "https://github.com/BerriAI/litellm" - -dependencies: - - name: lens - version: 0.1.0-dev.0 - repository: oci://ghcr.io/berriai/charts diff --git a/helm/litellm/README.md b/helm/litellm/README.md index 271052dd09f..97b94754402 100644 --- a/helm/litellm/README.md +++ b/helm/litellm/README.md @@ -1,10 +1,8 @@ # LiteLLM componentized chart -This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service +This chart deploys separate gateway, backend and UI services -Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment - -Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published +This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured. diff --git a/helm/litellm/charts/lens-0.1.0-dev.0.tgz b/helm/litellm/charts/lens-0.1.0-dev.0.tgz deleted file mode 100644 index 79021702c2ca9a8eb8f60bfb0ece3616903c7fdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9226 zcmV+lB=y@LiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhciT3$=zP|%z*2iwlCvyImLF+VH)~wSO?_%RzLt|tZ*H#z zky{eR6u|9niO0sq#(o2cA(7p^85JtHqVxD~5ApPR zz25fbCjQ^+^{W5(HrKcQw!YQh?)NvI_cosYt+(D^-(3G2={*QEl_!&miof+fyRYit z{zD#`BwSI&cp?WaLNqx9F;fu7^MJB$D!Aeyj|Zgvyxk&%i)p}9NJ5?{$`TMVU~@W! zK=Mq4Fd#FfQaR{$r%cVVaS-x(_a_j7?H+V>+bx>1w;&|r$$%{SEt;nJ&t9+|^g0nN zf?g{E848vv{IENPL=h!u2y%_EWC2pKsR4S8Ojr`JWGV?wA_5Br7Eh+66@ zg#n>y8nY0;wiZpje(A~h0uQc#jWm!C#sF?u|NZ{+es%qC_WEDf|6@Ec8w)C~24q5G z39U&M$9mnoUg$*|!GN^et?~<8P%M#(CSkdp0L>Fl!SDj<+6?(TrWq+hu9G~!*g;xV|FgfPx zM1X`unYK@|I37YMK*<4l_eYCHbC#U(3rNgc-M3*D*w_VJ4G5O$C|#&Er3xJKI(?p7T-Qv%V zh6co-za4xZYzNzamowVm+8)4o^SkFgcplQ{5IygO{jKi-#*=U|2G|ZK&%b}(TYo;; z9&c@KY=`~t``hF3cmf;IX4L;~^Sf;gNhU!INCXp_#Y*E+ae;vJd#_mWdoQL^8XP5A z93wS3I&2Z5fBu<)RQ4AkVVVuddapOPzvnPV!rNN!A2ECvvpFluyYKy1%){MFs_2+X zw=9Ye%PA`pv0f>V=-GHlujGKNZ#qKI6;dwLfZ(hV;>&09;abn504W=3F{NrYAl()r zVSx?o80oy|FSCWIwnoJFqC zI7^}!qCKx{Ky2Y}HNzu=A1hUgb~RtJvTFLWYaX2-2&r^Dwt&P5Y!`K-HN;8h7PQHsKufl zxQ1A2Y5A7N*<8zcHa~UZ)9QRU>dP4&@8JHt(gn`*3_DL&WwzFph9VW@wh+FR%^`qAWd$<(ZX zwW7fnFx7PX99G?k;=j{*OahMrzboqM5|Bj23khK+n7Z2IiGug4XiH?tt~}<+Dd#Fz zkG)TCP(NMot(RZF5PAIYB?SEVUmffbz8IOdlpf-P7KQ`nawjj3_i_!DX&rYRk>m?;Lb z+zKI);OT(8+dVw|!}|isqFm4oaPRP7@7G^mzZveIzZveIRvJTB#Wx%<5vAOkOAP zRY}Si3^efh8psBiBbt^G)x;7Og(`LCOh2bwR#gmAN3<{5387&KQXcULw2^)aH2O_2 z1+SA3{wPUgHX#W#Q9{*)LlPGIbWnHkG_&_wI}NP3Cis_7Z`EB?g5(AO{C=y7cdm2q zx#@afjoYAB_(FTP*@Zhfbl|13(FD(5VE zvA)wSeiQ{_K0UWsI@c`dMZ*g1hMbFtB{;xt%%?luS~aiBgr;)F)g7C4)#OgMxrL$! zzuvh7rC2hRFFd-~;i+{~om(z>K9?`nd%a$7r`!0=ul#4mMK(Xzt6#p*D}Se3dQ||@ zy#bOPK_}|m@PW*9TI$J8=#H5l7QNV+aj7(+y;$!*4|+i_Sl{X9?{>^w_cu0oy7;G? zQXiz{UG8)n>1gL)oSP3Xb`ll_Kqc5X^{i~1FMozoPj?mQJRVeelcSGGWO z<}}R#supZn?0PG~V-xPr&h=l>=UDasMUck)Y7U8dJpbSG&F3}$-}c5=|KDRgHUD2h z0LK5f+-vkQFWD@@$}-DE7Om&+bH!TO_fc=ld~KKP|4rWmRk)cvS?-4>O)ob(it1gv zDu)dqY@1f^=GD2~=yLDI|5uKe?X9hiexvK9{AJSo`rh3yzGCS7V`ZDB@Wqtp6yKXE zZEzsuICo)g-<^jUI7JNnfqXd5KG75MGmAl9Nd@!D)R6mi_8jgX56_QYzua&3H%LX3 zh{ncmzohobo1YF3_Rim&9xnCfXl0)Xh)DYo5=ru4I$<*Z*3Qvge7?ceb9ivJe|UIw zep@gK_iPcX_g%Ah?qB!2`|{}E`26hkulvUj;Ag5Z@JeEZi}Wmk(>&;p@*-s-ZA{Ze(eYJJAWb$ zyyaNOX0V?*u8M@H=J^EZt?{G!jAi z-B(GI?|=7f_ovZI;UHDOs;_K3|L%9K% zr|Rk@6Xnk45V34NAV-+_S9YKm35MQ^_A?L5rebMr`T2oGu~OFH=UAe({=D?VqLiPL zc(1*=Q{}!P(S!VE_gfl8=s6fa%r_z5H zll%Aol2w3;?MtgijD~Ez#lgxJpO(KUI`xUuY~bC^H&fOr=O$KA4q2GM&%Ni8uFN4_ z<~hHC@%k|%H_&grw`A}mi2%|)o@y-mUkXSg7J8JVNmg*KJcHBU*H(0Mt{a!2${n_6 z5YGdtZ3t7NlcHO`P1+X#X)|S^Dp+bp%3r8C^F!k4uwEufPRsG-YFMCY=e|@vI|95k1tM^s^e~hOtz6}91 z3vf#uUVOz*4@f{=a&f_uX>jo!F8VxI;K0;;_(TEHE`6**?tNUkH>Dm4x0UlR zx{{Vyc1V`Y{-q78*g6=LpU}|2>Ngu&6vsb%(XxdU>7FKQqX)pLL5D za7C`K4U3p~rmZIFH>O6LC(FV8d5 zY$lkgx3bp~0E%D3YoFdSAK;m`iLTl=MEsw*O!^8dzQexc&m2H4lI;L*yLff`nLj}u zyw<_7+6Qs!6A z{@+J=zOmc5jG>F($zLFzw_0a2CNUnW2!{CLSQ0e@67qQ(g91}w_vB!Wyg5KCp>!}S z5oYI%X%Y>xUo=m^lE30UOu`ew6Y!4sv1w-k*(HZe!QtVNJ1-Hdop7n90*3$VkWc~0 zvz*!9YW>Uw;do#yjmZq+luQJllPjJH67on}>~so=S|b{YcSBHwDUzjR!UdVbA5Vjz z8wA~mhq5a{Wobu4-oZM-JbD(81Iu^Rj4GludI~0{8NMjlTR)hCcPrBbWPAgXBm&qFT;&uU> zI=00pHX4sdbdpejR;?I~rItE~$s7gSwOX%JNXV%1JkxH*x}$)+ z9xKWcq_;dF#}N^U`IIFd$q{TKwzNB13f~lz^Z4PR3Ft5(9Zvo!d6H5!`|-&KbnFG* zA+gs_$VW1!61F!ta4#X(+mYv(A2dVjhu49kFCM+RLNir(VuP| z>KU1NMfOTVMMj;D9ogsyJw`UU!tqHqO}wt=G+`6K6d^OxNK27qN|a2X3eff!pe2Sm^~B^% zVb3@cN#_lq{Xi&*IgQAe#xw~*m<}S%oY^>2AoEjH2n_|7(r+W-hNivp5iuhx#Hyir z+{8>33o*xwYpiGymT)a;8&IalnkCX9yVWvIe8mNV6mk)flwN7rrPW*_cn_N9438iq zBmG!1JpwU>GXXF%z{$XAFZp2{;C74}#fNLT227AJ6GE#70v1e@q5J{oL6e7VVhL7A z#6ep1#++EM=Z;Irp$-VDXJeivkv7%vWP89BU2KUpbfCv)_AK!vfe^8*<$p!NYPI%I zED%b5&GU9_gvRY1afD4XHPZ|#M0Vvt|K{`%`_rOf)R<3b*MHN7rI8(!o;PjLfh?^e zubaY2L6EtbSx1_+lWq}wL{sc_o7GaHHET)gVU3!P1{wWOg5h1I4V>^?b2KV} zci>EM+Rih*zHue7WE#8kqnut9Ad?x@w^5CZ!k8~p5@b zUvXjtKQcjAW6e@EqG?VQ2u5S}FNny3!KGY7Wzen>2|^LUFRdh^u~ryubq%M#2v8r4 zfhJk1IeY;(ZdEh_c=zy-ygnv7nt7i)fL;XuJtCJ>l0P#Vvk8NU(5aSl%BKK#&UmXe zG~0X#OPDgCTSSL5m{Ww~QZR*!8%y}NKZ2&-?XDy}8=98|WN!xH#R_F4=2PF!u#&Dp+*^{;<=05a zg@905gRKJWRM13=ZoP;#hXHB|T8V4Qg&MAaIP~TmDFUS%zMKIvUoCB&Kv~834Rq2h zCvzkcw@VgKo-u2oD4z0|Pp=3a^M%!3oUrUTVQFc@G2EnyGzcGKr-&$xA3GAeAxjQg zt&cXk>&D3FG~rUqnUD5(E{8R}cyOXVwmy*+a(U_gd^4o^3>pONclg~F001y}M|mz~lA7{|ocs*^v;4@DX|KJQlHGS0SU zGoc;D$1J&kXc>dr;}MXJZB!;`EIBb3w2XlmA*hx>bH9iGHj#hSBQp?}dFU-_&Nb#+al zNO`yhumF+()>0Ktic`N<<0PeJjVvNE#A?|1DsD|awpu5?;$iuTi=^ba>X`c!k<5rM z0jRs^W~R980+CY2Iwv(#}EY#fP0+N#cqUC~z$DDEfV8!(m zMS?X4%M~@2r-e{sD0vlgYKhUl&~~g@q5|eB#VDJi8-P<=MA5HD@nbbhU-Yse76eas8VB;n=&}A+_owYFd(n2 zK}^sL!C6K{Jp~NU<#xOl+fv4uNJBa^cuzL^qktS}Q?@*=k!=N686M^2Q_2%bMhz!} zjD9dyacO`JEsd5Rx1r5wu4ax>v65A(2~|Yfp^9wuz1=4PIYZtOq?5|Sf9Q-11)4D zkA)+6>FhO7KNL)Dt_6^$UjwJv}Sz_);RMJse$s)ZM3Hn{h%2rvpE>SU_2$pEkvc3 zU3PheZgtsUpvQ)VE+BAGXpRu&hnY#=M=_^yOlU+?rPs1gQF>-aHZU*A$l6w#G95C_ znB7Sk)&|qOHoPjeWZB(_$V;Y|pj0f5$t4#TmeP=j^f2eCy0VMVx!f%}E{uZ?d_H7k zg`tPs!H;@*S@0soc||lWY138fPGn>b1vd0Ia@h2Mx2+6|n-E5X<*U8~CJM|`gLiD` z6;rQM*=ns?fMqwL9S}Mjjmxl@te7owf4xod49KpSan4Mu}OA;GLPJ=PSVae2p zZHM>@f6_`4LuNI zCXBYrLQ5Vy zjT=~%MZUaBLY7Q_AQ4BEG34_(Mrx46B1Z;Raw@JzgR8-u_~3lkvaA`jS|fkw<&Snd z>CTG%o-am3(F@SPL#P){2n0QiC0{kJ_`FXGGkTX4C!o#JoPJy>ZQ>UyGAZ__YPr7v znO8`;&q}idHEW1$Y|U@O)hIQSH}1DnG}CHnLgTo6jpLhNCLr|M&HZTsIpPZplok|R zP!>zjZ5Y=+B5Fk8K!))Rs+U6R9Uq^|%o;CIoK=9*{SA4;Umh&1y0$sYsF=q++sq9?M8L ziFu+KC%;Uj3kcY**U06J#rXslQ6$Y7Ol--xgYp$s4nLcrbvk;zbaD)h)=0?HD|Dt- zPm0j|x%xZb<6CnHmo}4*ipX}(13mvQnvB&g$Riio(i7vL#x!fi3jxjM^Q;T&~RKu(?5tO_XfkfL?&g72FA2_CYFjoH8x+LpP%QA{|3)gb0LG!h7D*g~Bvq@ybR z;Q5>*bhGG{E{e#Yw4NQ3a51N`nddRg=z?)!=iXvLL*eTQm}T*HPZ|pyJzi*U7$w96 zcHG7e$oo(ke%hj*KndT&7Kz2i=IlZrUt>s}_ zt7e5918V47DTP!7%zz3>HMGdUiJR( zt&QH-`~M#0!9_MGO9)TwSqETC`PYPWNI6ZoW$CtX5_z(tV{h)w;~MTtZg;yzhcGqB z*|IEi*_&81i85rRj&@sSi)g;3vwokka|xk!c8Rjt9cx{?)7W_?&G+hkXQj$Z#8;)i z$~81pO7PC`>lhW%-;Qdw)jIz$HtnDHX3j=jJ|vR+biI33O`8`iiPl^T#Zm6pH^CxP z2=lc0z&jAV4%dfOM~GOU*KTpKoEg&f1CPm(c#vb;oRsmdy?hw$KbFqM7np&^fz-P! z@@}C18xTkv0m2;}?|OgWa%$%T<0?+CbB2VWb6Ha(71w{V<%;tW~dbfeOPvauq#+i3Z536aQ7u;U^U}XIzz`F}Ae+l^VOQQZQ zp|gj2eBEmCAN+Xwzh?;EuL*F2{=d=Ntm*&Hzuy1zIFGOY`>aD`kyu>mpMwHeM}ZjX3btapnLw0@=w&5jnuw$!y8n1S=kdi68={U68i;Xi(wg&W5kg)4hkEj0?zP5Yz0I$~n)_dC<+g1H^gh-i@|mF>hPX+U9AglB$UXd~Bb`Npr-K3XMf*1MaskvH1(J9(2Y5Puc-{k`k` z5kRu=D&g@(czki?!n(Ek9d*|a*L~>(!&}_`w)9S<(v3(>x-MpW`Wqh~9;cNHxXRi@ zq3+(%&D#KmiMv&>G$7SYhq~WEFIW$Hod^~|@3Z@@T+vvApfpDhe`O>2;N8+#C_XM_ zk0zzufFh1=c|lcc&J$<`WHaao>!sHw`Z}MJLC36~2 zxR^ua;@l}qYIaTIRAh+~nN$!pBV~enGR{gD*_sE0;Nl3R#CuNf>p=c|WHuykHVVef zyI#$S4zqV7ndVzWouXbHcQx%U=QvJtFB#Lw`N}uVDn~sC;S3 zS1&|NBpHzY;94Nusdw3%>e!tKjo*4a^7Y z<&u`Uab?E}Hi${9H1~CQ&Jb3tqTBmA@-vv52;TW9?B*CujWHWU3}BrLupW? zyHewU2f>{b20X4^&&y1y6i-LjzgDY!_G-e~yL&!6-I{LY#~yH49DG;{k-J>EOF`2V z1g2^8%cRIS$L$Bzxw0Eeyg9n1EbJ8k;FjtlB{{yb`vr>py)rwz3y-VsCm0kv{yKKE z-&JGbUmbl@m5E#MOq0IguEDA+37-~f{|hk4I=(&Tfyn&}aL84&|M31F&;MujjeAf3 zyfOZ}(O<8f|Ji^3)&KVxk01Ya3TK`-9l`^7|Lmp%d&{iRQe{ZB*2@JEFz1Vvs=0-9 zKsI}?*wWA5%`8PK_#HTEEj#E=g1+eTEe>-prfh!d?yf$;C{x`z@Im9^rP;qfAg73^ zHAK)5@#Egz&e^pndUiI7aX6(A-r!}zfanpAKn}>eQ=rjrf+={Ngz!hH;ElL;PjS^T z`QUch56FkoR9@c;KliUez8zyXXJnK<`1c}%3cYiE?BN()v;QE7%tp`Ru%9w|vaxAo zRUzl5%M}ot6KxPF^W<@7yY$6zXuEhG@@xyFH0l63l7@j@hNBFopN^d5EZ@@7k4d zl~T)6suYN1K%VTxe0uH#jIREoyP%>Qv$20*QhuZs<4HH>)17Xu+LQYSO(ryzGp_Ck zpQ|Q!y3H*-S(4oDo%NDu+?VvpmHF$rOXUtZ=;b<euJ%^Y!`q{D(aMF8~1l|CR+&^Z>K~0B!#vcmMzZ diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 8cb2fbf441d..18e938564e9 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and collector containers through an emptyDir. Empty when the sidecar is off or gateway.collector.address is a tcp://127.0.0.1: address. */}} -{{- define "litellm.lensWorker.image" -}} -{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}} -{{- end -}} - {{- define "litellm.gateway.collectorSocketDir" -}} {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} @@ -635,93 +631,27 @@ shutdown drain window. {{- end -}} {{- define "litellm.lensConnectionEnv" -}} -{{- $mode := include "litellm.lens.mode" . -}} -{{- if ne $mode "disabled" }} -{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}} -{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}} +{{- with .Values.lensWorker }} +{{- if or .enabled (and .mode (ne .mode "disabled")) }} +{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }} {{- end }} -{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}} -{{- fail "lensWorker.gateway.secretName is required for external Lens" -}} -{{- end -}} +{{- end }} +{{- with .Values.lens }} +{{- if .enabled }} - name: LITELLM_LENS_URL - value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }} + value: {{ required "lens.url is required when lens.enabled is true" .url | quote }} - name: LITELLM_LENS_PUBLIC_URL - value: {{ include "litellm.lensWorker.publicUrl" . | quote }} + value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }} - name: LENS_GATEWAY_SECRET valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }} - key: {{ .Values.lensWorker.gateway.secretKey | quote }} + name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }} + key: {{ .gatewaySecret.key | quote }} - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} - key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} + name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }} + key: {{ .serviceTokenSecret.key | quote }} +{{- end }} {{- end }} {{- end -}} - -{{- define "litellm.lensWorker.labels" -}} -{{- $labels := include "litellm.commonLabels" . | fromYaml -}} -{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}} -{{- toYaml $labels -}} -{{- end -}} - -{{- define "litellm.lensWorker.serviceTokenSecretName" -}} -{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}} -{{- end -}} - -{{- define "litellm.lensWorker.gatewaySecretName" -}} -{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}} -{{- end -}} - -{{- define "litellm.lensWorker.bundledClickhouse" -}} -{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.publicUrl" -}} -{{- if .Values.lensWorker.publicUrl -}} -{{- .Values.lensWorker.publicUrl -}} -{{- else if eq (include "litellm.lens.mode" .) "external" -}} -{{- fail "lensWorker.publicUrl is required for external Lens" -}} -{{- else if .Values.lensWorker.ingress.enabled -}} -{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}} -{{- else if and .Values.ingress.enabled .Values.ingress.host -}} -{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}} -{{- else -}} -{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}} -{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.clickhouseName" -}} -{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}} -{{- end -}} - -{{- define "litellm.lens.mode" -}} -{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}} -{{- if not (has $mode (list "bundled" "external" "disabled")) -}} -{{- fail "lensWorker.mode must be bundled, external or disabled" -}} -{{- end -}} -{{- $mode -}} -{{- end -}} - -{{- define "litellm.lens.render" -}} -{{- $root := .root -}} -{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}} -{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}} -{{- $_ := set $values "component" "lens-worker" -}} -{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}} -{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}} -{{- $_ := set $values.gateway "enabled" true -}} -{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}} -{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}} -{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}} -{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}} -{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}} -{{- $_ := set $values "ingestionUrl" $ingestion -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}} -{{- end -}} -{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}} -{{- end -}} diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index e5dcfd28316..9f778e78087 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -166,14 +166,14 @@ spec: - host: {{ $componentHosts.gateway | quote }} http: paths: - {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }} + {{- if and .Values.lens.enabled .Values.lens.service.name }} - path: /lens-ingest pathType: Prefix backend: service: - name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }} + name: {{ .Values.lens.service.name }} port: - number: {{ .Values.lensWorker.service.port }} + number: {{ .Values.lens.service.port }} {{- end }} - path: / pathType: Prefix @@ -249,15 +249,15 @@ spec: port: number: {{ $gatewayPort }} {{- end }} - {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }} + {{- if and .Values.lens.enabled .Values.lens.service.name }} {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }} - path: /lens-ingest pathType: Prefix backend: service: - name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }} + name: {{ .Values.lens.service.name }} port: - number: {{ .Values.lensWorker.service.port }} + number: {{ .Values.lens.service.port }} {{- end }} {{- /* --- Operator-supplied extra paths (ingress.extraPaths) --- diff --git a/helm/litellm/templates/lens/clickhouse.yaml b/helm/litellm/templates/lens/clickhouse.yaml deleted file mode 100644 index d9daafda6ba..00000000000 --- a/helm/litellm/templates/lens/clickhouse.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }} -{{- end }} diff --git a/helm/litellm/templates/lens/deployment.yaml b/helm/litellm/templates/lens/deployment.yaml deleted file mode 100644 index 01a6acb1908..00000000000 --- a/helm/litellm/templates/lens/deployment.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }} -{{- end }} diff --git a/helm/litellm/templates/lens/ingress.yaml b/helm/litellm/templates/lens/ingress.yaml deleted file mode 100644 index b5994b22d92..00000000000 --- a/helm/litellm/templates/lens/ingress.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }} -{{- end }} diff --git a/helm/litellm/templates/lens/secrets.yaml b/helm/litellm/templates/lens/secrets.yaml deleted file mode 100644 index 76928c26e5f..00000000000 --- a/helm/litellm/templates/lens/secrets.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }} -{{- end }} diff --git a/helm/litellm/templates/lens/service.yaml b/helm/litellm/templates/lens/service.yaml deleted file mode 100644 index 60c7e995134..00000000000 --- a/helm/litellm/templates/lens/service.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "service") }} -{{- end }} diff --git a/helm/litellm/tests/component_hosts_tests.yaml b/helm/litellm/tests/component_hosts_tests.yaml index f9c9ce6fad2..b55b3b694a0 100644 --- a/helm/litellm/tests/component_hosts_tests.yaml +++ b/helm/litellm/tests/component_hosts_tests.yaml @@ -15,7 +15,6 @@ tests: ingress.componentHosts.ui: ui.example.com ingress.componentHosts.backend: backend.example.com ingress.componentHosts.gateway: gateway.example.com - lensWorker.mode: disabled ingress.tls: - hosts: - ui.example.com @@ -83,12 +82,12 @@ tests: ui.backendUrl: https://backend.example.com ui.publicUrl: https://ui.example.com backend.gatewayUrl: https://gateway.example.com - lensWorker.mode: external - lensWorker.externalServiceName: existing-lens - lensWorker.externalUrl: https://lens.example.com - lensWorker.publicUrl: https://lens.example.com - lensWorker.serviceTokenSecret.name: existing-lens-signing - lensWorker.gateway.secretName: existing-lens-signing + lens.enabled: true + lens.service.name: existing-lens + lens.url: http://existing-lens:4318 + lens.publicUrl: https://gateway.example.com/lens-ingest + lens.serviceTokenSecret.name: existing-lens-signing + lens.gatewaySecret.name: existing-lens-signing asserts: - equal: path: spec.rules[2].http.paths[0].path diff --git a/helm/litellm/tests/lens_connection_tests.yaml b/helm/litellm/tests/lens_connection_tests.yaml new file mode 100644 index 00000000000..2b7c32d50cc --- /dev/null +++ b/helm/litellm/tests/lens_connection_tests.yaml @@ -0,0 +1,166 @@ +suite: lens connection +templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - gateway/configmap.yaml + - ingress.yaml +values: + - ./values/required.yaml +tests: + - it: connects the gateway to an external Lens + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_URL + value: http://lens.lens.svc:4318 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://lens.example.com/lens-ingest + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_GATEWAY_SECRET + valueFrom: + secretKeyRef: + name: lens-gateway + key: gateway-secret + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: lens-service + key: service-token + + - it: connects the backend to an external Lens + template: backend/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_URL + value: http://lens.lens.svc:4318 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://lens.example.com/lens-ingest + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_GATEWAY_SECRET + valueFrom: + secretKeyRef: + name: lens-gateway + key: gateway-secret + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: lens-service + key: service-token + + - it: requires lens.url when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.url is required when lens.enabled is true + + - it: requires lens.publicUrl when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.publicUrl is required when lens.enabled is true + + - it: requires lens.gatewaySecret.name when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.gatewaySecret.name is required when lens.enabled is true + + - it: requires lens.serviceTokenSecret.name when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + asserts: + - failedTemplate: + errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true + + - it: routes /lens-ingest to the named Lens service on the single ingress host + template: ingress.yaml + set: + ingress.enabled: true + ingress.host: litellm.example.com + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + lens.service.name: existing-lens + asserts: + - contains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + pathType: Prefix + backend: + service: + name: existing-lens + port: + number: 4318 + + - it: leaves /lens-ingest off the ingress without a Lens service name + template: ingress.yaml + set: + ingress.enabled: true + ingress.host: litellm.example.com + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - exists: + path: spec.rules[0].http.paths + - notContains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + any: true diff --git a/helm/litellm/tests/lens_endpoint_defaults_tests.yaml b/helm/litellm/tests/lens_endpoint_defaults_tests.yaml deleted file mode 100644 index 2d98dcf471e..00000000000 --- a/helm/litellm/tests/lens_endpoint_defaults_tests.yaml +++ /dev/null @@ -1,86 +0,0 @@ -suite: Lens endpoint defaults -templates: - - gateway/deployment.yaml - - gateway/configmap.yaml -set: - lensWorker.enabled: true - ingress.enabled: true - ingress.host: gateway.example - ingress.tls: - - hosts: [gateway.example] - secretName: tls -tests: - - it: derives the trace endpoint from the deployment hostname and TLS - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - it: preserves an explicitly configured public address - set: - lensWorker.publicUrl: https://custom.example/prefix - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://custom.example/prefix - - it: uses the dedicated Lens ingress when configured - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: - - hosts: [traces.example] - secretName: traces-tls - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTPS for a dedicated ALB ingress with certificate annotations - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.className: alb - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' - alb.ingress.kubernetes.io/ssl-redirect: "443" - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTP for a dedicated ingress without its own TLS - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: {} - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://traces.example - - it: uses HTTP when ingress has no TLS - set: - ingress.tls: [] - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://gateway.example/lens-ingest -values: - - ./values/required.yaml diff --git a/helm/litellm/tests/lens_modes_tests.yaml b/helm/litellm/tests/lens_modes_tests.yaml deleted file mode 100644 index 3166a88e094..00000000000 --- a/helm/litellm/tests/lens_modes_tests.yaml +++ /dev/null @@ -1,208 +0,0 @@ ---- -suite: Lens deployment modes -templates: -- gateway/configmap.yaml -- backend/deployment.yaml -- lens/deployment.yaml -- lens/service.yaml -- lens/clickhouse.yaml -- lens/secrets.yaml -- ingress.yaml -values: -- "./values/required.yaml" -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: retains the existing deployment before transferring its release ownership - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.retainResources: true - asserts: - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep -- it: keeps the ingestion endpoint on an independently deployed Lens service - template: ingress.yaml - set: - ingress.enabled: true - lensWorker.mode: external - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: existing-lens - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: {number: 4318} -- it: connects an external deployment without installing Lens resources - templates: - - lens/deployment.yaml - - lens/service.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - hasDocuments: - count: 0 -- it: sends delegated requests to the explicitly selected external Lens - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens.other-namespace:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: external-lens-signing - key: gateway-secret -- it: disables Lens explicitly even with a retained legacy enable flag - template: lens/deployment.yaml - set: - lensWorker.mode: disabled - lensWorker.enabled: true - asserts: - - hasDocuments: - count: 0 -- it: bundles Lens through explicit mode selection - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_MODE - value: standalone -- it: uses separate credentials for external identity and service access - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.serviceTokenSecret: {name: worker-service, key: token} - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: {name: worker-service, key: token} -- it: supplies the same separate signing secret to the bundled service - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} -- it: routes explicit bundled mode with the enable flag off - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: bundled - lensWorker.enabled: false - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: gateway-lens-worker - port: {number: 4318} -- it: routes external mode to its selected service - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "existing-lens" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: {number: 4318} -- it: omits ingestion when an external service has no ingress backend - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true -- it: omits ingestion for disabled mode with the enable flag on - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: disabled - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true diff --git a/helm/litellm/tests/lens_modes_validation_tests.yaml b/helm/litellm/tests/lens_modes_validation_tests.yaml deleted file mode 100644 index 6380727893d..00000000000 --- a/helm/litellm/tests/lens_modes_validation_tests.yaml +++ /dev/null @@ -1,54 +0,0 @@ ---- -suite: Lens deployment mode validation -templates: -- gateway/configmap.yaml -- backend/deployment.yaml -values: -- "./values/required.yaml" -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: requires a separate identity signing reference for external Lens - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: worker-service - asserts: - - failedTemplate: - errorMessage: lensWorker.gateway.secretName is required for external Lens -- it: requires an external service address - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - failedTemplate: - errorMessage: lensWorker.externalUrl is required for external Lens -- it: requires the shared server credential for an external service - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - asserts: - - failedTemplate: - errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens -- it: requires an agent reachable URL for external ingestion - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - lensWorker.publicUrl: '' - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required for external Lens -- it: rejects unsupported deployment modes - template: backend/deployment.yaml - set: - lensWorker.mode: typo - asserts: - - failedTemplate: - errorMessage: lensWorker.mode must be bundled, external or disabled diff --git a/helm/litellm/tests/lens_saved_secrets_tests.yaml b/helm/litellm/tests/lens_saved_secrets_tests.yaml deleted file mode 100644 index a6e190ec15b..00000000000 --- a/helm/litellm/tests/lens_saved_secrets_tests.yaml +++ /dev/null @@ -1,61 +0,0 @@ -suite: Lens preserves credentials across Helm upgrades -templates: - - lens/secrets.yaml -set: - fullnameOverride: lens-test - lensWorker.enabled: true -release: - namespace: lens - name: lens-test - upgrade: true -kubernetesProvider: - scheme: - v1/Secret: - gvr: - version: v1 - resource: secrets - namespaced: true - objects: - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-gateway - namespace: lens - data: - gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk= - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-service - namespace: lens - data: - service-token: c2F2ZWQtc2VydmljZS10b2tlbg== - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-clickhouse - namespace: lens - data: - password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= -tests: - - it: preserves the separate signing credential across upgrades - documentSelector: {path: metadata.name, value: lens-test-lens-gateway} - asserts: - - equal: - path: data.gateway-secret - value: c2F2ZWQtc2lnbmluZy1rZXk= - - notExists: {path: data.service-token} - - it: reuses the service credential instead of breaking running services - documentIndex: 0 - asserts: - - equal: - path: data.service-token - value: c2F2ZWQtc2VydmljZS10b2tlbg== - - it: reuses the database password instead of locking out stored traces - documentIndex: 1 - asserts: - - equal: - path: data.password - value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= -values: - - ./values/required.yaml diff --git a/helm/litellm/tests/lens_service_tests.yaml b/helm/litellm/tests/lens_service_tests.yaml deleted file mode 100644 index 544549e364d..00000000000 --- a/helm/litellm/tests/lens_service_tests.yaml +++ /dev/null @@ -1,196 +0,0 @@ -suite: Lens service isolation and ingestion routing -templates: -- gateway/configmap.yaml -- gateway/deployment.yaml -- backend/deployment.yaml -- ingress.yaml -- lens/ingress.yaml -- lens/service.yaml -- lens/deployment.yaml -tests: -- it: connects gateway/deployment.yaml to the shared Lens service - template: gateway/deployment.yaml - set: &id001 - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens-test-lens-worker:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-test-lens-gateway - key: gateway-secret -- it: connects backend/deployment.yaml to the shared Lens service - template: backend/deployment.yaml - set: *id001 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens-test-lens-worker:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-test-lens-gateway - key: gateway-secret -- it: routes uploads directly to Lens instead of the gateway - template: ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - ingress.enabled: true - ingress.host: gateway.example - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - number: 4318 -- it: keeps internal routes out of a dedicated ingestion hostname - template: lens/ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - asserts: - - equal: - path: spec.rules[0].http.paths - value: - - path: /v1/ - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - name: otlp -- it: maps the Lens service to the ingestion listener - template: lens/service.yaml - set: *id001 - asserts: - - equal: - path: spec.selector - value: - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: lens-worker - - equal: - path: spec.ports - value: - - name: otlp - port: 4318 - targetPort: otlp -- it: gives only Lens the ClickHouse secret - template: lens/deployment.yaml - set: *id001 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_URL - valueFrom: - secretKeyRef: - name: lens-storage - key: url -- it: requires an agent reachable ingestion URL - template: gateway/deployment.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: '' - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname -- it: omits Lens connection settings when disabled in gateway/deployment.yaml - template: gateway/deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - any: true -- it: omits Lens connection settings when disabled in backend/deployment.yaml - template: backend/deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - any: true -- it: preserves an existing ClickHouse database and retention - template: lens/deployment.yaml - set: - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.clickhouseDatabase: existing_traces - lensWorker.retentionDays: 45 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_DATABASE - value: existing_traces - - contains: - path: spec.template.spec.containers[0].env - content: - name: AGENT_TRACING_RETENTION_DAYS - value: '45' -- it: keeps Lens pods outside the gateway autoscaling selector - template: lens/deployment.yaml - set: &id002 - nameOverride: inference - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/name"] - value: inference-lens-worker -- it: preserves the existing gateway deployment selector - template: gateway/deployment.yaml - set: *id002 - asserts: - - equal: - path: spec.selector.matchLabels["app.kubernetes.io/name"] - value: inference -values: -- ./values/required.yaml diff --git a/helm/litellm/tests/lens_setup_tests.yaml b/helm/litellm/tests/lens_setup_tests.yaml deleted file mode 100644 index d61c7cad55b..00000000000 --- a/helm/litellm/tests/lens_setup_tests.yaml +++ /dev/null @@ -1,164 +0,0 @@ -suite: Lens managed setup -set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example -release: - namespace: lens - name: lens-test -templates: - - lens/deployment.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml -tests: - - it: generates private login, gateway and storage credentials - template: lens/secrets.yaml - asserts: - - hasDocuments: - count: 4 - - matchRegex: - path: data.service-token - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 0 - - matchRegex: - path: data.password - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 1 - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - - it: connects Lens to its private bundled storage - template: lens/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: lens-test-lens-clickhouse - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: stores traces on a persistent volume with the chosen storage class - template: lens/clickhouse.yaml - documentIndex: 1 - set: - lensWorker.clickhouse.storage: 40Gi - lensWorker.clickhouse.storageClassName: fast - asserts: - - equal: - path: spec.volumeClaimTemplates[0].spec - value: - accessModes: [ReadWriteOnce] - storageClassName: fast - resources: - requests: - storage: 40Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: keeps an existing external database instead of creating a new one - template: lens/clickhouse.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - asserts: - - hasDocuments: - count: 0 - - it: leaves supplied secrets under their existing manager - template: lens/secrets.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - lensWorker.serviceTokenSecret.name: external-service - lensWorker.adminTokenSecret.name: external-admin - lensWorker.gateway.secretName: external-signing - asserts: - - hasDocuments: - count: 0 - - it: creates no credentials or database when Lens is disabled - templates: - - lens/secrets.yaml - - lens/clickhouse.yaml - set: - lensWorker.enabled: false - asserts: - - hasDocuments: - count: 0 - - it: requires an external database when bundled storage is explicitly disabled - template: lens/deployment.yaml - set: - lensWorker.clickhouse.enabled: false - asserts: - - failedTemplate: - errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled - - it: keeps storage names valid and uses the same name for the connection - set: - fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - asserts: - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 0 - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 0 - - equal: - path: spec.serviceName - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/secrets.yaml - documentIndex: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/deployment.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - key: password - template: lens/deployment.yaml - - it: allows cold storage startup and inherits registry credentials - template: lens/clickhouse.yaml - documentIndex: 1 - set: - imagePullSecrets: [{name: registry-auth}] - asserts: - - equal: - path: spec.template.spec.imagePullSecrets - value: [{name: registry-auth}] - - equal: - path: spec.template.spec.containers[0].startupProbe - value: - httpGet: {path: /ping, port: http} - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 60 - -values: - - ./values/required.yaml diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml deleted file mode 100644 index 614db9d6b2b..00000000000 --- a/helm/litellm/tests/lens_worker_tests.yaml +++ /dev/null @@ -1,79 +0,0 @@ -suite: Independent Lens release and credentials -templates: - - lens/deployment.yaml -values: - - ./values/required.yaml -set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage -tests: - - it: selects a Lens release independently of the gateway - chart: - appVersion: 9.8.7 - set: - backend.image.tag: gateway-9.8.7 - lensWorker.image.tag: 2.3.4 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/lens:2.3.4 - - it: retains the pinned Lens chart default when only the gateway changes - chart: - appVersion: 9.8.7 - set: - backend.image.tag: gateway-9.8.7 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/lens:0.1.0-dev.0 - - it: selects development publications explicitly - set: - lensWorker.image.repository: ghcr.io/berriai/lens-dev - lensWorker.image.tag: sha-0123456789abcdef - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef - - it: preserves a verified registry mirror and digest - set: - lensWorker.image.repository: registry.example/lens - lensWorker.image.tag: replaced-label - lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - - it: rejects a malformed digest - set: - lensWorker.image.digest: sha256:invalid - asserts: - - failedTemplate: - errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters - - it: keeps the legacy enablement switch working - set: - lensWorker.enabled: false - asserts: - - hasDocuments: {count: 0} - - it: keeps isolated local analysis with delegated gateway access - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: {name: LENS_MODE, value: standalone} - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret} - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - equal: - path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem - value: true - - equal: - path: spec.template.spec.volumes[0].emptyDir - value: {medium: Memory, sizeLimit: 1Gi} diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index aa6ee7a6056..92a48153c5a 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -654,70 +654,15 @@ ui: topologySpreadConstraints: [] lens: - library: true - -lensWorker: - retainResources: false - mode: "" - externalUrl: "" - externalServiceName: "" - gateway: - secretName: "" - secretKey: gateway-secret - standaloneUrl: "" - adminTokenSecret: - name: "" - key: admin-token - extraEnv: [] - extraEnvFrom: [] enabled: false - replicaCount: 1 - image: - repository: ghcr.io/berriai/lens - tag: "0.1.0-dev.0" - digest: "" - pullPolicy: IfNotPresent - tokenSecret: + url: "" + publicUrl: "" + service: name: "" - key: token + port: 4318 + gatewaySecret: + name: "" + key: gateway-secret serviceTokenSecret: name: "" key: service-token - clickhouse: - enabled: true - image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e - storage: 20Gi - storageClassName: null - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 2Gi - clickhouseDatabase: litellm - retentionDays: 14 - clickhouseSecret: - name: "" - key: url - publicUrl: "" - service: - port: 4318 - annotations: {} - ingress: - enabled: false - path: /v1/ - className: "" - host: "" - annotations: {} - tls: [] - url: "" - tmpSizeLimit: 1Gi - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 2Gi - nodeSelector: {} - tolerations: [] - affinity: {} diff --git a/tests/e2e/migrations/lens_helm_smoke.sh b/tests/e2e/migrations/lens_helm_smoke.sh deleted file mode 100644 index 19b55be75e1..00000000000 --- a/tests/e2e/migrations/lens_helm_smoke.sh +++ /dev/null @@ -1,477 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -qualification_mode=${LENS_QUALIFICATION_MODE:-smoke} -chart_selection=${LENS_HELM_CHART:-all} -case "$qualification_mode" in smoke|boundaries|release) ;; *) printf 'LENS_QUALIFICATION_MODE must be smoke, boundaries or release\n' >&2; exit 2 ;; esac -case "$chart_selection" in all|litellm-helm|litellm) ;; *) printf 'LENS_HELM_CHART must be all, litellm-helm or litellm\n' >&2; exit 2 ;; esac -if [[ "$qualification_mode" == release ]]; then - : "${LENS_RELEASE_PROVIDER_API_KEY:?A separately authorized provider credential is required}" - : "${LENS_RELEASE_PROVIDER_API_BASE:?Set the authorized provider API base}" - : "${LENS_RELEASE_PROVIDER_MODEL:?Set the provider model explicitly}" - [[ "$LENS_RELEASE_PROVIDER_API_BASE" == https://* ]] -fi -qa_dir=$(mktemp -d) -cluster="lens-install-$(openssl rand -hex 6)" -cluster_owned=false -forward_pids=() -cleanup() { - local status=$? - if declare -F qualification_finish > /dev/null; then qualification_finish "$status" || true; fi - if (( status != 0 )); then - for log in "$qa_dir"/*-forward.log; do - if [[ -f "$log" ]]; then cat "$log" >&2; fi - done - if [[ -n "${namespace:-}" ]]; then diagnose || true; fi - fi - for pid in "${forward_pids[@]}"; do kill "$pid" 2>/dev/null || true; done - if [[ "$cluster_owned" == true ]]; then kind delete cluster --name "$cluster" || true; fi - rm -rf "$qa_dir" - return "$status" -} -trap cleanup EXIT -umask 077 -export KUBECONFIG="$qa_dir/kubeconfig" -for component in gateway backend monolith; do - baseline_id=$(docker image inspect "lens-ci-$component:v0.0.0-lens-ci-baseline" --format '{{.Id}}') - current_id=$(docker image inspect "lens-ci-$component:v0.0.0-lens-ci" --format '{{.Id}}') - test "$baseline_id" != "$current_id" -done -cluster_owned=true -kind create cluster --name "$cluster" \ - --image kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f \ - --wait 120s -for component in gateway backend ui migrations monolith; do - kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci" -done -for component in gateway backend monolith; do - kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci-baseline" -done -test "$(docker image inspect lens-ci-worker:baseline --format '{{.Id}}')" != \ - "$(docker image inspect lens-ci-worker:upgrade --format '{{.Id}}')" -kind load docker-image --name "$cluster" lens-ci-worker:baseline lens-ci-worker:upgrade -test -f helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz - -api() { - curl --fail-with-body --silent --show-error --max-time 20 \ - -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \ - "http://127.0.0.1:14418$1" "${@:2}" -} - -saved_trace() { - for attempt in $(seq 1 30); do - if api "/v1/traces/$trace_id" > "$qa_dir/saved.json" && \ - jq -e --arg span "$span_id" 'any(.spans[]; .span_id == $span)' "$qa_dir/saved.json" > /dev/null; then - return 0 - fi - sleep 2 - done - return 1 -} - -diagnose() { - kubectl -n "$namespace" get pods - kubectl -n "$namespace" get services,endpoints - kubectl -n "$namespace" get events --sort-by=.lastTimestamp | tail -30 - if [[ "$qualification_mode" == smoke ]]; then - kubectl -n "$namespace" logs --all-containers -l app.kubernetes.io/instance=lens --tail=50 || true - fi - return 1 -} - -forward() { - local service=$1 local_port=$2 remote_port=$3 - local log="$qa_dir/$service-forward.log" - kubectl -n "$namespace" port-forward --address 127.0.0.1 --pod-running-timeout=30s \ - "service/$service" "$local_port:$remote_port" > "$log" 2>&1 & - local pid=$! - forward_pids+=("$pid") - for attempt in $(seq 1 150); do - if ! kill -0 "$pid" 2>/dev/null; then - cat "$log" >&2 - return 1 - fi - if grep -q "^Forwarding from 127\\.0\\.0\\.1:$local_port ->" "$log"; then return 0; fi - sleep 0.2 - done - cat "$log" >&2 - return 1 -} - -stop_forwards() { - for pid in "${forward_pids[@]}"; do - kill "$pid" 2>/dev/null || true - wait "$pid" 2>/dev/null || true - done - forward_pids=() -} - -forward_bundled() { - stop_forwards - qualification_forward_control - forward lens-lens-worker 14419 4318 -} - -deployment_pods() { - local deployment selector deployment_uid replica_sets attempt - local pods="$qa_dir/$1-pods.json" snapshot="$qa_dir/$1-snapshot.json" - printf '%s: wait for deployment/%s before recording pod identities\n' "$namespace" "$1" >&2 - kubectl -n "$namespace" rollout status "deployment/$1" --timeout=180s >&2 || return - deployment=$(kubectl -n "$namespace" get deployment "$1" -o json) || return - selector=$(jq -r '.spec.selector.matchLabels | to_entries | map("\(.key)=\(.value)") | join(",")' <<< "$deployment") - deployment_uid=$(jq -er .metadata.uid <<< "$deployment") - test -n "$selector" - replica_sets=$(kubectl -n "$namespace" get replicasets -l "$selector" -o json \ - | jq -ce --arg uid "$deployment_uid" '[.items[] - | select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "Deployment" and .uid == $uid)) - | .metadata.uid] | if length > 0 then . else error("Deployment ReplicaSet identity is missing") end') || return - for attempt in $(seq 1 30); do - kubectl -n "$namespace" get pods -l "$selector" -o json > "$pods" || return - if jq -S -e --argjson replica_sets "$replica_sets" '[.items[] - | select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "ReplicaSet" - and (.uid as $owner | $replica_sets | index($owner) != null))) - | select(.metadata.deletionTimestamp == null) - | {uid:.metadata.uid, images:.spec.containers | map({name,image}), - containers:(.status.containerStatuses // []) | map({name,imageID,restartCount,ready})}] | sort_by(.uid) - | if length > 0 and all(.[]; .containers | length > 0) - and all(.[].containers[]; .imageID != null and .imageID != "" and .ready) - then . else error("Ready pod runtime identity is missing") end' \ - "$pods" > "$snapshot" 2> "$qa_dir/$1-snapshot-error.log"; then - cat "$snapshot" - return 0 - fi - sleep 1 - done - printf '%s: deployment/%s pod identities did not converge\n' "$namespace" "$1" >&2 - jq '[.items[] | {name:.metadata.name, uid:.metadata.uid, deleting:.metadata.deletionTimestamp, - phase:.status.phase, containers:[.status.containerStatuses[]? | {name,imageID,restartCount,ready}]}]' \ - "$pods" >&2 - return 1 -} - -gateway_pods() { - local component=$1 tag=$2 deployment="lens-$1" - if [[ "$component" == monolith ]]; then deployment=lens; fi - deployment_pods "$deployment" \ - | jq -S -e --arg expected "lens-ci-$component:$tag" ' - if all(.[]; any(.images[]; .image == $expected)) then - [.[] as $pod | $pod.images[] | select(.image == $expected) | . as $requested - | $pod.containers[] | select(.name == $requested.name) - | {uid:$pod.uid, image:$requested.image, name, imageID, restartCount, ready}] - else error("Gateway pod does not use the expected image") end' -} - -migration_job() { - kubectl -n "$namespace" get job lens-migrations -o json \ - | jq -e --arg image "$1" ' - select(.metadata.annotations["helm.sh/hook"] == "pre-install,pre-upgrade") - | select(.metadata.annotations["helm.sh/hook-delete-policy"] == "before-hook-creation") - | select(.metadata.annotations["argocd.argoproj.io/hook"] == null) - | select(.status.succeeded == 1 and .spec.template.spec.containers[0].image == $image) - | .metadata.uid | select(type == "string" and length > 0)' -} - -source tests/e2e/migrations/lens_release_qualification.sh -qualification_initialize -if [[ "$qualification_mode" != smoke && "$chart_selection" != litellm ]]; then - qualification_bundled_postgres -fi -for chart in litellm-helm litellm; do - if [[ "$chart_selection" != all && "$chart_selection" != "$chart" ]]; then continue; fi - namespace="lens-$chart" - kubectl create namespace "$namespace" - master_key="sk-$(openssl rand -hex 24)" - kubectl -n "$namespace" create secret generic lens-secrets \ - --from-literal="master-key=$master_key" \ - --from-literal="service-token=$(openssl rand -hex 32)" \ - --from-literal="gateway-secret=$(openssl rand -hex 32)" \ - --from-literal="url=http://clickhouse:8123" \ - --from-literal=username=litellm --from-literal=password=isolated-helm-test - kubectl -n "$namespace" apply -f - <<'YAML' -apiVersion: apps/v1 -kind: Deployment -metadata: {name: postgres} -spec: - selector: {matchLabels: {app: postgres}} - template: - metadata: {labels: {app: postgres}} - spec: - containers: - - name: postgres - image: postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea - env: - - {name: POSTGRES_DB, value: litellm} - - {name: POSTGRES_USER, value: litellm} - - {name: POSTGRES_PASSWORD, value: isolated-helm-test} - readinessProbe: - exec: {command: [pg_isready, -U, litellm, -d, litellm]} ---- -apiVersion: v1 -kind: Service -metadata: {name: postgres} -spec: - selector: {app: postgres} - ports: [{port: 5432}] ---- -apiVersion: apps/v1 -kind: Deployment -metadata: {name: clickhouse} -spec: - selector: {matchLabels: {app: clickhouse}} - template: - metadata: {labels: {app: clickhouse}} - spec: - containers: - - name: clickhouse - image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e - env: [{name: CLICKHOUSE_SKIP_USER_SETUP, value: "1"}] - volumeMounts: - - {name: keeper, mountPath: /etc/clickhouse-server/config.d/lens-keeper.xml, subPath: lens-keeper.xml} - readinessProbe: - httpGet: {path: /ping, port: 8123} - volumes: - - name: keeper - configMap: {name: clickhouse-keeper} ---- -apiVersion: v1 -kind: ConfigMap -metadata: {name: clickhouse-keeper} -data: - lens-keeper.xml: | - - - 91811 - /var/lib/clickhouse/coordination/log - /var/lib/clickhouse/coordination/snapshots - 10000true - 1127.0.0.19234 - - 127.0.0.19181 - /lens/keeper-map - ---- -apiVersion: v1 -kind: Service -metadata: {name: clickhouse} -spec: - selector: {app: clickhouse} - ports: [{port: 8123}] -YAML - kubectl -n "$namespace" rollout status deployment/postgres --timeout=180s - kubectl -n "$namespace" rollout status deployment/clickhouse --timeout=180s - cat > "$qa_dir/common.yaml" <<'YAML' -fullnameOverride: lens -migrationJob: - ttlSecondsAfterFinished: 3600 - hooks: - helm: {enabled: true} - argocd: {enabled: false} -lensWorker: - enabled: true - image: {repository: lens-ci-worker, tag: baseline, pullPolicy: Never} - serviceTokenSecret: {name: lens-secrets, key: service-token} - gateway: {secretName: lens-secrets, secretKey: gateway-secret} - clickhouseSecret: {name: lens-secrets, key: url} - clickhouseDatabase: existing_traces - retentionDays: 45 - publicUrl: http://127.0.0.1:14419 -YAML - if [[ "$chart" == litellm-helm ]]; then - control=lens - control_port=4000 - cat > "$qa_dir/chart.yaml" <<'YAML' -image: {repository: lens-ci-monolith, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never} -masterkeySecretName: lens-secrets -masterkeySecretKey: master-key -envVars: {STORE_MODEL_IN_DB: "True"} -db: - deployStandalone: false - useExisting: true - endpoint: postgres - secret: {name: lens-secrets, usernameKey: username, passwordKey: password} -redis: {enabled: false} -proxy_config: - model_list: [] - general_settings: - master_key: os.environ/PROXY_MASTER_KEY - store_model_in_db: true - tracing: {enabled: true, store: {type: lens}} -YAML - else - control=lens-backend - control_port=4001 - cat > "$qa_dir/chart.yaml" <<'YAML' -masterKey: {secretName: lens-secrets, secretKey: master-key} -database: - writer: - host: postgres - dbname: litellm - passwordSecret: {name: lens-secrets, usernameKey: username, passwordKey: password} -migrationJob: - image: {repository: lens-ci-migrations, tag: v0.0.0-lens-ci, pullPolicy: Never} -gateway: - image: {repository: lens-ci-gateway, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never} - numWorkers: 1 - extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}] - hpa: {enabled: false} - resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}} - config: - create: true - proxy_config: - model_list: [] - general_settings: - store_model_in_db: true - tracing: {enabled: true, store: {type: lens}} -backend: - extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}] - image: {repository: lens-ci-backend, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never} - hpa: {enabled: false} - resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}} -ui: - image: {repository: lens-ci-ui, tag: v0.0.0-lens-ci, pullPolicy: Never} - hpa: {enabled: false} -YAML - fi - install=(helm upgrade --install lens "helm/$chart" -n "$namespace" \ - -f "$qa_dir/common.yaml" -f "$qa_dir/chart.yaml" --wait --wait-for-jobs --timeout 8m) - qualification_provider_values - "${install[@]}" || diagnose - forward_bundled - for attempt in $(seq 1 30); do - if api /lens/service > "$qa_dir/status.json" && jq -e '.connected and .status.storage_ready' "$qa_dir/status.json"; then break; fi - sleep 1 - done - jq -e '.connected and .status.storage_ready' "$qa_dir/status.json" - qualification_setup - api /lens/tracing/keys -d '{"name":"Helm smoke"}' > "$qa_dir/key.json" - tracing_key=$(jq -r .key "$qa_dir/key.json") - trace_id=$(openssl rand -hex 16) - span_id=$(openssl rand -hex 8) - jq -n --arg trace "$trace_id" --arg span "$span_id" --arg at "$(date +%s)000000000" \ - '{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Helm trace",kind:1, - startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/trace.json" - curl --fail-with-body --silent --show-error --retry 10 --retry-all-errors --retry-delay 1 \ - -H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \ - -d "@$qa_dir/trace.json" http://127.0.0.1:14419/v1/traces - saved_trace - qualification_read_boundaries baseline - kubectl -n "$namespace" exec deployment/clickhouse -- clickhouse-client --query \ - "SELECT count() FROM existing_traces.otel_traces WHERE TraceId = '$trace_id'" | grep -qx 1 - "${install[@]}" || diagnose - forward_bundled - saved_trace - qualification_read_boundaries reapplied - kubectl -n "$namespace" get deployments -l app.kubernetes.io/instance=lens -o json \ - | jq -S '[.items[] | select(.metadata.name != "lens-lens-worker") | {name:.metadata.name,template:.spec.template}] | sort_by(.name)' \ - > "$qa_dir/gateway-before.json" - "${install[@]}" --set lensWorker.image.tag=upgrade || diagnose - kubectl -n "$namespace" get deployments -l app.kubernetes.io/instance=lens -o json \ - | jq -S '[.items[] | select(.metadata.name != "lens-lens-worker") | {name:.metadata.name,template:.spec.template}] | sort_by(.name)' \ - > "$qa_dir/gateway-after.json" - cmp "$qa_dir/gateway-before.json" "$qa_dir/gateway-after.json" - forward_bundled - saved_trace - qualification_read_boundaries lens-upgrade - "${install[@]}" || diagnose - forward_bundled - saved_trace - qualification_read_boundaries lens-rollback - kubectl -n "$namespace" get deployment lens-lens-worker -o json \ - | jq -S .spec.template > "$qa_dir/lens-before.json" - deployment_pods lens-lens-worker > "$qa_dir/lens-pods-before.json" - gateway_components=(monolith) - gateway_upgrade=(--set image.tag=v0.0.0-lens-ci) - migration_baseline=lens-ci-monolith:v0.0.0-lens-ci-baseline - migration_current=lens-ci-monolith:v0.0.0-lens-ci - if [[ "$chart" == litellm ]]; then - gateway_components=(gateway backend) - gateway_upgrade=(--set gateway.image.tag=v0.0.0-lens-ci --set backend.image.tag=v0.0.0-lens-ci) - migration_baseline=lens-ci-migrations:v0.0.0-lens-ci - migration_current=$migration_baseline - fi - migration_before=$(migration_job "$migration_baseline") - for component in "${gateway_components[@]}"; do - gateway_pods "$component" v0.0.0-lens-ci-baseline > "$qa_dir/$component-before.json" - done - install+=("${gateway_upgrade[@]}") - "${install[@]}" || diagnose - migration_after=$(migration_job "$migration_current") - test "$migration_before" != "$migration_after" - kubectl -n "$namespace" get deployment lens-lens-worker -o json \ - | jq -S .spec.template > "$qa_dir/lens-after.json" - cmp "$qa_dir/lens-before.json" "$qa_dir/lens-after.json" - deployment_pods lens-lens-worker > "$qa_dir/lens-pods-after.json" - cmp "$qa_dir/lens-pods-before.json" "$qa_dir/lens-pods-after.json" - for component in "${gateway_components[@]}"; do - gateway_pods "$component" v0.0.0-lens-ci > "$qa_dir/$component-after.json" - before_id=$(jq -ce '[.[].imageID] | unique | if length == 1 then .[0] else error("Mixed gateway images") end' \ - "$qa_dir/$component-before.json") - after_id=$(jq -ce '[.[].imageID] | unique | if length == 1 then .[0] else error("Mixed gateway images") end' \ - "$qa_dir/$component-after.json") - test "$before_id" != "$after_id" - done - forward_bundled - saved_trace - qualification_read_boundaries gateway-upgrade - if [[ "$qualification_mode" != smoke ]]; then - install+=(--set lensWorker.image.tag=upgrade) - "${install[@]}" || diagnose - forward_bundled - qualification_read_boundaries both-candidates - fi - stop_forwards - kubectl -n "$namespace" rollout restart "deployment/$control" deployment/lens-lens-worker - kubectl -n "$namespace" rollout status "deployment/$control" --timeout=180s - kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s - qualification_forward_control - saved_trace - qualification_outage - printf '%s: fresh install, ingestion, Lens upgrade and rollback, independent gateway image upgrade, and restart passed\n' "$chart" - stop_forwards - helm upgrade --install external-lens helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz \ - -n "$namespace" --wait --timeout 5m \ - --set fullnameOverride=external-lens \ - --set image.repository=lens-ci-worker --set image.tag=upgrade --set image.pullPolicy=Never \ - --set adminTokenSecret.name=lens-secrets --set adminTokenSecret.key=master-key \ - --set gateway.enabled=true --set gateway.secretName=lens-secrets \ - --set gateway.secretKey=gateway-secret --set serviceTokenSecret.name=lens-secrets \ - --set clickhouseSecret.name=lens-secrets --set clickhouseDatabase=existing_traces \ - --set publicUrl=http://127.0.0.1:14419 - "${install[@]}" --set lensWorker.mode=external \ - --set lensWorker.externalUrl=http://external-lens:4318 || diagnose - test -z "$(kubectl -n "$namespace" get deployment lens-lens-worker --ignore-not-found -o name)" - qualification_forward_control - forward external-lens 14419 4318 - api /lens/service | jq -e '.configured and .connected and .status.storage_ready' - saved_trace - qualification_read_boundaries external-lens - trace_id=$(openssl rand -hex 16) - span_id=$(openssl rand -hex 8) - jq --arg trace "$trace_id" --arg span "$span_id" \ - '.resourceSpans[0].scopeSpans[0].spans[0] |= (.traceId=$trace | .spanId=$span)' \ - "$qa_dir/trace.json" > "$qa_dir/external-trace.json" - curl --fail-with-body --silent --show-error --max-time 20 \ - -H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \ - -d "@$qa_dir/external-trace.json" http://127.0.0.1:14419/v1/traces - saved_trace - kubectl -n "$namespace" get deployment external-lens -o json \ - | jq -S '{uid:.metadata.uid,template:.spec.template}' > "$qa_dir/external-before.json" - stop_forwards - tracing_setting=proxy_config.general_settings.tracing.enabled - if [[ "$chart" == litellm ]]; then - tracing_setting=gateway.config.proxy_config.general_settings.tracing.enabled - fi - "${install[@]}" --set lensWorker.mode=disabled --set "$tracing_setting=false" || diagnose - test -z "$(kubectl -n "$namespace" get deployment lens-lens-worker --ignore-not-found -o name)" - qualification_forward_control - api /lens/service | jq -e '.configured == false and .connected == false' - test "$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 20 \ - -H "Authorization: Bearer $master_key" http://127.0.0.1:14418/lens/datasets)" = 503 - kubectl -n "$namespace" get deployment external-lens -o json \ - | jq -S '{uid:.metadata.uid,template:.spec.template}' > "$qa_dir/external-after.json" - cmp "$qa_dir/external-before.json" "$qa_dir/external-after.json" - printf '%s: external Lens trace reads/writes and disabled gateway behavior passed\n' "$chart" - qualification_record chart-lifecycle passed - stop_forwards - kubectl delete namespace "$namespace" --wait=true -done diff --git a/tests/e2e/migrations/lens_release_qualification.sh b/tests/e2e/migrations/lens_release_qualification.sh deleted file mode 100644 index 86c6055f33d..00000000000 --- a/tests/e2e/migrations/lens_release_qualification.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash - -qualification_initialize() { - qualification_events="$qa_dir/qualification-events.jsonl" - : > "$qualification_events" - qualification_started=$(date -u +%FT%TZ) - qualification_output=${LENS_QUALIFICATION_RESULTS_DIR:-} - if [[ -n "$qualification_output" ]]; then - mkdir -p "$qualification_output" - test ! -e "$qualification_output/host-$chart_selection-$qualification_mode.json" - fi -} - -qualification_record() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - jq -nc --arg chart "${chart:-bootstrap}" --arg case "$1" --arg result "$2" \ - --arg at "$(date -u +%FT%TZ)" '{chart:$chart,case:$case,result:$result,at:$at}' >> "$qualification_events" -} - -qualification_finish() { - if [[ "$qualification_mode" == smoke || -z "$qualification_output" ]]; then return; fi - jq -n --slurpfile checks "$qualification_events" --arg mode "$qualification_mode" \ - --arg selection "$chart_selection" --arg started "$qualification_started" \ - --arg finished "$(date -u +%FT%TZ)" --arg source "$(git rev-parse HEAD)" --argjson exit_code "$1" \ - '{qualification:"isolated candidate host behavior",mode:$mode,chart_selection:$selection, - started_at:$started,finished_at:$finished,harness_source:$source,exit_code:$exit_code, - passed:($exit_code == 0),paid_provider_qualified:($mode == "release" and $exit_code == 0), - signed_release_qualified:false,production_deployed:false,checks:$checks}' \ - > "$qualification_output/host-$chart_selection-$qualification_mode.json" -} - -qualification_request() { - local credential=$1 method=$2 route=$3 expected=$4 label=$5 input=${6:-} - local port=14418 - if [[ "$chart" == litellm && "$route" == /v1/chat/completions ]]; then port=14420; fi - local response="$qa_dir/release-response.json" status - printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "$credential" > "$qa_dir/release-headers" - local arguments=(--silent --show-error --max-time 90 --request "$method" \ - --header "@$qa_dir/release-headers" --output "$response" --dump-header "$qa_dir/release-response.headers" \ - --write-out '%{http_code}') - if [[ -n "$input" ]]; then arguments+=(--data-binary "@$input"); fi - status=$(curl "${arguments[@]}" "http://127.0.0.1:$port$route") - if [[ "$status" != "$expected" ]]; then - qualification_record "$label" "failed-http-$status-expected-$expected" - printf '%s: expected HTTP %s, received %s\n' "$label" "$expected" "$status" >&2 - return 1 - fi - qualification_record "$label" "http-$status" -} - -qualification_forward_control() { - forward "$control" 14418 "$control_port" - if [[ "$qualification_mode" == release && "$chart" == litellm ]]; then - forward lens-gateway 14420 4000 - fi -} - -qualification_provider_values() { - if [[ "$qualification_mode" != release ]]; then return; fi - printf '%s' "$LENS_RELEASE_PROVIDER_API_KEY" > "$qa_dir/provider-key" - kubectl -n "$namespace" create secret generic lens-release-provider \ - --from-file="LENS_RELEASE_PROVIDER_API_KEY=$qa_dir/provider-key" > /dev/null - if [[ "$chart" == litellm-helm ]]; then - printf 'environmentSecrets: [lens-release-provider]\n' > "$qa_dir/provider.yaml" - else - cat > "$qa_dir/provider.yaml" <<'YAML' -gateway: - extraEnv: - - {name: STORE_MODEL_IN_DB, value: "True"} - - name: LENS_RELEASE_PROVIDER_API_KEY - valueFrom: {secretKeyRef: {name: lens-release-provider, key: LENS_RELEASE_PROVIDER_API_KEY}} -backend: - extraEnv: - - {name: STORE_MODEL_IN_DB, value: "True"} - - name: LENS_RELEASE_PROVIDER_API_KEY - valueFrom: {secretKeyRef: {name: lens-release-provider, key: LENS_RELEASE_PROVIDER_API_KEY}} -YAML - fi - install+=(-f "$qa_dir/provider.yaml") -} - -qualification_setup() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - local tenant user team credential trace span attempt status - for tenant in a b; do - user="lens-release-$tenant-$(openssl rand -hex 8)" - team="lens-release-team-$tenant-$(openssl rand -hex 8)" - jq -n --arg user "$user" '{user_id:$user,user_email:($user+"@example.test"),user_role:"internal_user",auto_create_key:false}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /user/new 200 "create-user-$tenant" "$qa_dir/request.json" - jq -e --arg user "$user" '.user_id == $user' "$qa_dir/release-response.json" > /dev/null - jq -n --arg user "$user" --arg team "$team" '{team_id:$team,team_alias:$team,members_with_roles:[{user_id:$user,role:"admin"}]}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /team/new 200 "create-team-$tenant" "$qa_dir/request.json" - jq -e --arg team "$team" '.team_id == $team' "$qa_dir/release-response.json" > /dev/null - jq -n --arg user "$user" --arg team "$team" '{user_id:$user,team_id:$team,key_alias:($team+"-isolation")}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/generate 200 "create-key-$tenant" "$qa_dir/request.json" - credential=$(jq -er '.key | select(type == "string" and length > 0)' "$qa_dir/release-response.json") - jq -n --arg user "$user" --arg team "$team" --arg key "$credential" '{user:$user,team:$team,key:$key}' > "$qa_dir/tenant-$tenant.json" - jq -n --arg team "$team" '{name:"Release tenant trace",team_id:$team}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /lens/tracing/keys 200 "create-ingestion-key-$tenant" "$qa_dir/request.json" - cp "$qa_dir/release-response.json" "$qa_dir/ingestion-$tenant.json" - jq -e --arg team "$team" '.record.tenant.team_id == $team' "$qa_dir/ingestion-$tenant.json" > /dev/null - credential=$(jq -er .key "$qa_dir/ingestion-$tenant.json") - trace=$(openssl rand -hex 16) - span=$(openssl rand -hex 8) - jq -n --arg trace "$trace" --arg span "$span" --arg at "$(date +%s)000000000" \ - '{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Lens release tenant",kind:1,startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/tenant-$tenant-trace.json" - printf '%s' "$trace" > "$qa_dir/tenant-$tenant-trace-id" - printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "$credential" > "$qa_dir/ingestion-headers" - for attempt in $(seq 1 30); do - status=$(curl --silent --show-error --max-time 20 --output "$qa_dir/ingestion-response.json" --write-out '%{http_code}' \ - --header "@$qa_dir/ingestion-headers" --data-binary "@$qa_dir/tenant-$tenant-trace.json" http://127.0.0.1:14419/v1/traces) - if [[ "$status" == 200 ]]; then break; fi - test "$status" = 401 - sleep 1 - done - test "$status" = 200 - qualification_record "ingest-team-$tenant" http-200 - for attempt in $(seq 1 30); do - if api "/v1/traces/$trace" > "$qa_dir/tenant-$tenant-stored.json" 2> /dev/null; then break; fi - sleep 1 - done - jq -e --arg trace "$trace" '.summary.trace_id == $trace' "$qa_dir/tenant-$tenant-stored.json" > /dev/null - done - jq -n --arg user "$(jq -r .user "$qa_dir/tenant-a.json")" --arg team "$(jq -r .team "$qa_dir/tenant-a.json")" \ - '{user_id:$user,team_id:$team,key_alias:"Lens release revocation"}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/generate 200 create-revocation-key "$qa_dir/request.json" - cp "$qa_dir/release-response.json" "$qa_dir/revoked-key.json" - credential=$(jq -er .key "$qa_dir/revoked-key.json") - qualification_request "$credential" GET /lens/service 200 key-valid-before-revocation - jq -n --arg key "$credential" '{keys:[$key]}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/delete 200 revoke-gateway-key "$qa_dir/request.json" - qualification_request "$credential" GET /lens/service 401 revoked-key-denied - if [[ "$qualification_mode" == release ]]; then - qualification_model="lens-release-$(openssl rand -hex 8)" - jq -n --arg model "$qualification_model" --arg provider_model "$LENS_RELEASE_PROVIDER_MODEL" \ - --arg base "$LENS_RELEASE_PROVIDER_API_BASE" \ - '{model_name:$model,litellm_params:{model:$provider_model,api_base:$base,api_key:"os.environ/LENS_RELEASE_PROVIDER_API_KEY"}}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /model/new 200 register-provider-model "$qa_dir/request.json" - jq -e '.model_info.id | type == "string" and length > 0' "$qa_dir/release-response.json" > /dev/null - qualification_paid_calls=0 - qualification_total_cost=0 - fi -} - -qualification_read_boundaries() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - local phase=$1 tenant other credential own foreign - for tenant in a b; do - other=a - if [[ "$tenant" == a ]]; then other=b; fi - credential=$(jq -r .key "$qa_dir/tenant-$tenant.json") - own=$(cat "$qa_dir/tenant-$tenant-trace-id") - foreign=$(cat "$qa_dir/tenant-$other-trace-id") - qualification_request "$credential" GET "/v1/traces/$own" 200 "$phase-$tenant-own-trace" - jq -e --arg trace "$own" '.summary.trace_id == $trace' "$qa_dir/release-response.json" > /dev/null - qualification_request "$credential" GET "/v1/traces/$foreign" 404 "$phase-$tenant-other-team-denied" - qualification_request "$credential" GET "/v1/traces/$foreign?all_teams=1" 404 "$phase-$tenant-query-cannot-expand-scope" - jq -n --arg sql "SELECT TraceId AS trace_id FROM otel_traces WHERE TraceId IN ('$own','$foreign') ORDER BY TraceId" '{sql:$sql}' > "$qa_dir/request.json" - qualification_request "$credential" POST /v1/traces/query 200 "$phase-$tenant-scoped-sql" "$qa_dir/request.json" - jq -e --arg trace "$own" '[.data[].trace_id] == [$trace]' "$qa_dir/release-response.json" > /dev/null - qualification_request "$credential" GET /lens/signals 403 "$phase-$tenant-admin-surface-denied" - done - qualification_request "$(jq -r .key "$qa_dir/revoked-key.json")" GET /lens/service 401 "$phase-revocation-retained" - qualification_request invalid-release-key GET "/v1/traces/$own" 401 "$phase-invalid-key-denied" - qualification_record "$phase-auth-and-retained-data" passed -} - -qualification_inference() { - if [[ "$qualification_mode" != release ]]; then return; fi - local phase=$1 credential response_id call_id cost attempt - test "$qualification_paid_calls" -lt 3 - credential=$(jq -r .key "$qa_dir/tenant-a.json") - jq -n --arg model "$qualification_model" --arg marker "$(openssl rand -hex 16)" \ - '{model:$model,messages:[{role:"user",content:("Reply with the single word ready. Test marker "+$marker)}],max_completion_tokens:64}' > "$qa_dir/request.json" - qualification_request "$credential" POST /v1/chat/completions 200 "$phase-real-inference" "$qa_dir/request.json" - qualification_paid_calls=$((qualification_paid_calls + 1)) - cp "$qa_dir/release-response.json" "$qa_dir/completion.json" - jq -e '.choices[0].message.content | type == "string" and length > 0' "$qa_dir/completion.json" > /dev/null - jq -e '.usage.prompt_tokens > 0 and .usage.completion_tokens > 0 and .usage.total_tokens == (.usage.prompt_tokens + .usage.completion_tokens)' "$qa_dir/completion.json" > /dev/null - response_id=$(jq -er .id "$qa_dir/completion.json") - call_id=$(awk 'tolower($1)=="x-litellm-call-id:" {gsub("\r", "", $2); print $2}' "$qa_dir/release-response.headers") - cost=$(awk 'tolower($1)=="x-litellm-response-cost:" {gsub("\r", "", $2); print $2}' "$qa_dir/release-response.headers") - [[ "$call_id" =~ ^[A-Za-z0-9_.:-]+$ ]] - jq -en --arg cost "$cost" '($cost|tonumber) > 0' > /dev/null - for attempt in $(seq 1 60); do - qualification_request "$master_key" GET "/spend/logs?request_id=$call_id" 200 "$phase-billing-read" - if jq -e --arg id "$response_id" '[.[] | select(.request_id == $id)] | length == 1' "$qa_dir/release-response.json" > /dev/null; then break; fi - sleep 2 - done - jq -e --arg id "$response_id" --arg cost "$cost" --arg team "$(jq -r .team "$qa_dir/tenant-a.json")" \ - --slurpfile completion "$qa_dir/completion.json" \ - '[.[] | select(.request_id == $id)] | length == 1 and (.[0] | .team_id == $team - and .prompt_tokens == $completion[0].usage.prompt_tokens - and .completion_tokens == $completion[0].usage.completion_tokens - and .total_tokens == $completion[0].usage.total_tokens - and ((.spend - ($cost|tonumber)) | fabs) < 0.00000001)' "$qa_dir/release-response.json" > /dev/null - qualification_total_cost=$(jq -en --arg total "$qualification_total_cost" --arg cost "$cost" '($total|tonumber)+($cost|tonumber)') - jq -en --arg total "$qualification_total_cost" '($total|tonumber) <= 0.10' > /dev/null - qualification_record "$phase-billing-matches-usage-and-team" passed - jq -nc --arg chart "$chart" --arg phase "$phase" --arg response_id "$response_id" \ - --arg call_id "$call_id" --argjson cost "$cost" '{chart:$chart,case:($phase+"-paid-receipt"),result:"passed",response_id:$response_id,request_id:$call_id,cost_usd:$cost}' >> "$qualification_events" -} - -qualification_outage() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - local replicas selector - qualification_read_boundaries before-outage - qualification_inference before-outage - deployment_pods "$control" > "$qa_dir/outage-gateway-before.json" - replicas=$(kubectl -n "$namespace" get deployment lens-lens-worker -o jsonpath='{.spec.replicas}') - test "$replicas" = 1 - selector=$(kubectl -n "$namespace" get deployment lens-lens-worker -o json | jq -r '.spec.selector.matchLabels|to_entries|map("\(.key)=\(.value)")|join(",")') - kubectl -n "$namespace" scale deployment/lens-lens-worker --replicas=0 - kubectl -n "$namespace" wait --for=delete pods -l "$selector" --timeout=90s - qualification_request "$master_key" GET /lens/service 200 outage-service-status - jq -e '.configured == true and .connected == false' "$qa_dir/release-response.json" > /dev/null - qualification_request "$master_key" GET /lens/signals 503 outage-lens-unavailable - qualification_request invalid-release-key GET /lens/signals 401 outage-invalid-key-still-denied - qualification_inference during-outage - kubectl -n "$namespace" scale deployment/lens-lens-worker --replicas="$replicas" - kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s - forward_bundled - saved_trace - qualification_read_boundaries after-outage - qualification_inference after-outage - deployment_pods "$control" > "$qa_dir/outage-gateway-after.json" - cmp "$qa_dir/outage-gateway-before.json" "$qa_dir/outage-gateway-after.json" - qualification_record lens-outage-without-gateway-restart passed -} - -qualification_bundled_postgres() { - local chart=bundled-postgresql release=lens-bootstrap - namespace=lens-bundled-postgresql - kubectl create namespace "$namespace" - master_key="sk-$(openssl rand -hex 24)" - printf '%s' "$master_key" > "$qa_dir/bootstrap-master" - kubectl -n "$namespace" create secret generic bootstrap-master --from-file="key=$qa_dir/bootstrap-master" > /dev/null - cat > "$qa_dir/bootstrap.yaml" < /dev/null - kubectl -n "$namespace" get job "$release-migrations" -o json \ - | jq -e '.status.succeeded == 1 and .metadata.annotations["helm.sh/hook"] == null and .metadata.annotations["argocd.argoproj.io/hook"] == null' > /dev/null - qualification_record bundled-postgres-migrated-before-gateway-start passed - kubectl -n "$namespace" scale deployment/"$release" --replicas=1 - kubectl -n "$namespace" rollout status deployment/"$release" --timeout=180s - forward "$release" 14418 4000 - jq -n '{key_alias:"Bundled PostgreSQL persisted key"}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/generate 200 bundled-postgres-key-create "$qa_dir/request.json" - local generated - generated=$(jq -er .key "$qa_dir/release-response.json") - qualification_request "$generated" GET /v1/models 200 bundled-postgres-key-auth - kubectl -n "$namespace" rollout restart deployment/"$release" - kubectl -n "$namespace" rollout status deployment/"$release" --timeout=180s - stop_forwards - forward "$release" 14418 4000 - qualification_request "$generated" GET /v1/models 200 bundled-postgres-key-survives-gateway-restart - jq -n --arg key "$generated" '{keys:[$key]}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/delete 200 bundled-postgres-key-delete "$qa_dir/request.json" - qualification_request "$generated" GET /v1/models 401 bundled-postgres-key-revoked - qualification_record bundled-postgres-ordinary-job-two-phase-bootstrap passed - stop_forwards - kubectl delete namespace "$namespace" --wait=true - namespace= -} diff --git a/tests/e2e_harness/migrations/test_lens_release_qualification.py b/tests/e2e_harness/migrations/test_lens_release_qualification.py deleted file mode 100644 index c0d5d272683..00000000000 --- a/tests/e2e_harness/migrations/test_lens_release_qualification.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -from pathlib import Path -from typing import Final - -import pytest - -HARNESS: Final = Path(__file__).resolve().parents[2] / "e2e/migrations/lens_release_qualification.sh" - - -@pytest.mark.parametrize("chart", ["litellm", "litellm-helm"]) -def test_inference_reaches_gateway_and_management_reaches_backend(tmp_path: Path, chart: str) -> None: - curl: Final = tmp_path / "curl" - curl.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "${@: -1}" >> "$qa_dir/requests"\nprintf 200\n') - curl.chmod(0o755) - forward: Final = tmp_path / "forward" - forward.write_text('#!/usr/bin/env bash\nprintf "%s %s %s\\n" "$@" >> "$qa_dir/forwards"\n') - forward.chmod(0o755) - result: Final = subprocess.run( - [ - "bash", - "-euo", - "pipefail", - "-c", - 'source "$HARNESS"\n' - "qualification_forward_control\n" - "qualification_initialize\n" - "qualification_request fixture POST /v1/chat/completions 200 inference\n" - "qualification_request fixture GET /spend/logs 200 billing\n" - "qualification_request fixture POST /model/new 200 management\n", - ], - env={ - **os.environ, - "PATH": f"{tmp_path}:{os.environ['PATH']}", - "HARNESS": str(HARNESS), - "qa_dir": str(tmp_path), - "chart": chart, - "qualification_mode": "release", - "control": "lens-backend" if chart == "litellm" else "lens", - "control_port": "4001" if chart == "litellm" else "4000", - "LENS_QUALIFICATION_RESULTS_DIR": "", - }, - capture_output=True, - text=True, - timeout=10, - check=False, - ) - assert result.returncode == 0, result.stderr - inference_port: Final = 14420 if chart == "litellm" else 14418 - assert (tmp_path / "requests").read_text().splitlines() == [ - f"http://127.0.0.1:{inference_port}/v1/chat/completions", - "http://127.0.0.1:14418/spend/logs", - "http://127.0.0.1:14418/model/new", - ] - expected_forwards: Final = ( - ["lens-backend 14418 4001", "lens-gateway 14420 4000"] if chart == "litellm" else ["lens 14418 4000"] - ) - assert (tmp_path / "forwards").read_text().splitlines() == expected_forwards From 23595f7fe56c7809f54ef051278a7a60a2976366 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 29 Oct 2026 19:46:57 -0700 Subject: [PATCH 17/34] fix(bedrock): backfill published regional service tier prices (#49113) * fix(bedrock): backfill published regional service tier prices * fix(bedrock): remove the eu.anthropic.claude-haiku-4-5-20251001-v1:0 row --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 460 ++++++++++++------ model_prices_and_context_window.json | 460 ++++++++++++------ .../test_get_supported_openai_params.py | 2 +- tests/unit/test_utils.py | 2 +- 4 files changed, 620 insertions(+), 304 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1eaac0fda24..1a1982fddd7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13804,13 +13804,17 @@ }, "bedrock/ap-northeast-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13818,34 +13822,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13854,7 +13864,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13872,20 +13884,24 @@ }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13896,13 +13912,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13959,13 +13975,17 @@ }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13973,34 +13993,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14009,7 +14035,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14027,20 +14055,24 @@ }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14051,22 +14083,24 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, + "input_cost_per_token_batches": 1.55e-07, "input_cost_per_token_flex": 1.55e-07, + "input_cost_per_token_priority": 5.425e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14075,7 +14109,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.24e-06, - "output_cost_per_token_flex": 6.2e-07 + "output_cost_per_token_batches": 6.2e-07, + "output_cost_per_token_flex": 6.2e-07, + "output_cost_per_token_priority": 2.17e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14086,14 +14122,14 @@ "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "supports_tool_choice": true, + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14105,13 +14141,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14120,16 +14156,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14140,13 +14176,13 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_audio_input": false, - "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "supports_vision": true }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14157,13 +14193,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14187,13 +14223,17 @@ }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14201,34 +14241,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14237,24 +14283,30 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -14337,29 +14389,35 @@ "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14368,7 +14426,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14379,13 +14439,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14410,29 +14470,35 @@ "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14441,7 +14507,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14452,13 +14520,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14483,29 +14551,35 @@ "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14514,18 +14588,24 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.86e-06, - "output_cost_per_token_flex": 9.3e-07 + "output_cost_per_token_batches": 9.3e-07, + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_batches": 1.15e-07, "input_cost_per_token_flex": 1.15e-07, + "input_cost_per_token_priority": 4.025e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, + "output_cost_per_token_batches": 5.05e-07, "output_cost_per_token_flex": 5.05e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_priority": 1.7675e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, @@ -14536,20 +14616,24 @@ }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, + "input_cost_per_token_batches": 3.9e-07, "input_cost_per_token_flex": 3.9e-07, + "input_cost_per_token_priority": 1.365e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14593,13 +14677,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14608,16 +14692,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14628,13 +14712,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14675,13 +14759,17 @@ }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14689,34 +14777,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14725,7 +14819,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14743,20 +14839,24 @@ }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14767,13 +14867,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -14908,13 +15008,17 @@ }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14922,42 +15026,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14975,47 +15087,59 @@ }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, + "input_cost_per_token_priority": 8.75e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15023,34 +15147,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -15059,7 +15189,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.2e-06, - "output_cost_per_token_flex": 6e-07 + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15077,20 +15209,24 @@ }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15101,13 +15237,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15543,13 +15679,17 @@ }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15557,42 +15697,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15610,20 +15758,24 @@ }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15634,13 +15786,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -25109,38 +25261,6 @@ "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, - "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { - "cache_creation_input_token_cost": 1.375e-06, - "cache_creation_input_token_cost_above_1hr": 2.2e-06, - "cache_read_input_token_cost": 1.1e-07, - "input_cost_per_token": 1.1e-06, - "deprecation_date": "2026-10-15", - "litellm_provider": "bedrock_converse", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096, - "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06, - "token_count_model": { - "model": "anthropic.claude-haiku-4-5-20251001-v1:0", - "litellm_provider": "bedrock" - } - }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -46798,12 +46918,16 @@ }, "qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, + "input_cost_per_token_priority": 2.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -46813,12 +46937,16 @@ }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46828,12 +46956,18 @@ }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_batches": 7.1e-07, + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46858,12 +46992,16 @@ }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46873,12 +47011,16 @@ }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46888,12 +47030,16 @@ }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -61028,11 +61174,17 @@ }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61041,15 +61193,21 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61058,7 +61216,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1eaac0fda24..1a1982fddd7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13804,13 +13804,17 @@ }, "bedrock/ap-northeast-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13818,34 +13822,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13854,7 +13864,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13872,20 +13884,24 @@ }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13896,13 +13912,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13959,13 +13975,17 @@ }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13973,34 +13993,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14009,7 +14035,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14027,20 +14055,24 @@ }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14051,22 +14083,24 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, + "input_cost_per_token_batches": 1.55e-07, "input_cost_per_token_flex": 1.55e-07, + "input_cost_per_token_priority": 5.425e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14075,7 +14109,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.24e-06, - "output_cost_per_token_flex": 6.2e-07 + "output_cost_per_token_batches": 6.2e-07, + "output_cost_per_token_flex": 6.2e-07, + "output_cost_per_token_priority": 2.17e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14086,14 +14122,14 @@ "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "supports_tool_choice": true, + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14105,13 +14141,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14120,16 +14156,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14140,13 +14176,13 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_audio_input": false, - "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "supports_vision": true }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14157,13 +14193,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14187,13 +14223,17 @@ }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14201,34 +14241,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14237,24 +14283,30 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -14337,29 +14389,35 @@ "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14368,7 +14426,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14379,13 +14439,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14410,29 +14470,35 @@ "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14441,7 +14507,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14452,13 +14520,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14483,29 +14551,35 @@ "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14514,18 +14588,24 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.86e-06, - "output_cost_per_token_flex": 9.3e-07 + "output_cost_per_token_batches": 9.3e-07, + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_batches": 1.15e-07, "input_cost_per_token_flex": 1.15e-07, + "input_cost_per_token_priority": 4.025e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, + "output_cost_per_token_batches": 5.05e-07, "output_cost_per_token_flex": 5.05e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_priority": 1.7675e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, @@ -14536,20 +14616,24 @@ }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, + "input_cost_per_token_batches": 3.9e-07, "input_cost_per_token_flex": 3.9e-07, + "input_cost_per_token_priority": 1.365e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14593,13 +14677,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14608,16 +14692,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14628,13 +14712,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14675,13 +14759,17 @@ }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14689,34 +14777,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14725,7 +14819,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14743,20 +14839,24 @@ }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14767,13 +14867,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -14908,13 +15008,17 @@ }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14922,42 +15026,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14975,47 +15087,59 @@ }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, + "input_cost_per_token_priority": 8.75e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15023,34 +15147,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -15059,7 +15189,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.2e-06, - "output_cost_per_token_flex": 6e-07 + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15077,20 +15209,24 @@ }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15101,13 +15237,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15543,13 +15679,17 @@ }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15557,42 +15697,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15610,20 +15758,24 @@ }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15634,13 +15786,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -25109,38 +25261,6 @@ "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, - "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { - "cache_creation_input_token_cost": 1.375e-06, - "cache_creation_input_token_cost_above_1hr": 2.2e-06, - "cache_read_input_token_cost": 1.1e-07, - "input_cost_per_token": 1.1e-06, - "deprecation_date": "2026-10-15", - "litellm_provider": "bedrock_converse", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096, - "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06, - "token_count_model": { - "model": "anthropic.claude-haiku-4-5-20251001-v1:0", - "litellm_provider": "bedrock" - } - }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -46798,12 +46918,16 @@ }, "qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, + "input_cost_per_token_priority": 2.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -46813,12 +46937,16 @@ }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46828,12 +46956,18 @@ }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_batches": 7.1e-07, + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46858,12 +46992,16 @@ }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46873,12 +47011,16 @@ }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46888,12 +47030,16 @@ }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -61028,11 +61174,17 @@ }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61041,15 +61193,21 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61058,7 +61216,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, diff --git a/tests/unit/litellm_core_utils/test_get_supported_openai_params.py b/tests/unit/litellm_core_utils/test_get_supported_openai_params.py index f9e285cf9fb..8fca7bafedf 100644 --- a/tests/unit/litellm_core_utils/test_get_supported_openai_params.py +++ b/tests/unit/litellm_core_utils/test_get_supported_openai_params.py @@ -4,7 +4,7 @@ from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) -BEDROCK_REAL_MODEL = "eu.anthropic.claude-haiku-4-5-20251001-v1:0" +BEDROCK_REAL_MODEL = "us.anthropic.claude-haiku-4-5-20251001-v1:0" BEDROCK_LABEL = "claude-haiku-4-5" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 243a20a2551..0f471b1de71 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4391,7 +4391,7 @@ class TestBedrockBaseModelLabelKeepsTools: from litellm.utils import get_optional_params result = get_optional_params( - model="eu.anthropic.claude-haiku-4-5-20251001-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", custom_llm_provider="bedrock", base_model="claude-haiku-4-5", tools=self.TOOLS, From 369f3cb5f0b4284ab6b7d8faba2019137fb145d0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:08:45 -0700 Subject: [PATCH 18/34] feat(cost-map): add baseten joey-1 decision model (#49123) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 9 +++++++++ model_prices_and_context_window.json | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1a1982fddd7..77ac4f49e8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -81059,5 +81059,14 @@ "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true + }, + "baseten/decision-models/baseten-joey-2b-latest": { + "input_cost_per_token": 1e-08, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1a1982fddd7..77ac4f49e8b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -81059,5 +81059,14 @@ "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true + }, + "baseten/decision-models/baseten-joey-2b-latest": { + "input_cost_per_token": 1e-08, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true } } From 733cfeba935284056e77baf2dcc9517fdc96a2ba Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 29 Oct 2026 20:30:42 -0700 Subject: [PATCH 19/34] fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced (#48953) * fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced * test(integration): a reported cost is attributed to output in the gateway parity cases * fix(cost): skip a chat usage.cost that already holds litellm's stamped bill and Responses API objects * fix(cost): record litellm's bill on the Responses terminal event so a later pricing pass does not read it as the upstream's charge * fix(cost): mark usage.cost litellm wrote as its own bill and keep a breakdown for a reported cost * fix(cost): keep the token breakdown when a header reported cost repeats an already priced bill * fix(cost): read the stored breakdown total through the logging object's typed field * refactor(cost): name the providers billed from their own usage fields for what they are * fix(cost): keep the upstream's usage.cost beside litellm's bill and stamp a zero operator price on the stream frame --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 21 +- litellm/litellm_core_utils/core_helpers.py | 37 ++- .../streaming_chunk_builder_utils.py | 23 +- litellm/responses/streaming_iterator.py | 3 +- .../cost_calculation/cases/openai.py | 225 ++++++++++++++++++ .../integration/cost_calculation/cost_map.py | 6 + .../litellm_core_utils/test_core_helpers.py | 78 +++--- .../test_streaming_handler.py | 6 +- .../llms/openai_like/test_kyma_provider.py | 8 +- tests/unit/test_main.py | 40 ++++ 10 files changed, 387 insertions(+), 60 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6677b7327dc..ee102e82ca4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1544,6 +1544,12 @@ def _response_time_ms_for_cost( return litellm_logging_obj.get_response_ms() +def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None: + breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown + total: Final[object] = None if breakdown is None else breakdown.get("total_cost") + return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None + + def _adjusted_provider_billed_cost( reported_cost: float, custom_llm_provider: str | None, @@ -1561,9 +1567,9 @@ def _adjusted_provider_billed_cost( ) _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, - prompt_tokens_cost_usd_dollar=None, - completion_tokens_cost_usd_dollar=None, - cost_for_built_in_tools_cost_usd_dollar=None, + prompt_tokens_cost_usd_dollar=0.0, + completion_tokens_cost_usd_dollar=adjusted_cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=adjusted_cost, original_cost=reported_cost, discount_percent=discount_percent, @@ -2376,6 +2382,15 @@ def response_cost_calculator( ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) if provider_response_cost is not None and not custom_pricing: + if _breakdown_total(litellm_logging_obj) != provider_response_cost: + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=0.0, + completion_tokens_cost_usd_dollar=provider_response_cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, + total_cost_usd_dollar=provider_response_cost, + original_cost=provider_response_cost, + ) return provider_response_cost response_cost = completion_cost( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index d940baf8b56..28f45d2bb2c 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) -USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) +PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"}) + + +UPSTREAM_USAGE_COST: Final = "upstream_usage_cost" + + +def _numeric(value: object) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None: + """Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own + ``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of + reading litellm's bill back as if the upstream had reported it.""" + hidden: Final[object] = getattr(response, "_hidden_params", None) + if isinstance(hidden, dict): + hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict + upstream: Final = ( + hidden_params[UPSTREAM_USAGE_COST] + if UPSTREAM_USAGE_COST in hidden_params + else _numeric(getattr(usage, "cost", None)) + ) + hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream}) + setattr(usage, "cost", bill) def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: - if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + """The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own + usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal + event do), the upstream's figure is read from ``upstream_usage_cost`` instead.""" + if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: return None - cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None)) + cost: Final = ( + _numeric(hidden[UPSTREAM_USAGE_COST]) + if hidden is not None and UPSTREAM_USAGE_COST in hidden + else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + ) return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index c69d9277d65..94351e72ab1 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( - USAGE_COST_BILLED_PROVIDERS, + PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS, chunk_usage, provider_reported_usage_cost, + stamp_litellm_bill_on_usage, ) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping @@ -1380,9 +1381,6 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS - - def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: return None @@ -1390,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object: return provider -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: - return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS - - def _stream_builder_response_cost( response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] ) -> float | None: @@ -1404,9 +1398,6 @@ def _stream_builder_response_cost( for chunk in reversed(chunks) ) return next((cost for cost in reported if cost is not None), None) - usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): - return float(usage_cost) if logging_obj is not None: return None provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor @@ -1453,13 +1444,15 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if isinstance(getattr(usage, "cost", None), (int, float)): + if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance( + getattr(usage, "cost", None), (int, float) + ): return computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: + if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0: return - setattr(usage, "cost", computed_cost) - _set_hidden_response_cost(response, float(computed_cost)) + if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)): + stamp_litellm_bill_on_usage(response, usage, float(computed_cost)) def _field_of(value: object, name: str) -> object: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6a82304b76a..f0bb2201162 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.core_helpers import ( process_response_headers, provider_reported_usage_cost, set_response_cost_in_hidden_params, + stamp_litellm_bill_on_usage, ) from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import ( @@ -1869,7 +1870,7 @@ def stamp_responses_usage_cost( if isinstance(bill, bool) or not isinstance(bill, (int, float)): return if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)): - setattr(usage_obj, "cost", float(bill)) + stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill)) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 53fd5c1cec2..1443cc5ab59 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +from collections.abc import Mapping from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( @@ -11878,7 +11879,215 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS } ) + +# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill, +# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed +# replies alike. A deployment the operator priced keeps the operator's rates on both. +_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected( + spend=0.037, input_cost=0.0, output_cost=0.037, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06) +_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected( + spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_CHUNK: Final = { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1, + "model": "gateway-reported-cost-fixture", +} + +GATEWAY_REPORTED_COST: Final = CostTrackingTestCase( + name="gateway-reported-cost", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False}, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1, + "model": "gateway-reported-cost-fixture", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": _GATEWAY_REPORTED_USAGE, + }, + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase( + name="gateway-reported-cost-stream", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported cost"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE}, + done=True, + ), + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = { + "id": "msg_gateway", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gateway-reported-cost-fixture", + "output": [_GATEWAY_RESPONSES_ITEM], + "usage": _GATEWAY_RESPONSES_USAGE, +} + +GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase( + name="gateway-reported-cost-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/responses", + request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}}, + response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-stream", + "request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True}, + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_gateway", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY}, + ), + ), + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + + +_GATEWAY_ZERO_USAGE: Final = {**_GATEWAY_REPORTED_USAGE, "cost": 0} + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced", + "response": JsonResponse( + content_type="application/json", + body={**GATEWAY_REPORTED_COST.response.body, "usage": _GATEWAY_ZERO_USAGE}, + ), + } +) + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced-stream", + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_ZERO_USAGE}, + done=True, + ), + ), + } +) + +_GATEWAY_OPENROUTER_OPERATOR_PRICED: Final = _GATEWAY_OPERATOR_PRICED.model_copy( + update={"model": "openrouter/vendor/gateway-reported-cost-fixture"} +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = ( + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced-stream", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } + ) +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + GATEWAY_REPORTED_COST, + GATEWAY_REPORTED_COST_STREAM, + GATEWAY_REPORTED_COST_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_RESPONSES, + GATEWAY_REPORTED_COST_RESPONSES_STREAM, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12120,6 +12329,22 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( + StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, + ), + StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 32722b37c9b..da4c2b6e859 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_128k_tokens": 2.2e-07, "output_cost_per_token_above_128k_tokens": 8.8e-07, }, + "gateway-reported-cost-fixture": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, "openrouter/anthropic/claude-sonnet-5": { "litellm_provider": "openrouter", "mode": "chat", diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index a4a3ae285b1..1c86c4e30d8 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -69,9 +69,7 @@ def test_preserve_upstream_non_openai_attributes_keeps_scalar_and_structured_val scalar_attribute="scalar", structured_attribute={"items": ["value"]}, ) - model_response: Final = ModelResponseStream( - choices=[StreamingChoices(index=0, delta=Delta(content="hello"))] - ) + model_response: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="hello"))]) preserve_upstream_non_openai_attributes(model_response, original_chunk) @@ -262,9 +260,7 @@ class TestMapFinishReasonAnthropic: ("content_filtered", "content_filter"), ], ) - def test_anthropic_finish_reasons( - self, provider_reason: str, expected: str - ) -> None: + def test_anthropic_finish_reasons(self, provider_reason: str, expected: str) -> None: assert map_finish_reason(provider_reason) == expected def test_refusal(self): @@ -336,9 +332,7 @@ class TestMapFinishReasonZhipu: class TestMapFinishReasonOpenAIPassthrough: - @pytest.mark.parametrize( - "reason", ["stop", "length", "tool_calls", "function_call", "content_filter"] - ) + @pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "function_call", "content_filter"]) def test_openai_values_pass_through(self, reason): assert map_finish_reason(reason) == reason @@ -349,9 +343,7 @@ class TestMapFinishReasonGenericError: assert map_finish_reason("error") == "stop" def test_lowercase_error_does_not_warn(self, mocker): - warn = mocker.patch( - "litellm.litellm_core_utils.core_helpers.verbose_logger.warning" - ) + warn = mocker.patch("litellm.litellm_core_utils.core_helpers.verbose_logger.warning") assert map_finish_reason("error") == "stop" warn.assert_not_called() @@ -369,8 +361,7 @@ class TestFinishReasonMapOutputsAreValid: """Every value in _FINISH_REASON_MAP must be a valid OpenAI finish reason.""" for provider_reason, openai_reason in _FINISH_REASON_MAP.items(): assert openai_reason in VALID_OPENAI_FINISH_REASONS, ( - f"Mapped value '{openai_reason}' (from '{provider_reason}') " - f"is not a valid OpenAI finish reason" + f"Mapped value '{openai_reason}' (from '{provider_reason}') is not a valid OpenAI finish reason" ) @@ -380,28 +371,18 @@ class TestRedactNestedMatchAndRegexKeys: "assessments": [ { "sensitiveInformationPolicy": { - "piiEntities": [ - {"type": "NAME", "match": "secret-name", "action": "BLOCKED"} - ] - }, - "wordPolicy": { - "customWords": [{"match": "badword", "action": "BLOCKED"}] + "piiEntities": [{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}] }, + "wordPolicy": {"customWords": [{"match": "badword", "action": "BLOCKED"}]}, } ], "regex": "should-redact-key-named-regex", } out = redact_nested_match_and_regex_keys(payload) - assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ - "match" - ] == "[REDACTED]" - assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ( - "[REDACTED]" - ) + assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "[REDACTED]" + assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ("[REDACTED]") assert out["regex"] == "[REDACTED]" - assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][ - 0 - ]["match"] == "secret-name" + assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "secret-name" def test_passes_through_none_and_str(self): assert redact_nested_match_and_regex_keys(None) is None @@ -564,13 +545,17 @@ class TestIsExpectedClientError: assert is_expected_client_error(over_budget) is True litellm_limit = RateLimitError( - message="key over rpm", llm_provider="anthropic", model="claude-haiku-4-5", + message="key over rpm", + llm_provider="anthropic", + model="claude-haiku-4-5", category=RateLimitErrorCategory.LITELLM_RATE_LIMIT, ) assert is_expected_client_error(litellm_limit) is True vendor_limit = RateLimitError( - message="rate limited upstream", llm_provider="anthropic", model="claude-haiku-4-5", + message="rate limited upstream", + llm_provider="anthropic", + model="claude-haiku-4-5", category=RateLimitErrorCategory.VENDOR_RATE_LIMIT, ) assert is_expected_client_error(vendor_limit) is False @@ -803,3 +788,34 @@ class TestReceiveStampSurvivesASecondMetadataBucket: } assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own + + +def test_billed_usage_cost_bills_any_chat_upstream_without_its_own_reader() -> None: + from types import SimpleNamespace + + from litellm.litellm_core_utils.core_helpers import billed_usage_cost + from litellm.types.utils import ModelResponse, Usage + + usage = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037) + chat = ModelResponse(usage=usage) + assert billed_usage_cost(chat, "openai") == 0.037 + assert billed_usage_cost(chat, "openrouter") == 0.037 + assert billed_usage_cost(chat, "xai") is None + assert billed_usage_cost(SimpleNamespace(usage=usage), "openai") == 0.037 + + +def test_billed_usage_cost_keeps_the_upstream_charge_after_litellm_wrote_its_bill_into_usage_cost() -> None: + from litellm.litellm_core_utils.core_helpers import billed_usage_cost, stamp_litellm_bill_on_usage + from litellm.types.utils import ModelResponse, Usage + + reported = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037)) + stamp_litellm_bill_on_usage(reported, reported.usage, 0.0407) + stamp_litellm_bill_on_usage(reported, reported.usage, 0.0407) + assert reported.usage.cost == 0.0407 + assert reported._hidden_params["response_cost"] == 0.0407 + assert billed_usage_cost(reported, "openai") == 0.037 + + unreported = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)) + stamp_litellm_bill_on_usage(unreported, unreported.usage, 0.0015) + assert unreported.usage.cost == 0.0015 + assert billed_usage_cost(unreported, "openai") is None diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index f4502937fce..41e8c50fa85 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2659,7 +2659,7 @@ def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator(): ) == pytest.approx(0.00503) -def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): +def test_openai_compatible_streaming_reported_cost_is_billed_instead_of_the_cost_map(): import litellm from litellm.cost_calculator import ( get_response_cost_from_hidden_params, @@ -2681,7 +2681,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): id="chatcmpl-openai-compatible", model=model, choices=[], - usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000), + usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.037), ) CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai") @@ -2693,7 +2693,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): custom_llm_provider="openai", call_type="completion", optional_params={}, - ) == pytest.approx(2e-5) + ) == pytest.approx(0.037) def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch): diff --git a/tests/unit/llms/openai_like/test_kyma_provider.py b/tests/unit/llms/openai_like/test_kyma_provider.py index 2b3740ca676..e834e0fc0e9 100644 --- a/tests/unit/llms/openai_like/test_kyma_provider.py +++ b/tests/unit/llms/openai_like/test_kyma_provider.py @@ -254,7 +254,7 @@ def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports(): @pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat")) -def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str): +def test_interaction_on_another_openai_compatible_provider_is_billed_at_its_reported_cost(model: str): chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721} responses_body: Final = { "id": "resp_upstream", @@ -278,7 +278,7 @@ def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body) interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k") - assert "cost" not in interaction.model_dump(exclude_none=True)["usage"] + assert interaction.model_dump(exclude_none=True)["usage"]["cost"] == pytest.approx(0.00721) def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports(): @@ -410,8 +410,8 @@ def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_repo assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004) assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004) assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044) - assert headers["x-litellm-response-cost-input"] is None - assert headers["x-litellm-response-cost-output"] is None + assert float(headers["x-litellm-response-cost-input"] or "nan") == 0.0 + assert float(headers["x-litellm-response-cost-output"] or "nan") == pytest.approx(0.0044) @pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"})) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 571f890a4e6..b2ae6b43d26 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5443,6 +5443,46 @@ def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): assert response._hidden_params["response_cost"] == pytest.approx(0.5) +def test_completion_cost_on_a_stamped_stream_still_bills_the_reported_charge(): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=_stream_builder_logging_obj() + ) + + assert response is not None + assert litellm.completion_cost(completion_response=response) == pytest.approx(0.5) + + +def test_stream_chunk_builder_stamps_a_zero_operator_price_over_a_reported_cost(): + logging_obj: Final = _stream_builder_logging_obj() + logging_obj.litellm_params = { + **logging_obj.litellm_params, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + logging_obj.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert getattr(response.usage, "cost", None) == 0.0 + assert response._hidden_params["response_cost"] == 0.0 + + def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): from openai.types.completion_usage import CompletionUsage From 342ce6ae4a4549cacaf96e47aa80a096ae1035bd Mon Sep 17 00:00:00 2001 From: DewellNaa Date: Thu, 29 Oct 2026 20:30:50 -0700 Subject: [PATCH 20/34] feat(zai): support response_format and max_completion_tokens, add glm-5v-turbo pricing (#48335) * feat(zai): fall back to BIGMODEL/ZHIPUAI api keys and add GLM-5.3 aliases and glm-5v-turbo Resolve the zai api key from ZAI_API_KEY, BIGMODEL_API_KEY or ZHIPUAI_API_KEY so deployments predating the zai rename keep working. Accept response_format and max_completion_tokens as supported openai params. Register zai/GLM-5.3 and zai/GLM-5.3-Flash aliases plus the glm-5v-turbo vision model in both the main and backup cost maps * fix(zai): read only ZAI_API_KEY, dropping the BigModel key fallbacks * fix(zai): map max_completion_tokens to max_tokens, correct glm-5v-turbo pricing, drop redundant aliases * fix(zai): reject json_schema response_format, add glm-5v-turbo cache read price * fix(zai): forward every response_format type so allowed_openai_params keeps working * fix(zai): follow json_schema natively on glm-5.3, prompt other models, drop over-limit max_tokens under drop_params * fix(zai): keep BigModel's json_schema as sent and never drop the caller's max_tokens under drop_params * fix(zai): write the prompted JSON schema as JSON so models answer with valid JSON * test(zai): type the zai provider tests and mark their locals Final --------- Co-authored-by: DewellNaa <210961864+DewellNaa@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../prompt_templates/factory.py | 34 ++- litellm/llms/bigmodel/chat/transformation.py | 4 + litellm/llms/zai/chat/transformation.py | 94 ++++++++- ...odel_prices_and_context_window_backup.json | 17 ++ model_prices_and_context_window.json | 17 ++ .../llms/bigmodel/test_bigmodel_provider.py | 23 ++ tests/unit/llms/zai/test_zai_provider.py | 197 ++++++++++++++++++ 7 files changed, 364 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..834d26b4fe8 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5878,38 +5878,36 @@ def function_call_prompt( ] -def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str: +def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str: """ Decides if a user-defined custom prompt or default needs to be used Returns the prompt str that's passed to the model as a user message """ - custom_prompt_details: dict | None = None - response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}] - if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict: - custom_prompt_details = litellm.custom_prompt_dict[ - f"{model}/response_schema_prompt" - ] # allow user to define custom response schema prompt by model - elif "response_schema_prompt" in litellm.custom_prompt_dict: - custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"] - - if custom_prompt_details is not None: - return render_custom_prompt_template( - template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider - ) - else: + prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict + model_prompt_key: Final = f"{model}/response_schema_prompt" + custom_prompt_details: Final = prompts.get( + model_prompt_key if model_prompt_key in prompts else "response_schema_prompt" + ) + if custom_prompt_details is None: return default_response_schema_prompt(response_schema=response_schema) + return render_custom_prompt_template( + template=custom_prompt_details, + messages=[{"role": "user", "content": f"{response_schema}"}], + model=model, + llm_provider=llm_provider, + ) -def default_response_schema_prompt(response_schema: dict) -> str: +def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str: """ Used if provider/model doesn't support 'response_schema' param. This is the default prompt. Allow user to override this with a custom_prompt. """ prompt_str: Final = f"""Use this JSON schema: - ```json - {response_schema} + ```json + {json.dumps(response_schema, default=str)} ```""" return prompt_str diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py index 12d5e912c15..93e9e24fe7b 100644 --- a/litellm/llms/bigmodel/chat/transformation.py +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig): def custom_llm_provider(self) -> str: return "bigmodel" + @property + def prompts_unsupported_json_schema(self) -> bool: + return False + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) diff --git a/litellm/llms/zai/chat/transformation.py b/litellm/llms/zai/chat/transformation.py index e0e42d772f8..9e2b97d2bed 100644 --- a/litellm/llms/zai/chat/transformation.py +++ b/litellm/llms/zai/chat/transformation.py @@ -1,11 +1,43 @@ +from collections.abc import Mapping from typing import Final +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 + +import litellm +from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage +from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration from ...openai.chat.gpt_transformation import OpenAIGPTConfig ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4" +TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"}) + + +def _is_json_object( + value: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: cost-map rows and request params are JSON + return isinstance(value, Mapping) + + +def _json_field(value: object, key: str) -> object: + return value.get(key) if _is_json_object(value) else None + + +def _exceeds_output_limit(model: str, max_tokens: object) -> bool: + info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai") + limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens") + return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit + + +def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None: + if _json_field(response_format, "type") != "json_schema": + return None + if litellm.supports_response_schema(model=model, custom_llm_provider="zai"): + return None + schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema") + return schema if _is_json_object(schema) else None class ZAIChatConfig(OpenAIGPTConfig): @@ -13,6 +45,10 @@ class ZAIChatConfig(OpenAIGPTConfig): def custom_llm_provider(self) -> str | None: return "zai" + @property + def prompts_unsupported_json_schema(self) -> bool: + return True + def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: @@ -40,9 +76,11 @@ class ZAIChatConfig(OpenAIGPTConfig): # GLM/ZAI supports cache_control, so return messages and tools unchanged return messages, tools - def get_supported_openai_params(self, model: str) -> list: + def get_supported_openai_params(self, model: str) -> list[str]: base_params: Final = [ "max_tokens", + "max_completion_tokens", + "response_format", "stream", "stream_options", "temperature", @@ -52,8 +90,6 @@ class ZAIChatConfig(OpenAIGPTConfig): "tool_choice", ] - import litellm - try: if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_params.append("thinking") @@ -61,3 +97,53 @@ class ZAIChatConfig(OpenAIGPTConfig): pass return base_params + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + supported_params: Final = self.get_supported_openai_params(model) + sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params} + max_completion_tokens: Final = sent.get("max_completion_tokens") + falls_back_to_max_tokens: Final = max_completion_tokens is None or ( + (litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_completion_tokens) + ) + max_tokens: Final = sent.get("max_tokens") if falls_back_to_max_tokens else max_completion_tokens + return { + **optional_params, + **{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS}, + **({} if max_tokens is None else {"max_tokens": max_tokens}), + } + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature + optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature + ) -> dict[str, object]: # mutable-ok: inherited contract returns the request body + schema: Final = ( + _schema_without_native_support(model, optional_params.get("response_format")) + if self.prompts_unsupported_json_schema + else None + ) + schema_message: Final = ( + () + if schema is None + else ( + ChatCompletionUserMessage( + role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai") + ), + ) + ) + return super().transform_request( + model, + [*messages, *schema_message], + optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}}, + litellm_params, + headers, + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 77ac4f49e8b..96a04c96f44 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -54512,6 +54512,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "zai/glm-5.3-flash": { @@ -54526,10 +54527,26 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-5v-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://vercel.com/ai-gateway/models/glm-5v-turbo", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "zai/glm-5.1": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2.6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 77ac4f49e8b..96a04c96f44 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -54512,6 +54512,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "zai/glm-5.3-flash": { @@ -54526,10 +54527,26 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-5v-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://vercel.com/ai-gateway/models/glm-5v-turbo", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "zai/glm-5.1": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2.6e-07, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index cdf98c8f9a4..6abfcb060d0 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -197,6 +197,29 @@ def test_chat_does_not_send_the_openai_organization_to_bigmodel(monkeypatch: pyt assert "openai-organization" not in route.calls.last.request.headers +@respx.mock +def test_chat_sends_json_schema_response_format_as_given() -> None: + route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "object": "chat.completion", + "created": 1, + "model": "glm-4.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "{}"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + messages: Final = [{"role": "user", "content": "Ann likes 7"}] + response_format: Final = { + "type": "json_schema", + "json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"n": {"type": "integer"}}}}, + } + _ = litellm.completion(model="bigmodel/glm-4.6", messages=messages, response_format=response_format) + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == response_format + assert body["messages"] == messages + + @pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.asyncio @respx.mock diff --git a/tests/unit/llms/zai/test_zai_provider.py b/tests/unit/llms/zai/test_zai_provider.py index 32849d5eef1..fb126bd8f41 100644 --- a/tests/unit/llms/zai/test_zai_provider.py +++ b/tests/unit/llms/zai/test_zai_provider.py @@ -2,13 +2,27 @@ Tests for Z.AI (Zhipu AI) provider - GLM models """ +import json import math +from collections.abc import Mapping +from typing import Final import pytest +import respx import litellm from litellm import completion from litellm.cost_calculator import cost_per_token +from litellm.llms.zai.chat.transformation import ZAIChatConfig +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + +JSON_SCHEMA_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "profile", + "schema": {"type": "object", "properties": {"favorite_number": {"type": "integer"}}}, + }, +} @pytest.fixture @@ -99,3 +113,186 @@ def test_zai_sync_completion(respx_mock, zai_response, monkeypatch): assert response.choices[0].message.content == "Hello! How can I help you today?" assert response.usage.total_tokens == 25 + + +def test_zai_sends_response_format_and_maps_max_completion_tokens( + respx_mock: respx.MockRouter, zai_response: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + + completion( + model="zai/glm-5.3", + messages=[{"role": "user", "content": "Reply in JSON"}], + response_format={"type": "json_object"}, + max_completion_tokens=64, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == {"type": "json_object"} + assert body["max_tokens"] == 64 + assert "max_completion_tokens" not in body + + +@pytest.mark.parametrize( + "token_limits", + [{"max_tokens": 300, "max_completion_tokens": 15}, {"max_completion_tokens": 15, "max_tokens": 300}], +) +def test_zai_max_completion_tokens_wins_over_max_tokens(token_limits: dict[str, int]) -> None: + mapped: Final = ZAIChatConfig().map_openai_params(token_limits, {}, "glm-5.3", drop_params=False) + + assert mapped["max_tokens"] == 15 + assert "max_completion_tokens" not in mapped + + +@pytest.mark.parametrize( + ("over_limit_by", "drop_params", "forwarded"), + [(1, True, False), (1, False, True), (0, True, True)], +) +def test_zai_drop_params_drops_max_completion_tokens_above_model_limit( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + over_limit_by: int, + drop_params: bool, + forwarded: bool, +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + requested: Final = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + over_limit_by + + completion( + model="zai/glm-4.5-flash", + messages=[{"role": "user", "content": "Count"}], + max_completion_tokens=requested, + drop_params=drop_params, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body.get("max_tokens") == (requested if forwarded else None) + + +@pytest.mark.parametrize("with_over_limit_max_completion_tokens", [False, True]) +def test_zai_drop_params_keeps_max_tokens_above_model_limit( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + with_over_limit_max_completion_tokens: bool, +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + over_limit: Final = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + 1 + over_limit_max_completion_tokens: Final = ( + {"max_completion_tokens": over_limit + 1} if with_over_limit_max_completion_tokens else {} + ) + + completion( + model="zai/glm-4.5-flash", + messages=[{"role": "user", "content": "Count"}], + max_tokens=over_limit, + drop_params=True, + **over_limit_max_completion_tokens, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["max_tokens"] == over_limit + + +@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}]) +def test_zai_forwards_json_schema_to_models_that_enforce_it( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + pass_through: dict[str, list[str]], +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + messages: Final = [{"role": "user", "content": "Ann likes 7"}] + + completion(model="zai/glm-5.3", messages=messages, response_format=JSON_SCHEMA_RESPONSE_FORMAT, **pass_through) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT + assert body["messages"] == messages + + +@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}]) +def test_zai_puts_json_schema_in_prompt_for_models_that_ignore_it( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + pass_through: dict[str, list[str]], +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + + completion( + model="zai/glm-4.6", + messages=[{"role": "user", "content": "Ann likes 7"}], + response_format=JSON_SCHEMA_RESPONSE_FORMAT, + **pass_through, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == {"type": "json_object"} + assert body["messages"][0] == {"role": "user", "content": "Ann likes 7"} + assert body["messages"][-1]["role"] == "user" + prompt: Final = body["messages"][-1]["content"] + rendered_schema: Final = prompt[prompt.index("{") : prompt.rindex("}") + 1] + assert json.loads(rendered_schema) == JSON_SCHEMA_RESPONSE_FORMAT["json_schema"]["schema"] + + +def test_zai_glm_5v_turbo_bills_cached_input_at_cache_read_price(local_model_cost_map: None) -> None: + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800), + ) + + prompt_cost, _ = cost_per_token(model="glm-5v-turbo", custom_llm_provider="zai", usage_object=usage) + + entry: Final = litellm.model_cost["zai/glm-5v-turbo"] + assert math.isclose( + prompt_cost, 200 * entry["input_cost_per_token"] + 800 * entry["cache_read_input_token_cost"] + ) + + +@pytest.mark.parametrize("model", ["glm-5v-turbo", "GLM-5V-Turbo", "GLM-5.3", "GLM-5.3-Flash"]) +def test_zai_cost_resolves_case_insensitively(local_model_cost_map: None, model: str) -> None: + prompt_cost, completion_cost = cost_per_token( + model=model, custom_llm_provider="zai", prompt_tokens=1000, completion_tokens=1000 + ) + + entry: Final = litellm.model_cost[f"zai/{model.lower()}"] + assert math.isclose(prompt_cost, 1000 * entry["input_cost_per_token"]) + assert math.isclose(completion_cost, 1000 * entry["output_cost_per_token"]) + + +def test_zai_glm_5v_turbo_accepts_thinking( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + + completion( + model="zai/glm-5v-turbo", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["thinking"] == {"type": "disabled"} From b91fc92b6d0815df038e3496b669871fd917c056 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:32:04 -0700 Subject: [PATCH 21/34] fix(router): cap the default OTPM reservation and reconcile token counters after each call (#49054) * fix(router): cap the default OTPM reservation for requests without max_tokens * fix(router): reconcile ITPM and OTPM reservations after every successful call * fix(router): read the default OTPM reservation from the served model and make its fraction configurable The default reservation now looks up max output from litellm_params.model, as main does, so a pricing base_model never changes it. NO_MAX_TOKENS_TPM_RESERVATION_FRACTION sets the fraction for both the router check and the key and team TPM floor. A max_tokens that is not a number or string counts as no cap instead of raising inside the check. * fix(router): refund a failed attempt's io token reservation in make_call The proxy passes one logging object to every retry, and it runs failure callbacks only for the first attempt, so the last attempt's ITPM and OTPM reservation stayed held until the minute ended. Each attempt that raises now gives its reservation back before the retry or fallback * fix(router): count Anthropic cache writes once in ITPM reconcile Native /v1/messages usage reports input_tokens without cache reads and writes, so the reconcile now maps it through AnthropicConfig.calculate_usage before subtracting cached tokens. * fix(router): use a deployment max_tokens, charge late usage to its finish minute, and read the reservation fraction as a fraction * fix(router): reserve with the deployment cap only through the param the request's API sends * fix(router): never let a deployment's output cap raise the reservation above the default * test(router): move the reservation fraction env var test to the integration suite The unit test started a child interpreter, which the unit guide forbids. The integration test boots a real proxy with the env var set and checks admission against the held reservation. * test(router): retry the reservation fraction scenario when the minute rolls over The OTPM counter is keyed by minute, so a rollover between the held stream and the capped requests gave the capped request a fresh counter. The control request now leaves room for usage the first attempt left in the new minute. * test(router): give the rollover retry its own deployment --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 4 + litellm/litellm_core_utils/env_utils.py | 10 +- .../hooks/parallel_request_limiter_v3.py | 17 +- litellm/router.py | 30 +- .../io_token_rate_limit_check.py | 185 +++--- .../pre_call_checks/model_rate_limit_check.py | 37 +- .../test_otpm_default_reservation_fraction.py | 137 +++++ .../unit/litellm_core_utils/test_env_utils.py | 10 + tests/unit/test_constants.py | 9 +- .../test_router/test_io_token_rate_limits.py | 530 +++++++++++++++++- 10 files changed, 831 insertions(+), 138 deletions(-) create mode 100644 tests/integration/routing/test_otpm_default_reservation_fraction.py diff --git a/litellm/constants.py b/litellm/constants.py index e07d727973d..cfee777f36a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -7,6 +7,7 @@ from typing import Final, Literal from litellm.litellm_core_utils.env_utils import ( get_env_float, + get_env_float_in_range, get_env_float_or_none, get_env_int, get_env_int_in_range, @@ -149,6 +150,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float( "DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5 ) # default cooldown a deployment if 50% of requests fail in a given minute DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096) +NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range( + "NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0 +) DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3) DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1) DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5) diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py index 6883af98bbd..462b0b3df47 100644 --- a/litellm/litellm_core_utils/env_utils.py +++ b/litellm/litellm_core_utils/env_utils.py @@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) Values outside the range fall back to the default and warn, so a misconfigured knob can neither crash the caller nor silently change the meaning of what it computes. """ - value: Final = get_env_int(env_var, default) + return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum) + + +def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float: + """Float counterpart of `get_env_int_in_range`.""" + return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum) + + +def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT: if minimum <= value <= maximum: return value logging.getLogger("LiteLLM").warning( diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 404aff84994..14b08a09116 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -42,7 +42,11 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + NO_MAX_TOKENS_TPM_RESERVATION_FRACTION, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens @@ -508,11 +512,6 @@ REDIS_NODE_HASHTAG_NAME: Final = "all_keys" # *some* output budget; these define that fallback estimate. DEFAULT_MAX_TOKENS_ESTIMATE: Final = 4096 DEFAULT_CHARS_PER_TOKEN: Final = 4 -# Fraction of the available output budget reserved as the upfront floor when -# the request omits max_tokens. Applied to both DEFAULT_MAX_TOKENS_ESTIMATE -# (baseline floor) and to the smallest configured TPM limit (capped floor for -# small per-tenant TPM caps). -_TPM_FLOOR_FRACTION: Final = 4 # Both embeddings and the Responses API put their prompt in data["input"], # but only embeddings have no output tokens. Every "is this an embedding" # check on data["input"] must exclude these call types, or a Responses call @@ -1003,10 +1002,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): per-tenant cap can't be tripped by the floor alone. Returns the baseline floor when no limit is provided. """ - baseline: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION + baseline: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) if min_configured_tpm_limit is None: return baseline - return min(baseline, max(1, min_configured_tpm_limit // _TPM_FLOOR_FRACTION)) + return min(baseline, max(1, int(min_configured_tpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))) @staticmethod def _is_embedding_request(data: object, call_type: str | None) -> bool: @@ -1138,7 +1137,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if call_type in RESPONSES_API_CALL_TYPES else base_capped_floor ) - baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION + baseline_floor: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) is_embedding: Final = PROXY_MaxParallelRequestsHandler_v3._is_embedding_request(data, call_type) if ( capped_floor >= baseline_floor diff --git a/litellm/router.py b/litellm/router.py index 3a423b79e49..20af2e5649e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -377,6 +377,7 @@ from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check impor EncryptedContentAffinityCheck, ) from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + async_io_token_refund_failure, build_io_token_rate_limit_headers, deployment_has_io_token_limits, refund_stale_reservation_before_retry, @@ -9500,7 +9501,7 @@ class Router(DeprecatedRouterMembers): raise original_exception - async def make_call(self, original_function: Any, *args, **kwargs): + async def make_call(self, original_function: Any, *args: object, **kwargs: object): """ Handler for making a call to the .completion()/.embeddings()/etc. functions. """ @@ -9509,17 +9510,26 @@ class Router(DeprecatedRouterMembers): endpoint_function: Final = ( generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function) ) - with endpoint_selection( - getattr(endpoint_function, "__name__", None), - getattr(endpoint_function, "__module__", None), - model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group), - ): - response = original_function(*args, **kwargs) - if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): - response = await response + try: + with endpoint_selection( + getattr(endpoint_function, "__name__", None), + getattr(endpoint_function, "__module__", None), + model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group), + ): + response = original_function(*args, **kwargs) + if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): + response = await response + except Exception: + with contextlib.suppress(Exception): + await async_io_token_refund_failure( + self.cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) + ) + raise await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) ## PROCESS RESPONSE HEADERS - response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs) + response = await self.set_response_headers( + response=response, model_group=model_group if isinstance(model_group, str) else None, request_kwargs=kwargs + ) return response diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 0b3d48c9e1d..7580c7dadd3 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -1,7 +1,8 @@ """ Separate ITPM/OTPM (input/output tokens per minute) deployment rate limits. -- Pre-call: atomically reserve estimated_input against ITPM and max_tokens against OTPM +- Pre-call: atomically reserve estimated_input against ITPM and max_tokens against OTPM; a request + without max_tokens reserves the model's max output, capped at a fraction of the OTPM limit - Post-call: reconcile ITPM to actual input tokens and OTPM to actual output tokens - Cached prompt-read tokens are excluded from ITPM post-call accounting @@ -22,6 +23,7 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache +from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET @@ -127,10 +129,15 @@ def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | return itpm_key, otpm_key -def _tokenizer_model(deployment: object) -> str: +def _served_model(deployment: object) -> str: params: Final = deployment.get("litellm_params") if is_str_keyed_mapping(deployment) else None served: Final = params.get("model") if is_str_keyed_mapping(params) else None - return deployment_openai_model(deployment) or (served if isinstance(served, str) else "") + return served if isinstance(served, str) else "" + + +def _tokenizer_and_served_models(deployment: object) -> tuple[str, str]: + served: Final = _served_model(deployment) + return deployment_openai_model(deployment) or served, served def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "") -> int: @@ -160,24 +167,61 @@ def _model_max_output_tokens(model_name: str) -> int | None: return None -def _resolve_max_tokens(request_kwargs: dict[str, Any] | None, deployment: dict) -> int: - if request_kwargs: - # An explicit max_tokens=0 must be honored, not treated as absent and - # replaced by the model default. - explicit = request_kwargs.get("max_tokens") - if explicit is None: - explicit = request_kwargs.get("max_completion_tokens") - if explicit is None: - explicit = request_kwargs.get("max_output_tokens") - if explicit is not None: - return max(0, int(explicit)) +_CHAT_OUTPUT_CAP_KEYS: Final = ("max_tokens", "max_completion_tokens") +_RESPONSES_OUTPUT_CAP_KEYS: Final = ("max_output_tokens",) +_OUTPUT_CAP_KEYS: Final = _CHAT_OUTPUT_CAP_KEYS + _RESPONSES_OUTPUT_CAP_KEYS - model_name: Final = (deployment.get("litellm_params") or {}).get("model") - if model_name: - model_max: Final = _model_max_output_tokens(model_name) - if model_max is not None: - return model_max - return 4096 + +def _explicit_max_tokens( + request_kwargs: Mapping[str, object] | None, cap_keys: tuple[str, ...] = _OUTPUT_CAP_KEYS +) -> int | None: + if not request_kwargs: + return None + explicit: Final = next( + (request_kwargs[key] for key in cap_keys if request_kwargs.get(key) is not None), + None, + ) + match explicit: + case int() | float() | str(): + return max(0, int(explicit)) + case _: + return None + + +def _sent_deployment_cap_keys(request_kwargs: Mapping[str, object] | None) -> tuple[str, ...]: + is_responses_request: Final = ( + request_kwargs is not None and "input" in request_kwargs and "messages" not in request_kwargs + ) + return _RESPONSES_OUTPUT_CAP_KEYS if is_responses_request else _CHAT_OUTPUT_CAP_KEYS + + +def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployment: Mapping[str, object]) -> int | None: + params: Final = deployment.get("litellm_params") + return ( + _explicit_max_tokens(params, _sent_deployment_cap_keys(request_kwargs)) + if is_str_keyed_mapping(params) + else None + ) + + +def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int: + model_max: Final = _model_max_output_tokens(model_name) if model_name else None + share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)) + return min(4096 if model_max is None else model_max, share_of_limit) + + +def _otpm_reservation( + request_kwargs: Mapping[str, object] | None, + deployment: Mapping[str, object], + model_name: str, + otpm_limit: int, +) -> int: + from_request: Final = _explicit_max_tokens(request_kwargs) + if from_request is not None: + return from_request + default: Final = _default_otpm_reservation(model_name, otpm_limit) + from_deployment: Final = _deployment_output_cap(request_kwargs, deployment) + return default if from_deployment is None else min(from_deployment, default) def _get_usage_tokens(usage: object) -> tuple[int, int, int]: @@ -210,6 +254,12 @@ def _extract_response_usage(response_obj: object) -> object: return getattr(response_obj, "usage", None) +def _openai_shaped_usage(usage: object) -> object: + if isinstance(usage, dict) and litellm.AnthropicConfig.is_anthropic_usage_object(usage): + return litellm.AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None) + return usage + + def _usage_is_present(usage: object) -> bool: """ True only if usage carries an actual input/output breakdown. @@ -238,7 +288,7 @@ def _resolve_reconcile_usage_tokens( fields. When usage cannot be resolved, return ``usage_resolved=False`` so callers keep the pre-call reservation instead of refunding it as zero usage. """ - usage: Final = _extract_response_usage(response_obj) + usage: Final = _openai_shaped_usage(_extract_response_usage(response_obj)) if _usage_is_present(usage): prompt_tokens, completion_tokens, cached_tokens = _get_usage_tokens(usage) return max(0, prompt_tokens - cached_tokens), completion_tokens, True @@ -261,6 +311,21 @@ def _resolve_reconcile_usage_tokens( return 0, 0, False +def _key_for_this_minute(reserved_key: str) -> str: + counter, _, _ = reserved_key.rpartition(":") + return f"{counter}:{get_utc_datetime().strftime('%H-%M')}" + + +def _reconcile_increments( + reservation: tuple[int, int, str | None, str | None], billable_input: int, completion_tokens: int +) -> tuple[tuple[str, int], ...]: + itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation + deltas: Final = ((itpm_key, billable_input - itpm_reserved), (otpm_key, completion_tokens - otpm_reserved)) + return tuple( + (key if delta < 0 else _key_for_this_minute(key), delta) for key, delta in deltas if key is not None and delta + ) + + def _stash_reservation_in_metadata( request_kwargs: dict[str, object] | None, *, @@ -311,7 +376,9 @@ def _reservation_channels(kwargs: Mapping[str, object] | None) -> tuple[object, return () top_level: Final = (kwargs.get("metadata"), kwargs.get("litellm_metadata")) litellm_params: Final = _as_mutable_mapping(kwargs.get("litellm_params")) - from_params: Final = () if litellm_params is None else (litellm_params.get("metadata"),) + from_params: Final = ( + () if litellm_params is None else (litellm_params.get("metadata"), litellm_params.get("litellm_metadata")) + ) standard_logging_object: Final = _as_mutable_mapping(kwargs.get("standard_logging_object")) from_logging_object: Final = () if standard_logging_object is None else (standard_logging_object.get("metadata"),) return top_level + from_params + from_logging_object @@ -433,9 +500,11 @@ def io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = _tokenizer_model(deployment) + _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) - max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) + otpm_to_reserve: Final = ( + _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0 + ) dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") @@ -458,7 +527,7 @@ def io_token_pre_call_check( ) if otpm_limit is not None: - otpm_reserved = 0 if max_tokens == 0 else _reservation_value(max_tokens, otpm_limit) + otpm_reserved = otpm_to_reserve try: _sync_increment_with_rollback( dual_cache, @@ -497,9 +566,11 @@ async def async_io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = _tokenizer_model(deployment) + _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) - max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) + otpm_to_reserve: Final = ( + _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0 + ) dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") @@ -523,7 +594,7 @@ async def async_io_token_pre_call_check( ) if otpm_limit is not None: - otpm_reserved = 0 if max_tokens == 0 else _reservation_value(max_tokens, otpm_limit) + otpm_reserved = otpm_to_reserve try: await _increment_with_rollback( dual_cache, @@ -564,7 +635,8 @@ def io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) + reservation: Final = _read_reservation_from_kwargs(request_kwargs) + itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation if itpm_key is None and otpm_key is None: return @@ -572,23 +644,8 @@ def io_token_reconcile_success( try: if usage_resolved: - if itpm_key is not None: - itpm_delta: Final = billable_input - itpm_reserved - if itpm_delta != 0: - dual_cache.increment_cache( - key=itpm_key, - value=itpm_delta, - ttl=RoutingArgsTTL, - ) - - if otpm_key is not None: - otpm_delta: Final = completion_tokens - otpm_reserved - if otpm_delta != 0: - dual_cache.increment_cache( - key=otpm_key, - value=otpm_delta, - ttl=RoutingArgsTTL, - ) + for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens): + dual_cache.increment_cache(key=key, value=delta, ttl=RoutingArgsTTL) else: verbose_router_logger.debug( "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", @@ -618,39 +675,23 @@ async def async_io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) + reservation: Final = _read_reservation_from_kwargs(request_kwargs) + itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation if itpm_key is None and otpm_key is None: return billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) - # Reconcile against the exact key that held the reservation (which encodes - # the reservation's minute), not a key recomputed at response time. This - # tracks actual usage even when the pre-call estimate was 0, and avoids a - # minute-boundary mismatch for calls that span into the next minute. Always - # clear the stash afterwards (even if an increment throws) so a retry or a - # duplicate success event can't re-process it. + # A refund goes back to the minute that held the reservation, while usage above the reservation is charged + # to the minute the response finished in, so a call that runs past its minute still counts against a live + # window. Always clear the stash afterwards (even if an increment throws) so a retry or a duplicate success + # event can't re-process it. try: if usage_resolved: - if itpm_key is not None: - itpm_delta: Final = billable_input - itpm_reserved - if itpm_delta != 0: - await dual_cache.async_increment_cache( - key=itpm_key, - value=itpm_delta, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) - - if otpm_key is not None: - otpm_delta: Final = completion_tokens - otpm_reserved - if otpm_delta != 0: - await dual_cache.async_increment_cache( - key=otpm_key, - value=otpm_delta, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) + for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens): + await dual_cache.async_increment_cache( + key=key, value=delta, ttl=RoutingArgsTTL, parent_otel_span=parent_otel_span + ) else: verbose_router_logger.debug( "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index f4b41211c6e..f41d19cd62c 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -26,7 +26,6 @@ from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.integrations.custom_logger import CustomLogger from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( - ITPM_RESERVED_KEY, async_io_token_pre_call_check, async_io_token_reconcile_success, async_io_token_refund_failure, @@ -485,24 +484,12 @@ class ModelRateLimitingCheck(CustomLogger): try: standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") - - # IO token reconciliation works purely from the cache keys stashed in - # kwargs/metadata, so it must run before the model_id guard below - # (which only the TPM-tracking path needs). Otherwise a request whose - # standard_logging_object lacks model_id would never return its - # reservation, leaving the counter elevated until the TTL expires. - slo_metadata: Final = (standard_logging_object.get("metadata") or {}) if standard_logging_object else {} - kwargs_metadata: Final = kwargs.get("metadata") or {} - if ITPM_RESERVED_KEY in slo_metadata or ITPM_RESERVED_KEY in kwargs_metadata: - await async_io_token_reconcile_success( - self.dual_cache, - kwargs, - response_obj, - parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), - ) - # Fall through: a deployment can also configure tpm/rpm alongside - # itpm/otpm, and that path's pre-call check reads the tpm_key - # counter tracked below, so it must still be incremented here. + await async_io_token_reconcile_success( + self.dual_cache, + kwargs, + response_obj, + parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), + ) if standard_logging_object is None: return @@ -557,17 +544,7 @@ class ModelRateLimitingCheck(CustomLogger): """ try: standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") - slo_metadata: Final = (standard_logging_object.get("metadata") or {}) if standard_logging_object else {} - kwargs_metadata: Final = kwargs.get("metadata") or {} - if ITPM_RESERVED_KEY in slo_metadata or ITPM_RESERVED_KEY in kwargs_metadata: - io_token_reconcile_success( - self.dual_cache, - kwargs, - response_obj, - ) - # Fall through: a deployment can also configure tpm/rpm alongside - # itpm/otpm, and that path's pre-call check reads the tpm_key - # counter tracked below, so it must still be incremented here. + io_token_reconcile_success(self.dual_cache, kwargs, response_obj) if standard_logging_object is None: return diff --git a/tests/integration/routing/test_otpm_default_reservation_fraction.py b/tests/integration/routing/test_otpm_default_reservation_fraction.py new file mode 100644 index 00000000000..589a2535a03 --- /dev/null +++ b/tests/integration/routing/test_otpm_default_reservation_fraction.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import dataclasses +import threading +import uuid +from collections.abc import Callable, Iterator +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from queue import SimpleQueue +from typing import Final + +import httpx +import pytest +from integration._support.client import JSON_OBJECT, gateway_from_environment +from integration._support.openai_wire import chat_reply +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._cache_control_marks_support import owned_config +from integration.routing.test_payment_required_mapping import OPENAI_MODEL, is_model_info_probe, model_list_reply + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +GROUPS: Final = ("otpm-fraction-first", "otpm-fraction-retry") +OTPM_LIMIT: Final = 1000 +FRACTION: Final = 0.5 +DEFAULT_RESERVATION: Final = int(OTPM_LIMIT * FRACTION) + + +@dataclass(frozen=True, slots=True) +class Rig: + owned: OwnedProxy + upstream: Wire + gates: SimpleQueue[threading.Event] + + +@dataclass(frozen=True, slots=True) +class Admissions: + held_forwarded_max_tokens: bool + statuses: tuple[int, int, int] + one_minute: bool + + +def _reply_for(gates: SimpleQueue[threading.Event]) -> Callable[[Request], Reply]: + def reply(request: Request) -> Reply: + if is_model_info_probe(request): + return model_list_reply() + if JSON_OBJECT.validate_json(request.body).get("stream") is True: + return dataclasses.replace( + chat_reply("chatcmpl-held", OPENAI_MODEL, "held", stream=True), gate_before_first=gates.get(timeout=60) + ) + return chat_reply("chatcmpl-capped", OPENAI_MODEL, "capped", stream=False) + + return reply + + +def _config(directory: Path, upstream: Wire) -> Path: + return owned_config( + directory, + [ + { + "model_name": group, + "litellm_params": { + "model": OPENAI_MODEL, + "api_base": upstream.url + "/v1", + "api_key": "synthetic-openai-key", + "otpm": OTPM_LIMIT, + }, + "model_info": {"id": group}, + } + for group in GROUPS + ], + litellm_settings={"cache": False}, + router_settings={"optional_pre_call_checks": ["enforce_model_rate_limits"], "num_retries": 0}, + ) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("otpm-fraction") + gates: Final[SimpleQueue[threading.Event]] = SimpleQueue() + with ( + gateway_from_environment() as gateway, + wire_server(_reply_for(gates)) as upstream, + owned_proxy_process( + gateway, + directory, + {"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION": str(FRACTION)}, + config=_config(directory, upstream), + ) as owned, + ): + yield Rig(owned, upstream, gates) + + +def _chat(rig: Rig, group: str, **params: int | bool) -> httpx.Response: + return rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": group, "messages": [{"role": "user", "content": f"fraction {uuid.uuid4().hex}"}], **params}, + ) + + +def _await_forwarded_chat(upstream: Wire) -> Request: + request: Final = upstream.received.get(timeout=60) + return _await_forwarded_chat(upstream) if is_model_info_probe(request) else request + + +def _utc_minute() -> str: + return datetime.now(UTC).strftime("%Y-%m-%d %H:%M") + + +def _admissions_beside_a_held_stream(rig: Rig, group: str) -> Admissions: + release: Final = threading.Event() + rig.gates.put(release) + started: Final = _utc_minute() + with ThreadPoolExecutor(max_workers=1) as pool: + held: Final = pool.submit(_chat, rig, group, stream=True) + try: + forwarded: Final = JSON_OBJECT.validate_json(_await_forwarded_chat(rig.upstream).body) + over: Final = _chat(rig, group, max_tokens=OTPM_LIMIT - DEFAULT_RESERVATION + 1) + fits: Final = _chat(rig, group, max_tokens=OTPM_LIMIT - DEFAULT_RESERVATION) + ended: Final = _utc_minute() + finally: + release.set() + return Admissions( + held_forwarded_max_tokens="max_tokens" in forwarded, + statuses=(over.status_code, fits.status_code, held.result(timeout=60).status_code), + one_minute=started == ended, + ) + + +def test_reservation_fraction_env_var_sets_what_a_request_without_max_tokens_holds(rig: Rig) -> None: + first: Final = _admissions_beside_a_held_stream(rig, GROUPS[0]) + admissions: Final = first if first.one_minute else _admissions_beside_a_held_stream(rig, GROUPS[1]) + + assert admissions == Admissions(held_forwarded_max_tokens=False, statuses=(429, 200, 200), one_minute=True) diff --git a/tests/unit/litellm_core_utils/test_env_utils.py b/tests/unit/litellm_core_utils/test_env_utils.py index 1ddd172e274..89b091e47c2 100644 --- a/tests/unit/litellm_core_utils/test_env_utils.py +++ b/tests/unit/litellm_core_utils/test_env_utils.py @@ -6,6 +6,7 @@ import pytest from litellm._logging import verbose_logger from litellm.litellm_core_utils.env_utils import ( get_env_float, + get_env_float_in_range, get_env_float_or_none, get_env_int, get_env_int_or_none, @@ -94,3 +95,12 @@ def test_unparsable_stdout_log_limit_does_not_break_logging( assert "first line" in messages assert "second line" in messages assert len([message for message in messages if raw in message]) == 1 + + +@pytest.mark.parametrize(("raw", "expected"), [("0.5", 0.5), ("1", 1.0), ("4", 0.25), ("0", 0.25), ("half", 0.25)]) +def test_float_in_range_keeps_values_inside_the_range_and_falls_back_otherwise( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: float +) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_FRACTION", raw) + + assert get_env_float_in_range("ENV_UTILS_TEST_FRACTION", 0.25, 0.01, 1.0) == expected diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 99b049f1803..d9c34e95364 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -19,7 +19,14 @@ from unittest.mock import patch _ENV_HELPERS = frozenset( - {"get_env_int", "get_env_float", "get_env_int_in_range", "get_env_int_or_none", "get_env_float_or_none"} + { + "get_env_int", + "get_env_float", + "get_env_int_in_range", + "get_env_float_in_range", + "get_env_int_or_none", + "get_env_float_or_none", + } ) diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index 05921aa70b3..e5ff4531f84 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -3,6 +3,8 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits). """ import asyncio +from collections.abc import Awaitable, Callable, Mapping +from datetime import datetime from typing import Final import pytest @@ -10,13 +12,16 @@ import pytest import litellm from litellm import Router from litellm.caching.dual_cache import DualCache +from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( ITPM_CACHE_KEY, ITPM_RESERVED_KEY, OTPM_CACHE_KEY, OTPM_RESERVED_KEY, + _explicit_max_tokens, _reservation_value, - _resolve_max_tokens, async_io_token_pre_call_check, async_io_token_reconcile_success, build_io_token_rate_limit_headers, @@ -30,7 +35,16 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, ) +from litellm.types.router import RouterCacheEnum from litellm.types.utils import ModelResponse, Usage +from litellm.utils import get_utc_datetime + +MANTLE_OPUS = "bedrock_mantle/anthropic.claude-opus-4-7" +MANTLE_OPUS_MAX_OUTPUT = litellm.get_model_info(model=MANTLE_OPUS)["max_output_tokens"] +UNMAPPED_MODEL = "openai/litellm-io-unmapped-model" +UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT = 4096 +AZURE_BASE_MODEL = "azure/gpt-5" +AZURE_BASE_MODEL_MAX_OUTPUT = litellm.get_model_info(model=AZURE_BASE_MODEL)["max_output_tokens"] class TestIOTokenRateLimitHelpers: @@ -48,13 +62,23 @@ class TestIOTokenRateLimitHelpers: # A real non-zero estimate is reserved as-is. assert _reservation_value(42, 100) == 42 - def test_resolve_max_tokens_respects_explicit_zero(self): - deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}} - # An explicit max_tokens=0 is honored, not replaced by the model default. - assert _resolve_max_tokens({"max_tokens": 0}, deployment) == 0 - # max_completion_tokens is the fallback only when max_tokens is absent. - assert _resolve_max_tokens({"max_completion_tokens": 12}, deployment) == 12 - assert _resolve_max_tokens({"max_output_tokens": 9}, deployment) == 9 + @pytest.mark.parametrize( + "request_kwargs, expected", + [ + ({"max_tokens": 0}, 0), + ({"max_completion_tokens": 12}, 12), + ({"max_output_tokens": 9}, 9), + ({"max_tokens": 5, "max_completion_tokens": 12}, 5), + ({"max_tokens": None, "max_output_tokens": 9}, 9), + ({"max_tokens": [100]}, None), + ({"messages": []}, None), + (None, None), + ], + ) + def test_explicit_max_tokens_reads_every_output_cap_key( + self, request_kwargs: dict[str, object] | None, expected: int | None + ): + assert _explicit_max_tokens(request_kwargs) == expected def test_build_io_token_rate_limit_headers(self): headers = build_io_token_rate_limit_headers( @@ -386,6 +410,244 @@ class TestModelRateLimitingCheckIOTokens: normal_output_otpm_key = normal_output_kwargs["metadata"][OTPM_CACHE_KEY] assert await dual_cache.async_get_cache(key=normal_output_otpm_key) == 5 + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + @pytest.mark.parametrize( + "model, otpm_limit", + [ + (MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT // 2), + (MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT - 1), + (UNMAPPED_MODEL, 1000), + (UNMAPPED_MODEL, 3), + ], + ) + async def test_request_without_max_tokens_is_admitted_below_model_max_output( + self, model: str, otpm_limit: int, use_sync_check: bool + ): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": model, "otpm": otpm_limit}, + "model_info": {"id": f"io-no-max-tokens-{otpm_limit}"}, + "model_name": "no-max-tokens", + } + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + + reserved: Final = request_kwargs["metadata"][OTPM_RESERVED_KEY] + assert isinstance(reserved, int) + assert 0 < reserved < otpm_limit + assert await dual_cache.async_get_cache(key=request_kwargs["metadata"][OTPM_CACHE_KEY]) == reserved + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model, otpm_limit", + [(MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT), (UNMAPPED_MODEL, UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT)], + ) + async def test_concurrent_requests_without_max_tokens_share_otpm_without_overshoot( + self, model: str, otpm_limit: int + ): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": model, "otpm": otpm_limit}, + "model_info": {"id": "io-no-max-tokens-race-id"}, + "model_name": "no-max-tokens", + } + + async def _attempt() -> dict[str, object] | None: + request_kwargs: Final = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + try: + await check.async_pre_call_check(deployment) + except litellm.RateLimitError: + return None + return request_kwargs["metadata"] + + admitted: Final = [m for m in await asyncio.gather(*[_attempt() for _ in range(8)]) if m is not None] + otpm_key: Final = admitted[0][OTPM_CACHE_KEY] + + assert len(admitted) == round(1 / NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) + assert {m[OTPM_CACHE_KEY] for m in admitted} == {otpm_key} + assert await dual_cache.async_get_cache(key=otpm_key) == sum(m[OTPM_RESERVED_KEY] for m in admitted) + assert await dual_cache.async_get_cache(key=otpm_key) <= otpm_limit + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model, model_max_output", + [(MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT), (UNMAPPED_MODEL, UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT)], + ) + async def test_request_without_max_tokens_reserves_model_max_output_when_otpm_allows( + self, model: str, model_max_output: int + ): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": model, "otpm": model_max_output * 10}, + "model_info": {"id": "io-no-max-tokens-roomy-id"}, + "model_name": "roomy", + } + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + + await check.async_pre_call_check(deployment) + + assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == model_max_output + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_base_model_does_not_change_the_default_otpm_reservation(self, use_sync_check: bool): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + otpm_limit: Final = AZURE_BASE_MODEL_MAX_OUTPUT * 10 + + async def _reserved(model_info: dict[str, str]) -> int: + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + deployment: Final = { + "litellm_params": {"model": "azure/litellm-io-custom-deployment", "otpm": otpm_limit}, + "model_info": model_info, + "model_name": "custom-name", + } + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + return request_kwargs["metadata"][OTPM_RESERVED_KEY] + + with_base_model: Final = await _reserved({"id": "io-with-base-model", "base_model": AZURE_BASE_MODEL}) + without_base_model: Final = await _reserved({"id": "io-without-base-model"}) + + assert with_base_model == without_base_model + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_unparseable_max_tokens_leaves_itpm_uncharged(self, use_sync_check: bool): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": MANTLE_OPUS, "itpm": 2000, "otpm": 2000}, + "model_info": {"id": "io-bad-max-tokens-id"}, + "model_name": "opus", + } + + async def _pre_call(request_kwargs: dict[str, object]) -> None: + set_io_token_rate_limit_request_kwargs(request_kwargs) + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + + await _pre_call({"messages": [{"role": "user", "content": "hello there"}], "max_tokens": "abc", "metadata": {}}) + valid_kwargs: Final = { + "messages": [{"role": "user", "content": "hello there"}], + "max_tokens": 1, + "metadata": {}, + } + await _pre_call(valid_kwargs) + + itpm_counter: Final = await dual_cache.async_get_cache(key=valid_kwargs["metadata"][ITPM_CACHE_KEY]) + assert itpm_counter == valid_kwargs["metadata"][ITPM_RESERVED_KEY] + + @pytest.mark.asyncio + async def test_explicit_max_tokens_above_default_share_is_reserved_in_full(self): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": MANTLE_OPUS, "otpm": 100}, + "model_info": {"id": "io-explicit-max-tokens-id"}, + "model_name": "opus", + } + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 100, "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + + await check.async_pre_call_check(deployment) + + assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == 100 + set_io_token_rate_limit_request_kwargs({"messages": [{"role": "user", "content": "hi"}], "metadata": {}}) + with pytest.raises(litellm.RateLimitError): + await check.async_pre_call_check(deployment) + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_deployment_max_tokens_sets_the_reservation_unless_the_request_sets_one(self, use_sync_check: bool): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + deployment: Final = { + "litellm_params": {"model": MANTLE_OPUS, "otpm": 4000, "max_tokens": 100}, + "model_info": {"id": "io-deployment-max-tokens-id"}, + "model_name": "opus", + } + without_cap: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + with_cap: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 300, "metadata": {}} + + for request_kwargs in (without_cap, with_cap): + set_io_token_rate_limit_request_kwargs(request_kwargs) + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + + assert without_cap["metadata"][OTPM_RESERVED_KEY] == 100 + assert with_cap["metadata"][OTPM_RESERVED_KEY] == 300 + + @pytest.mark.asyncio + async def test_deployment_cap_sets_the_reservation_only_through_the_param_its_api_sends(self): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + chat: Final = {"messages": [{"role": "user", "content": "hello"}]} + responses: Final = {"input": "hello"} + + async def reserved(deployment_cap: Mapping[str, int], request_shape: Mapping[str, object]) -> int: + request_kwargs: Final = {**request_shape, "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + await check.async_pre_call_check( + { + "litellm_params": {"model": MANTLE_OPUS, "otpm": 4000, **deployment_cap}, + "model_info": {"id": f"io-sent-cap-{sorted(deployment_cap)}-{sorted(request_shape)}"}, + "model_name": "opus", + } + ) + return request_kwargs["metadata"][OTPM_RESERVED_KEY] + + chat_default: Final = await reserved({}, chat) + responses_default: Final = await reserved({}, responses) + + assert chat_default == responses_default > 100 + assert await reserved({"max_tokens": 100}, chat) == 100 + assert await reserved({"max_completion_tokens": 100}, chat) == 100 + assert await reserved({"max_output_tokens": 100}, chat) == chat_default + assert await reserved({"max_output_tokens": 100}, responses) == 100 + assert await reserved({"max_tokens": 100}, responses) == responses_default + assert await reserved({"max_completion_tokens": 100}, responses) == responses_default + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_deployment_cap_above_the_default_reserves_the_default(self, use_sync_check: bool): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + otpm_limit: Final = UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT * 5 + + async def reserved(deployment_cap: Mapping[str, int]) -> int: + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + deployment: Final = { + "litellm_params": {"model": UNMAPPED_MODEL, "otpm": otpm_limit, **deployment_cap}, + "model_info": {"id": f"io-deployment-cap-above-default-{deployment_cap}"}, + "model_name": "unmapped", + } + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + return request_kwargs["metadata"][OTPM_RESERVED_KEY] + + default: Final = await reserved({}) + + assert await reserved({"max_tokens": default + 1}) == default + assert await reserved({"max_tokens": otpm_limit * 2}) == default + def test_sync_io_pre_call_reserves_and_reconciles(self): from litellm.utils import get_utc_datetime @@ -415,9 +677,9 @@ class TestModelRateLimitingCheckIOTokens: "standard_logging_object": { "model_id": "io-sync-id", "hidden_params": {"litellm_model_name": "bedrock_mantle/anthropic.claude-opus-4-7"}, - "metadata": dict(request_kwargs["metadata"]), + "metadata": {}, }, - "metadata": request_kwargs["metadata"], + "litellm_params": {"metadata": request_kwargs["metadata"]}, } response = ModelResponse( choices=[{"message": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": "stop"}], @@ -709,6 +971,32 @@ class TestModelRateLimitingCheckIOTokens: assert await dual_cache.async_get_cache(key=itpm_key) == 4 + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_reconcile", [False, True]) + async def test_reconcile_charges_usage_above_the_reservation_to_the_minute_it_finished_in( + self, use_sync_reconcile: bool + ): + dual_cache: Final = DualCache() + counter: Final = "global_router:io-late-id:bedrock_mantle/test:otpm" + reserved_minute_key: Final = f"{counter}:99-99" + await dual_cache.async_increment_cache(key=reserved_minute_key, value=10, ttl=60) + kwargs: Final = {"metadata": {ITPM_RESERVED_KEY: 0, OTPM_RESERVED_KEY: 10, OTPM_CACHE_KEY: reserved_minute_key}} + response: Final = ModelResponse( + choices=[{"message": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": "stop"}], + usage=Usage(prompt_tokens=4, completion_tokens=25, total_tokens=29), + ) + minute_before: Final = get_utc_datetime().strftime("%H-%M") + + if use_sync_reconcile: + io_token_reconcile_success(dual_cache, kwargs, response) + else: + await async_io_token_reconcile_success(dual_cache, kwargs, response) + + finished_minutes: Final = {minute_before, get_utc_datetime().strftime("%H-%M")} + charged: Final = [await dual_cache.async_get_cache(key=f"{counter}:{minute}") for minute in finished_minutes] + assert await dual_cache.async_get_cache(key=reserved_minute_key) == 10 + assert sum(value or 0 for value in charged) == 25 - 10 + @pytest.mark.asyncio async def test_reconcile_missing_usage_keeps_reservation(self): dual_cache = DualCache() @@ -799,16 +1087,16 @@ class TestModelRateLimitingCheckIOTokens: assert await dual_cache.async_get_cache(key=itpm_key) == 4 - def test_sync_reconcile_anthropic_dict_usage(self): + def test_sync_reconcile_anthropic_dict_usage_counts_uncached_input_once(self): dual_cache = DualCache() itpm_key = "global_router:io-anthropic:bedrock_mantle/test:itpm:00-00" otpm_key = "global_router:io-anthropic:bedrock_mantle/test:otpm:00-00" - dual_cache.set_cache(key=itpm_key, value=6, ttl=60) + dual_cache.set_cache(key=itpm_key, value=10, ttl=60) dual_cache.set_cache(key=otpm_key, value=4, ttl=60) kwargs = { "metadata": { - ITPM_RESERVED_KEY: 6, + ITPM_RESERVED_KEY: 10, OTPM_RESERVED_KEY: 4, ITPM_CACHE_KEY: itpm_key, OTPM_CACHE_KEY: otpm_key, @@ -816,12 +1104,17 @@ class TestModelRateLimitingCheckIOTokens: } response = { "type": "message", - "usage": {"input_tokens": 3, "output_tokens": 2, "cache_read_input_tokens": 1}, + "usage": { + "input_tokens": 3, + "output_tokens": 2, + "cache_read_input_tokens": 1, + "cache_creation_input_tokens": 4, + }, } io_token_reconcile_success(dual_cache, kwargs, response) - assert dual_cache.get_cache(key=itpm_key) == 2 + assert dual_cache.get_cache(key=itpm_key) == 3 + 4 assert dual_cache.get_cache(key=otpm_key) == 2 @pytest.mark.asyncio @@ -958,7 +1251,214 @@ class TestModelRateLimitingCheckIOTokens: assert current == 0 +ROUTER_IO_MODEL: Final = "anthropic/claude-haiku-5-5" +RouterCall = Callable[[Router, str | Exception], Awaitable[tuple[int, int]]] + + +def _router_with_io_limits() -> Router: + return Router( + model_list=[ + { + "model_name": "limited", + "litellm_params": {"model": ROUTER_IO_MODEL, "api_key": "sk-test", "itpm": 50000, "otpm": 50000}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + num_retries=1, + ) + + +def _current_minute() -> str: + return get_utc_datetime().strftime("%H-%M") + + +def _counter(router: Router, template: RouterCacheEnum, minutes: frozenset[str]) -> int: + deployment: Final = router.model_list[0] + return sum( + router.cache.get_cache( + key=template.value.format( + id=deployment["model_info"]["id"], + model=deployment["litellm_params"]["model"], + current_minute=minute, + ) + ) + or 0 + for minute in minutes + ) + + +async def _io_counters_since(router: Router, first_minute: str) -> tuple[int, int]: + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) + minutes: Final = frozenset({first_minute, _current_minute()}) + return _counter(router, RouterCacheEnum.ITPM, minutes), _counter(router, RouterCacheEnum.OTPM, minutes) + + +async def _sdk_chat(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.acompletion( + model="limited", + messages=[{"role": "user", "content": "hello there"}], + max_tokens=100, + mock_response=mock_response, + ) + return response.usage.prompt_tokens, response.usage.completion_tokens + + +async def _proxy_chat(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.acompletion( + model="limited", + messages=[{"role": "user", "content": "hello there"}], + max_tokens=100, + metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_response, + ) + return response.usage.prompt_tokens, response.usage.completion_tokens + + +async def _proxy_responses(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.aresponses( + model="limited", + input="hello there", + max_output_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_response, + ) + return response.usage.input_tokens, response.usage.output_tokens + + +async def _proxy_messages(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.aanthropic_messages( + model="limited", + messages=[{"role": "user", "content": "hello there"}], + max_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_response, + ) + return response["usage"]["input_tokens"], response["usage"]["output_tokens"] + + +ProxyCallSharingLogging = Callable[[Router, str | Exception, LiteLLMLoggingObject], Awaitable[object]] +PROXY_MESSAGES: Final = [{"role": "user", "content": "hello there"}] + + +def _logging_obj_shared_across_attempts(original_function: str) -> LiteLLMLoggingObject: + logging_obj, _ = litellm.utils.function_setup( + original_function, + litellm.utils.Rules(), + datetime.now(), + model="limited", + messages=PROXY_MESSAGES, + litellm_call_id="proxy-call", + ) + return logging_obj + + +async def _proxy_chat_sharing_logging( + router: Router, mock_error: str | Exception, logging_obj: LiteLLMLoggingObject +) -> object: + return await router.acompletion( + model="limited", + messages=PROXY_MESSAGES, + max_tokens=100, + metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_error, + litellm_call_id="proxy-call", + litellm_logging_obj=logging_obj, + ) + + +async def _proxy_responses_sharing_logging( + router: Router, mock_error: str | Exception, logging_obj: LiteLLMLoggingObject +) -> object: + return await router.aresponses( + model="limited", + input="hello there", + max_output_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_error, + litellm_call_id="proxy-call", + litellm_logging_obj=logging_obj, + ) + + +async def _proxy_messages_sharing_logging( + router: Router, mock_error: str | Exception, logging_obj: LiteLLMLoggingObject +) -> object: + return await router.aanthropic_messages( + model="limited", + messages=PROXY_MESSAGES, + max_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_error, + litellm_call_id="proxy-call", + litellm_logging_obj=logging_obj, + ) + + class TestRouterIOTokenIntegration: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "call", + [_sdk_chat, _proxy_chat, _proxy_responses, _proxy_messages], + ids=["sdk_chat", "proxy_chat_metadata", "proxy_responses_litellm_metadata", "proxy_messages_litellm_metadata"], + ) + async def test_success_reconciles_io_counters_to_real_usage(self, call: RouterCall): + router: Final = _router_with_io_limits() + first_minute: Final = _current_minute() + + usage: Final = await call(router, "hi") + + assert await _io_counters_since(router, first_minute) == usage + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call", "mock_error"), + [ + ( + _proxy_responses, + litellm.InternalServerError(message="mock 500", llm_provider="anthropic", model=ROUTER_IO_MODEL), + ), + (_proxy_messages, "litellm.InternalServerError"), + ], + ids=["proxy_responses_litellm_metadata", "proxy_messages_litellm_metadata"], + ) + async def test_retried_failure_refunds_each_io_reservation_once( + self, call: RouterCall, mock_error: str | Exception + ): + router: Final = _router_with_io_limits() + first_minute: Final = _current_minute() + + with pytest.raises(litellm.InternalServerError): + await call(router, mock_error) + + assert await _io_counters_since(router, first_minute) == (0, 0) + + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call", "original_function", "mock_error"), + [ + (_proxy_chat_sharing_logging, "acompletion", "litellm.InternalServerError"), + ( + _proxy_responses_sharing_logging, + "aresponses", + litellm.InternalServerError(message="mock 500", llm_provider="anthropic", model=ROUTER_IO_MODEL), + ), + (_proxy_messages_sharing_logging, "anthropic_messages", "litellm.InternalServerError"), + ], + ids=["proxy_chat", "proxy_responses", "proxy_messages"], + ) + async def test_retried_failure_sharing_one_logging_object_refunds_every_attempt( + self, call: ProxyCallSharingLogging, original_function: str, mock_error: str | Exception + ): + router: Final = _router_with_io_limits() + first_minute: Final = _current_minute() + + with pytest.raises(litellm.InternalServerError): + await call(router, mock_error, _logging_obj_shared_across_attempts(original_function)) + + assert await _io_counters_since(router, first_minute) == (0, 0) + @pytest.mark.asyncio async def test_model_group_info_aggregates_io_limits(self): router = Router( From ccc56d27adeeece2a1069bde587c76486ba9ab39 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:04:22 -0700 Subject: [PATCH 22/34] fix(azure): add batch prices to the unversioned o3, o4-mini, o3-mini, gpt-4o and gpt-4o-mini rows (#49182) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 15 +++++++++++++++ model_prices_and_context_window.json | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 96a04c96f44..0799a298942 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6014,11 +6014,14 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -6313,16 +6316,19 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -11303,16 +11309,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_batches": 4e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11411,11 +11420,14 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -11513,16 +11525,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 96a04c96f44..0799a298942 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6014,11 +6014,14 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -6313,16 +6316,19 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -11303,16 +11309,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_batches": 4e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11411,11 +11420,14 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -11513,16 +11525,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", From f32ac94ad1fc3bce79da08752c7bb6a17ebdbfa1 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:30:02 -0700 Subject: [PATCH 23/34] fix(openai): keep the upstream error body on e.response for OpenAI and Azure errors (#49079) * fix(openai): keep the upstream error response on mapped exceptions so e.response.text has the wire body * fix: keep the upstream error body on 429 and 5xx errors RateLimitError, InternalServerError, BadGatewayError, and ServiceUnavailableError rebuilt their response and dropped the upstream body. They now keep the body and exactly the upstream headers, and an unread streaming response no longer raises. * fix: keep a gzip-encoded 429 or 5xx error readable The kept body is already decoded, so the rebuilt response drops content-encoding instead of decoding the body a second time and raising DecodingError. * fix: keep the upstream headers and drop credentials on every mapped provider error The copied error response now keeps every upstream header and carries a request with only the method and the URL, so no provider key, query secret, or prompt reaches e.request. Timeout and APIError get the response too, and azure/ chat, streaming, embedding, and image errors attach it * fix: keep the upstream error body on gateway timeouts and re-raised gateway errors * test(azure): cover the 404, 500, 502, and 504 error bodies and the missing-key placeholder * fix: keep no e.response on timeouts and errors litellm raises itself * fix: keep the upstream elapsed time on error responses and the upstream body on azure_text errors * fix: keep the upstream body without credentials on Azure content filter errors and message-matched OpenAI errors --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/exceptions.py | 93 +++--- .../exception_mapping_utils.py | 57 +++- litellm/llms/azure/azure.py | 12 +- litellm/llms/azure/completion/handler.py | 22 +- litellm/llms/azure/exception_mapping.py | 5 +- litellm/llms/openai/common_utils.py | 9 + litellm/llms/openai/completion/handler.py | 23 +- .../llms/openai/image_variations/handler.py | 16 +- litellm/llms/openai/openai.py | 24 +- .../test_exception_mapping_utils.py | 82 +++++- .../azure/test_azure_exception_mapping.py | 201 +++++++++++++ .../completion/test_completion_handler.py | 24 +- .../llms/openai/image_variations/__init__.py | 0 .../test_openai_image_variations_handler.py | 51 ++++ tests/unit/llms/openai/test_openai.py | 274 +++++++++++++++++- .../proxy/test_common_request_processing.py | 63 ++++ .../test_exception_header_preservation.py | 19 ++ 17 files changed, 895 insertions(+), 80 deletions(-) create mode 100644 tests/unit/llms/openai/image_variations/__init__.py create mode 100644 tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 28211b3dab2..12425c45b01 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,6 +9,7 @@ ## LiteLLM versions of the OpenAI Exception Types +import datetime import enum from collections.abc import Mapping, Sequence from typing import Any, Final @@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None: return None +def _upstream_body(response: object) -> bytes: + if not isinstance(response, httpx.Response): + return b"" + try: + return response.content + except httpx.ResponseNotRead: + return b"" + + +def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request: + if not isinstance(upstream, httpx.Response): + return httpx.Request(method="POST", url=fallback_url) + try: + upstream_request: Final = upstream.request + except RuntimeError: + return httpx.Request(method="POST", url=fallback_url) + return httpx.Request( + method=upstream_request.method, + url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None), + ) + + +def _upstream_elapsed(upstream: object) -> datetime.timedelta | None: + if not isinstance(upstream, httpx.Response): + return None + try: + return upstream.elapsed + except RuntimeError: + return None + + +def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response: + response: Final = httpx.Response( + status_code=status_code, + stream=httpx.ByteStream(_upstream_body(upstream)), + request=_request_without_credentials(upstream, fallback_url), + ) + _ = response.read() + response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None) + elapsed: Final = _upstream_elapsed(upstream) + if elapsed is not None: + response.elapsed = elapsed + return response + + +_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/" + + _MINIMAL_ERROR_RESPONSE: httpx.Response | None = None @@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError): # headers stay reachable on `e.response.headers` for callers that # explicitly want them; only the proxy-supplied `headers=` kwarg # makes it onto `self.headers`. - _response_headers: Final = getattr(response, "headers", None) if response is not None else None self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None # Mirrors FastAPI HTTPException.detail so the same instance can be # serialized through both the ProxyException and HTTPException paths. self.detail = detail if detail is not None else self.message - self.response = httpx.Response( - status_code=429, - headers=_response_headers, - content=response.content if response is not None else None, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime @@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries - _response_headers: Final = getattr(response, "headers", None) if response is not None else None - self.response = httpx.Response( - status_code=self.status_code, - headers=_response_headers, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs @@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries - _response_headers: Final = getattr(response, "headers", None) if response is not None else None - self.response = httpx.Response( - status_code=self.status_code, - headers=_response_headers, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime @@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries - _response_headers: Final = getattr(response, "headers", None) if response is not None else None - self.response = httpx.Response( - status_code=self.status_code, - headers=_response_headers, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime @@ -908,6 +924,7 @@ class APIError(openai.APIError): max_retries: int | None = None, num_retries: int | None = None, body: object | None = None, + response: httpx.Response | None = None, ): self.status_code = status_code self.message = f"litellm.APIError: {message}" @@ -916,6 +933,8 @@ class APIError(openai.APIError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + if response is not None: + self.response = response if request is None: request = httpx.Request(method="POST", url="https://api.openai.com/v1") super().__init__(self.message, request=request, body=body) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 64a95eb5f28..5a27f5446ad 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -37,6 +37,7 @@ from ..exceptions import ( ServiceUnavailableError, Timeout, UnprocessableEntityError, + error_response_without_credentials, ) @@ -305,6 +306,16 @@ def _litellm_proxy_response( ) +def _without_credentials(response: httpx.Response | None) -> httpx.Response | None: + if not isinstance(response, httpx.Response): + return response + return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs") + + +def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None: + return response if response is not None and (response.headers or response.content) else None + + def _map_openai_exception( *, model: str, @@ -315,7 +326,7 @@ def _map_openai_exception( exception_provider: str, extra_information: str, ) -> None: - response: Final = _litellm_proxy_response(original_exception, custom_llm_provider) + response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider)) if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None: raise ContentPolicyViolationError( message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}", @@ -378,6 +389,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, + response=_upstream_reply(response), ) elif ( ("invalid_request_error" in error_str and "content_policy_violation" in error_str) @@ -449,6 +461,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, body=getattr(original_exception, "body", None), + response=_upstream_reply(response), ) elif "Request too large" in error_str: raise RateLimitError( @@ -479,6 +492,7 @@ def _map_openai_exception( model=model, request=_request, litellm_debug_info=extra_information, + response=_upstream_reply(response), ) elif hasattr(original_exception, "status_code"): if original_exception.status_code == 400: @@ -512,6 +526,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, + response=_upstream_reply(response), ) elif original_exception.status_code == 422: raise BadRequestError( @@ -563,6 +578,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, litellm_debug_info=extra_information, exception_status_code=original_exception.status_code, + response=_upstream_reply(response), ) else: raise APIError( @@ -572,6 +588,7 @@ def _map_openai_exception( model=model, request=getattr(original_exception, "request", None), litellm_debug_info=extra_information, + response=_upstream_reply(response), ) else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors @@ -2019,6 +2036,7 @@ def _map_azure_exception( exception_provider: str, extra_information: str, ) -> None: + response: Final = _without_credentials(getattr(original_exception, "response", None)) message = get_error_message(error_obj=original_exception) if message is None: if hasattr(original_exception, "message"): @@ -2054,7 +2072,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif "DeploymentNotFound" in error_str: raise NotFoundError( @@ -2062,7 +2080,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error( error_str @@ -2076,6 +2094,7 @@ def _map_azure_exception( model=model, extra_information=extra_information, original_exception=original_exception, + response=response, ) elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str: helpful_message: Final = ( @@ -2094,7 +2113,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif getattr(original_exception, "status_code", None) in (None, 400) and ( @@ -2106,7 +2125,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400): @@ -2115,7 +2134,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str: @@ -2124,7 +2143,7 @@ def _map_azure_exception( llm_provider=custom_llm_provider, model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif "Connection error" in error_str: raise APIConnectionError( @@ -2140,7 +2159,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif original_exception.status_code == 401: @@ -2149,7 +2168,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 404: raise NotFoundError( @@ -2157,7 +2176,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 408: raise Timeout( @@ -2165,6 +2184,7 @@ def _map_azure_exception( model=model, litellm_debug_info=extra_information, llm_provider="azure", + response=_upstream_reply(response), ) elif original_exception.status_code == 422: raise BadRequestError( @@ -2172,7 +2192,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 429: raise RateLimitError( @@ -2180,7 +2200,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 502: raise BadGatewayError( @@ -2188,7 +2208,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 503: raise ServiceUnavailableError( @@ -2196,7 +2216,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 504: # gateway timeout error raise Timeout( @@ -2205,6 +2225,7 @@ def _map_azure_exception( litellm_debug_info=extra_information, llm_provider="azure", exception_status_code=original_exception.status_code, + response=_upstream_reply(response), ) else: raise APIError( @@ -2214,6 +2235,7 @@ def _map_azure_exception( litellm_debug_info=extra_information, model=model, request=httpx.Request(method="POST", url="https://openai.com/"), + response=_upstream_reply(response), ) else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors @@ -2589,17 +2611,20 @@ def exception_type( model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, + response=_upstream_reply( + _without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)) + ), ) if ( custom_llm_provider == "litellm_proxy" ): # handle special case where calling litellm proxy + exception str contains error message extract_and_raise_litellm_exception( - response=_litellm_proxy_response(mappable_exception, custom_llm_provider), + response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)), error_str=error_str, model=model, custom_llm_provider=custom_llm_provider, - body=getattr(original_exception, "body", None), + body=getattr(mappable_exception, "body", None), ) if ( custom_llm_provider == "openai" diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index a1ba8de6d7f..69ffac5d6f9 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -42,6 +42,7 @@ from ..openai.common_utils import ( build_output_token_limit_response, is_output_token_limit_error, model_dump_provider_response, + upstream_error_response, ) from .common_utils import ( AzureOpenAIError, @@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): raise AzureOpenAIError( status_code=status_code, message=str(e), + response=upstream_error_response(e), headers=error_headers, body=error_body, ) @@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): raise AzureOpenAIError( status_code=status_code, message=message, + response=upstream_error_response(e), headers=error_headers, body=error_body, ) @@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) error_text = error_response.text - raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) async def make_async_azure_httpx_request( self, @@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except Exception as e: error_code: Final[object] = getattr(e, "status_code", None) if error_code is not None: - raise AzureOpenAIError(status_code=error_code, message=str(e)) + raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e)) else: raise AzureOpenAIError(status_code=500, message=str(e)) diff --git a/litellm/llms/azure/completion/handler.py b/litellm/llms/azure/completion/handler.py index 34e51eaf1ec..c2744e7f750 100644 --- a/litellm/llms/azure/completion/handler.py +++ b/litellm/llms/azure/completion/handler.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse +from ...openai.common_utils import upstream_error_response from ...openai.completion.transformation import OpenAITextCompletionConfig from ..common_utils import AzureOpenAIError, BaseAzureLLM @@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=str(e), + headers=error_headers, + response=upstream_error_response(e), + ) async def acompletion( self, @@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=str(e), + headers=error_headers, + response=upstream_error_response(e), + ) def streaming( self, @@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=str(e), + headers=error_headers, + response=upstream_error_response(e), + ) diff --git a/litellm/llms/azure/exception_mapping.py b/litellm/llms/azure/exception_mapping.py index f7146a36384..6eb5161f7e4 100644 --- a/litellm/llms/azure/exception_mapping.py +++ b/litellm/llms/azure/exception_mapping.py @@ -1,5 +1,7 @@ from typing import Final +import httpx + from litellm.exceptions import ContentPolicyViolationError @@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping: model: str, extra_information: str, original_exception: Exception, + response: httpx.Response | None, ) -> ContentPolicyViolationError: """ Create a content policy violation error @@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping: llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, provider_specific_fields={ # Preserve legacy key for backward compatibility. "innererror": inner_error, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 6c416124d1a..aa14a985afe 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -29,6 +29,7 @@ if TYPE_CHECKING: from aiohttp import ClientSession import litellm +from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model @@ -148,6 +149,14 @@ class OpenAIError(BaseLLMException): ) +def upstream_error_response(error: Exception) -> httpx.Response | None: + if not isinstance(error, (openai.APIStatusError, httpx.HTTPStatusError)): + return None + return error_response_without_credentials( + error.response.status_code, error.response, "https://docs.litellm.ai/docs" + ) + + _DECODED_JSON: Final = TypeAdapter(object) diff --git a/litellm/llms/openai/completion/handler.py b/litellm/llms/openai/completion/handler.py index 278427e8d77..8bfdd536f90 100644 --- a/litellm/llms/openai/completion/handler.py +++ b/litellm/llms/openai/completion/handler.py @@ -10,7 +10,7 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse from litellm.utils import ProviderConfigManager -from ..common_utils import BaseOpenAILLM, OpenAIError +from ..common_utils import BaseOpenAILLM, OpenAIError, upstream_error_response from ..workload_identity import build_async_openai_client, build_openai_client from .transformation import OpenAITextCompletionConfig @@ -164,7 +164,12 @@ class OpenAITextCompletion(BaseLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) async def acompletion( self, @@ -220,7 +225,12 @@ class OpenAITextCompletion(BaseLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) def streaming( self, @@ -261,7 +271,12 @@ class OpenAITextCompletion(BaseLLM): error_response: object = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) streamwrapper: Final = CustomStreamWrapper( completion_stream=response, model=model, diff --git a/litellm/llms/openai/image_variations/handler.py b/litellm/llms/openai/image_variations/handler.py index 01f01d3de1d..504dfb50015 100644 --- a/litellm/llms/openai/image_variations/handler.py +++ b/litellm/llms/openai/image_variations/handler.py @@ -14,7 +14,7 @@ from litellm.utils import ProviderConfigManager from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, upstream_error_response from ..workload_identity import build_async_openai_client, build_openai_client @@ -129,7 +129,12 @@ class OpenAIImageVariationsHandler: error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) def image_variations( self, @@ -245,4 +250,9 @@ class OpenAIImageVariationsHandler: error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index af2e85c5f20..e0a23d6c99f 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -64,6 +64,7 @@ from .common_utils import ( is_openai_backed_api_base, is_output_token_limit_error, model_dump_provider_response, + upstream_error_response, ) from .workload_identity import ( build_async_openai_client, @@ -983,6 +984,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError( status_code=status_code, message=error_text, + response=upstream_error_response(e), headers=error_headers, body=error_body, ) @@ -1110,6 +1112,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError( status_code=status_code, message=message, + response=upstream_error_response(e), headers=error_headers, body=exception_body, ) @@ -1282,6 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError( status_code=getattr(e, "status_code", 500), message=str(e), + response=upstream_error_response(e), headers=error_headers, body=exception_body, ) @@ -1409,7 +1413,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) def embedding( self, @@ -1499,7 +1508,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) async def aimage_generation( self, @@ -1666,7 +1680,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): original_response=str(e), ) if hasattr(e, "status_code"): - raise OpenAIError(status_code=getattr(e, "status_code", 500), message=str(e)) + raise OpenAIError( + status_code=getattr(e, "status_code", 500), + message=str(e), + response=upstream_error_response(e), + ) else: raise OpenAIError(status_code=500, message=str(e)) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 19e5387da9a..88501a23235 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -24,6 +24,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.proxy.common_utils.openai_error_payload import openai_error_type from litellm.types.utils import LlmProviders @@ -54,8 +55,8 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st assert caught.value.code == code assert caught.value.body == body assert caught.value.status_code == 400 - assert caught.value.response is response - assert caught.value.request is request + assert caught.value.response.status_code == 400 + assert caught.value.request.url == request.url assert caught.value.response.headers["x-request-id"] == "cyber-policy-request" @@ -2940,3 +2941,80 @@ def test_transport_timeout_maps_to_timeout( assert exc_info.value.status_code == 408 assert f"Request timed out. Error_str: {reason}" in str(exc_info.value) + + +def test_bedrock_server_error_keeps_the_aws_body_without_the_signed_request() -> None: + body: Final = '{"message":"Bedrock is unable to process your request."}' + signed_request: Final = httpx.Request( + "POST", + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-opus-5-5/converse", + headers={"authorization": "AWS4-HMAC-SHA256 Credential=AKIDSIGNED/20261029/us-east-1/bedrock/aws4_request"}, + content=b'{"messages":[{"role":"user","content":[{"text":"hi"}]}]}', + ) + upstream: Final = httpx.Response( + 500, content=body, headers={"x-amzn-requestid": "req-aws"}, request=signed_request + ) + + with pytest.raises(litellm.ServiceUnavailableError) as raised: + exception_type( + model="bedrock/anthropic.claude-opus-5-5", + original_exception=BedrockError(status_code=500, message=body, response=upstream), + custom_llm_provider="bedrock", + ) + + assert raised.value.response.text == body + assert raised.value.response.headers["x-amzn-requestid"] == "req-aws" + assert "AKIDSIGNED" not in str(raised.value.request.headers) + assert raised.value.request.content == b"" + + +@pytest.mark.parametrize("provider", ("gemini", "vertex_ai")) +def test_gemini_server_error_response_carries_the_upstream_error_body(provider: str) -> None: + body: Final = '{"error":{"code":500,"message":"Internal error encountered.","status":"INTERNAL"}}' + + with pytest.raises(litellm.InternalServerError) as raised: + exception_type( + model="gemini-3.1-pro", + original_exception=VertexAIError(status_code=500, message=body, headers=None), + custom_llm_provider=provider, + ) + + assert raised.value.response.status_code == 500 + assert raised.value.response.json() == json.loads(body) + + +@pytest.mark.parametrize( + ("status_code", "message", "mapped_error"), + ( + (500, "The server had an error processing your request. Sorry about that!", litellm.InternalServerError), + (520, "Web server is returning an unknown error", litellm.InternalServerError), + (504, "A timeout occurred", litellm.Timeout), + (500, "Mistral API raised a streaming error", litellm.APIError), + ), +) +def test_openai_error_matched_by_its_message_keeps_the_upstream_body( + status_code: int, message: str, mapped_error: type[Exception] +) -> None: + body: Final = json.dumps({"error": {"message": message, "type": "server_error", "param": None, "code": None}}) + sent_request: Final = httpx.Request( + "POST", + "https://api.openai.com/v1/chat/completions", + headers={"authorization": "Bearer sk-openai-secret"}, + content=b'{"messages":[{"role":"user","content":"hi"}]}', + ) + upstream: Final = httpx.Response( + status_code, content=body, headers={"x-request-id": "req-openai"}, request=sent_request + ) + + with pytest.raises(mapped_error) as raised: + exception_type( + model="gpt-5.6", + original_exception=openai.APIStatusError(message, response=upstream, body=json.loads(body)), + custom_llm_provider="openai", + ) + + error_response: Final = getattr(raised.value, "response") + assert error_response.text == body + assert error_response.headers["x-request-id"] == "req-openai" + assert "sk-openai-secret" not in str(error_response.request.headers) + assert error_response.request.content == b"" diff --git a/tests/unit/llms/azure/test_azure_exception_mapping.py b/tests/unit/llms/azure/test_azure_exception_mapping.py index cbbf70c691b..018b499f401 100644 --- a/tests/unit/llms/azure/test_azure_exception_mapping.py +++ b/tests/unit/llms/azure/test_azure_exception_mapping.py @@ -1,3 +1,4 @@ +import datetime import json from collections.abc import Awaitable, Callable, Mapping from typing import Final @@ -12,6 +13,7 @@ import respx import litellm from litellm.exceptions import BadRequestError, ContentPolicyViolationError, ContextWindowExceededError from litellm.litellm_core_utils.exception_mapping_utils import exception_type +from litellm.llms.azure.common_utils import AzureOpenAIError @respx.mock @@ -632,3 +634,202 @@ def test_azure_content_policy_wins_over_context_length_code(monkeypatch: pytest. error: Final = _azure_chat_error(monkeypatch, _azure_error_response(400, policy_error)) assert type(error) is ContentPolicyViolationError + + +_AZURE_REFUSAL_BODY: Final = '{"error":{"message":"azure refused","type":"upstream_refusal","code":"refused"}}' +_AZURE_DEPLOYMENT_NOT_FOUND_BODY: Final = '{"error":{"code":"DeploymentNotFound","message":"no such deployment"}}' +_AZURE_INTERNAL_ERROR_BODY: Final = '{"error":{"code":"InternalServerError","message":"Internal server error"}}' +_AZURE_CONTENT_FILTER_BODY: Final = json.dumps( + { + "error": { + "message": "The response was filtered due to the prompt triggering Azure OpenAI's content management policy.", + "type": None, + "param": "prompt", + "code": "content_filter", + "status": 400, + "innererror": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_result": {"violence": {"filtered": True, "severity": "high"}}, + }, + } + } +) +_AZURE_KEY: Final = "azure-provider-secret" +_AZURE_BASE: Final = "https://azure-upstream.openai.azure.com" +_AZURE_ARGS: Final = {"api_base": _AZURE_BASE, "api_version": "2024-10-21", "api_key": _AZURE_KEY, "max_retries": 0} + + +async def _azure_completion() -> None: + litellm.completion(model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], **_AZURE_ARGS) + + +async def _azure_completion_stream() -> None: + list( + litellm.completion( + model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], stream=True, **_AZURE_ARGS + ) + ) + + +async def _azure_acompletion() -> None: + await litellm.acompletion(model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], **_AZURE_ARGS) + + +async def _azure_acompletion_stream() -> None: + stream: Final = await litellm.acompletion( + model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], stream=True, **_AZURE_ARGS + ) + async for _ in stream: + pass + + +async def _azure_embedding() -> None: + litellm.embedding(model="azure/text-embedding-3-large", input=["hi"], **_AZURE_ARGS) + + +async def _azure_aembedding() -> None: + await litellm.aembedding(model="azure/text-embedding-3-large", input=["hi"], **_AZURE_ARGS) + + +async def _azure_image_generation() -> None: + litellm.image_generation(model="azure/gpt-image-2", prompt="a cat", **_AZURE_ARGS) + + +async def _azure_text_completion() -> None: + litellm.text_completion(model="azure_text/gpt-35-turbo-instruct", prompt="hi", **_AZURE_ARGS) + + +async def _azure_text_completion_stream() -> None: + list(litellm.text_completion(model="azure_text/gpt-35-turbo-instruct", prompt="hi", stream=True, **_AZURE_ARGS)) + + +async def _azure_atext_completion() -> None: + await litellm.atext_completion(model="azure_text/gpt-35-turbo-instruct", prompt="hi", **_AZURE_ARGS) + + +async def _azure_atext_completion_stream() -> None: + stream: Final = await litellm.atext_completion( + model="azure_text/gpt-35-turbo-instruct", prompt="hi", stream=True, **_AZURE_ARGS + ) + async for _ in stream: + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + ( + _azure_completion, + _azure_completion_stream, + _azure_acompletion, + _azure_acompletion_stream, + _azure_embedding, + _azure_aembedding, + _azure_image_generation, + _azure_text_completion, + _azure_text_completion_stream, + _azure_atext_completion, + _azure_atext_completion_stream, + ), + ids=lambda route: route.__name__.removeprefix("_azure_"), +) +@pytest.mark.parametrize( + ("status_code", "body", "mapped_error"), + ( + (400, _AZURE_REFUSAL_BODY, BadRequestError), + (400, _AZURE_CONTENT_FILTER_BODY, BadRequestError), + (403, _AZURE_REFUSAL_BODY, litellm.APIError), + (404, _AZURE_DEPLOYMENT_NOT_FOUND_BODY, litellm.NotFoundError), + (500, _AZURE_INTERNAL_ERROR_BODY, openai.APIError), + (502, _AZURE_REFUSAL_BODY, litellm.BadGatewayError), + (504, _AZURE_REFUSAL_BODY, litellm.Timeout), + ), +) +@respx.mock +async def test_azure_error_keeps_the_upstream_body_without_the_api_key( + route: Callable[[], Awaitable[None]], + status_code: int, + body: str, + mapped_error: type[Exception], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx.post(url__regex=r"https://azure-upstream\.openai\.azure\.com/.*").mock( + return_value=httpx.Response(status_code, content=body, headers={"content-type": "application/json"}) + ) + + with pytest.raises(mapped_error) as raised: + await route() + + error_response: Final = getattr(raised.value, "response") + assert error_response.status_code == status_code + assert error_response.text == body + assert error_response.json() == json.loads(body) + for exposed in (error_response.request, getattr(raised.value, "request")): + assert _AZURE_KEY not in str(exposed.headers) + assert exposed.content == b"" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + ( + _azure_completion, + _azure_completion_stream, + _azure_acompletion, + _azure_acompletion_stream, + _azure_embedding, + _azure_aembedding, + _azure_text_completion, + _azure_atext_completion, + _azure_atext_completion_stream, + ), + ids=lambda route: route.__name__.removeprefix("_azure_"), +) +@respx.mock +async def test_azure_error_response_keeps_the_upstream_elapsed_time( + route: Callable[[], Awaitable[None]], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx.post(url__regex=r"https://azure-upstream\.openai\.azure\.com/.*").mock( + return_value=httpx.Response(400, content=_AZURE_REFUSAL_BODY, headers={"content-type": "application/json"}) + ) + + with pytest.raises(BadRequestError) as raised: + await route() + + assert raised.value.response.elapsed > datetime.timedelta(0) + + +def test_azure_client_side_missing_key_error_keeps_the_empty_placeholder_response() -> None: + missing_key: Final = openai.OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the " + "OPENAI_API_KEY environment variable" + ) + + with pytest.raises(litellm.AuthenticationError) as raised: + exception_type(model="azure/gpt-4.1-mini", original_exception=missing_key, custom_llm_provider="azure") + + assert raised.value.response.status_code == 401 + assert raised.value.response.text == "" + + +@pytest.mark.parametrize( + ("status_code", "message", "mapped_error"), + ( + (500, "azure_client is not an instance of AzureOpenAI or OpenAI", litellm.APIError), + (408, "the deployment did not answer in time", litellm.Timeout), + (504, "the deployment did not answer in time", litellm.Timeout), + ), +) +def test_azure_error_litellm_raises_itself_has_no_upstream_response( + status_code: int, message: str, mapped_error: type[Exception] +) -> None: + with pytest.raises(mapped_error) as raised: + exception_type( + model="azure/gpt-4.1-mini", + original_exception=AzureOpenAIError(status_code=status_code, message=message), + custom_llm_provider="azure", + ) + + assert getattr(raised.value, "response", None) is None diff --git a/tests/unit/llms/openai/completion/test_completion_handler.py b/tests/unit/llms/openai/completion/test_completion_handler.py index 9e2754eef68..628c9bbd783 100644 --- a/tests/unit/llms/openai/completion/test_completion_handler.py +++ b/tests/unit/llms/openai/completion/test_completion_handler.py @@ -93,24 +93,34 @@ async def test_acompletion_forwards_client_headers_to_provider( assert request_headers["x-mycorp-llmcall-id"] == "abc-123" +_REJECTION_BODY: Final = '{"error":{"message":"bad prompt","type":"invalid_request_error"}}' + + @pytest.fixture def rejected_completions_endpoint(): return respx.post("https://api.openai.com/v1/completions").mock( return_value=Response( 400, - json={"error": {"message": "bad prompt", "type": "invalid_request_error"}}, - headers={"x-request-id": "req-1"}, + content=_REJECTION_BODY, + headers={"x-request-id": "req-1", "content-type": "application/json"}, ) ) +def _assert_carries_the_upstream_rejection(rejected: litellm.BadRequestError) -> None: + assert rejected.status_code == 400 + assert rejected.litellm_response_headers["x-request-id"] == "req-1" + assert rejected.response.text == _REJECTION_BODY + assert rejected.response.json() == json.loads(_REJECTION_BODY) + assert rejected.response.headers["x-request-id"] == "req-1" + + @respx.mock def test_completion_surfaces_the_status_and_headers_of_a_rejected_request(rejected_completions_endpoint): with pytest.raises(litellm.BadRequestError) as rejected: text_completion(model="gpt-3.5-turbo-instruct", prompt="hello") - assert rejected.value.status_code == 400 - assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" + _assert_carries_the_upstream_rejection(rejected.value) @respx.mock @@ -118,8 +128,7 @@ def test_streaming_completion_surfaces_the_status_and_headers_of_a_rejected_requ with pytest.raises(litellm.BadRequestError) as rejected: list(text_completion(model="gpt-3.5-turbo-instruct", prompt="hello", stream=True)) - assert rejected.value.status_code == 400 - assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" + _assert_carries_the_upstream_rejection(rejected.value) @respx.mock @@ -130,8 +139,7 @@ async def test_acompletion_surfaces_the_status_and_headers_of_a_rejected_request with pytest.raises(litellm.BadRequestError) as rejected: await atext_completion(model="gpt-3.5-turbo-instruct", prompt="hello") - assert rejected.value.status_code == 400 - assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" + _assert_carries_the_upstream_rejection(rejected.value) @respx.mock diff --git a/tests/unit/llms/openai/image_variations/__init__.py b/tests/unit/llms/openai/image_variations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py b/tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py new file mode 100644 index 00000000000..83ddbec6038 --- /dev/null +++ b/tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py @@ -0,0 +1,51 @@ +import json +from io import BytesIO +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.llms.openai.common_utils import OpenAIError + +_UPSTREAM_ERROR_BODY: Final = '{"error":{"message":"image is not a valid PNG","type":"invalid_request_error"}}' + + +def _upstream_rejection(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 400, + content=_UPSTREAM_ERROR_BODY, + headers={"x-request-id": "req_variation_400", "content-type": "application/json"}, + ) + + +_VARIATION_KWARGS: Final = { + "custom_llm_provider": "openai", + "api_key": "transport-only", + "api_base": "https://upstream.example/v1", + "max_retries": 0, +} + + +async def _request_variation(is_async: bool) -> None: + if is_async: + await litellm.aimage_variation(image=BytesIO(b"image-bytes"), **_VARIATION_KWARGS) + return + litellm.image_variation(image=BytesIO(b"image-bytes"), **_VARIATION_KWARGS) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", (False, True), ids=("sync", "async")) +async def test_rejected_image_variation_carries_the_upstream_error_response( + is_async: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + transport: Final = httpx.MockTransport(_upstream_rejection) + monkeypatch.setattr(litellm, "client_session", httpx.Client(transport=transport)) + monkeypatch.setattr(litellm, "aclient_session", httpx.AsyncClient(transport=transport)) + + with pytest.raises((litellm.BadRequestError, OpenAIError)) as raised: + await _request_variation(is_async) + + assert raised.value.response.text == _UPSTREAM_ERROR_BODY + assert raised.value.response.json() == json.loads(_UPSTREAM_ERROR_BODY) + assert raised.value.response.headers["x-request-id"] == "req_variation_400" diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 5b27a446fb3..3e1cbad9308 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -1,8 +1,9 @@ import asyncio +import gzip import importlib import json import os -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -375,6 +376,277 @@ async def test_acompletion_preserves_public_errors_for_both_http_clients( assert not http_client.is_closed +_UPSTREAM_ERROR_BODY: Final = ( + '{"error":{"message":"upstream rejected the request","type":"invalid_request_error",' + '"param":"response_format.type","code":"invalid_value"}}' +) +_UPSTREAM_BASE: Final = "https://upstream.example/v1" +_UPSTREAM_TRANSPORT: Final = httpx.MockTransport( + lambda _request: httpx.Response( + 400, + content=_UPSTREAM_ERROR_BODY, + headers={"x-request-id": "req_upstream_400", "x-upstream-trace": "trace-7", "content-type": "application/json"}, + ) +) +_MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +def _rejecting_client() -> OpenAI: + return OpenAI( + api_key="transport-only", + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.Client(transport=_UPSTREAM_TRANSPORT), + ) + + +def _rejecting_async_client() -> AsyncOpenAI: + return AsyncOpenAI( + api_key="transport-only", + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.AsyncClient(transport=_UPSTREAM_TRANSPORT), + ) + + +def _upstream(provider: str, model: str) -> dict[str, object]: + return {"model": f"{provider}/{model}", "api_base": _UPSTREAM_BASE, "api_key": "transport-only", "num_retries": 0} + + +async def _completion(provider: str) -> None: + litellm.completion(messages=_MESSAGES, client=_rejecting_client(), **_upstream(provider, "gpt-5.6")) + + +async def _completion_stream(provider: str) -> None: + stream: Final = litellm.completion( + messages=_MESSAGES, client=_rejecting_client(), stream=True, **_upstream(provider, "gpt-5.6") + ) + list(stream) + + +async def _acompletion(provider: str) -> None: + await litellm.acompletion(messages=_MESSAGES, client=_rejecting_async_client(), **_upstream(provider, "gpt-5.6")) + + +async def _acompletion_stream(provider: str) -> None: + stream: Final = await litellm.acompletion( + messages=_MESSAGES, client=_rejecting_async_client(), stream=True, **_upstream(provider, "gpt-5.6") + ) + async for _ in stream: + pass + + +async def _embedding(provider: str) -> None: + litellm.embedding(input=["hi"], client=_rejecting_client(), **_upstream(provider, "text-embedding-3-large")) + + +async def _aembedding(provider: str) -> None: + await litellm.aembedding( + input=["hi"], client=_rejecting_async_client(), **_upstream(provider, "text-embedding-3-large") + ) + + +async def _image_generation(provider: str) -> None: + litellm.image_generation(prompt="a cat", client=_rejecting_client(), **_upstream(provider, "gpt-image-2")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize( + "route", + (_completion, _completion_stream, _acompletion, _acompletion_stream, _embedding, _aembedding, _image_generation), + ids=lambda route: route.__name__.lstrip("_"), +) +async def test_mapped_error_carries_the_upstream_error_response( + provider: str, route: Callable[[str], Awaitable[None]] +) -> None: + with pytest.raises(litellm.BadRequestError) as raised: + await route(provider) + + assert raised.value.response.text == _UPSTREAM_ERROR_BODY + assert raised.value.response.json() == json.loads(_UPSTREAM_ERROR_BODY) + assert raised.value.response.headers["x-request-id"] == "req_upstream_400" + assert raised.value.response.headers["x-upstream-trace"] == "trace-7" + + +_UPSTREAM_SERVER_ERROR_BODY: Final = ( + '{"error":{"message":"The server had an error while processing your request.","type":"server_error",' + '"param":null,"code":null}}' +) +_CHUNKED_UPSTREAM_HEADERS: Final = { + "content-type": "application/json", + "transfer-encoding": "chunked", + "x-request-id": "req_upstream_5xx", +} + + +def _chunked_upstream_client(status_code: int, content_encoding: Literal["gzip"] | None) -> httpx.AsyncClient: + body: Final = _UPSTREAM_SERVER_ERROR_BODY.encode() + encoding_headers: Final = {"content-encoding": content_encoding} if content_encoding else {} + return httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _request: httpx.Response( + status_code, + headers={**_CHUNKED_UPSTREAM_HEADERS, **encoding_headers}, + stream=httpx.ByteStream(gzip.compress(body) if content_encoding else body), + ) + ) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize( + ("status_code", "mapped_error"), + ( + (429, litellm.RateLimitError), + (500, litellm.InternalServerError), + (502, litellm.BadGatewayError), + (503, litellm.ServiceUnavailableError), + ), +) +@pytest.mark.parametrize("content_encoding", (None, "gzip")) +async def test_retryable_error_keeps_the_decoded_upstream_body_and_its_headers( + provider: str, + stream: bool, + status_code: int, + mapped_error: type[Exception], + content_encoding: Literal["gzip"] | None, +) -> None: + client: Final = AsyncOpenAI( + api_key="transport-only", + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=_chunked_upstream_client(status_code, content_encoding), + ) + + with pytest.raises(mapped_error) as raised: + await litellm.acompletion(messages=_MESSAGES, client=client, stream=stream, **_upstream(provider, "gpt-5.6")) + + error_response: Final = getattr(raised.value, "response") + assert error_response.status_code == status_code + assert error_response.text == _UPSTREAM_SERVER_ERROR_BODY + assert error_response.json() == json.loads(_UPSTREAM_SERVER_ERROR_BODY) + assert dict(error_response.headers) == { + **_CHUNKED_UPSTREAM_HEADERS, + **({"content-encoding": content_encoding} if content_encoding else {}), + } + + +_REFUSAL_BODY: Final = '{"error":{"message":"upstream refused","type":"upstream_refusal","param":null,"code":null}}' +_GATEWAY_TIMEOUT_BODY: Final = ( + '{"error":{"message":"litellm.Timeout: APITimeoutError - Request timed out.","type":"invalid_request_error",' + '"param":null,"code":"408"}}' +) +_PROVIDER_KEY: Final = "sk-provider-secret" +_QUERY_SECRET: Final = "query-secret" + + +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize( + ("status_code", "mapped_error", "body"), + ( + (400, litellm.BadRequestError, _REFUSAL_BODY), + (403, litellm.APIError, _REFUSAL_BODY), + (408, litellm.Timeout, _REFUSAL_BODY), + (408, litellm.Timeout, _GATEWAY_TIMEOUT_BODY), + (409, litellm.APIError, _REFUSAL_BODY), + (413, litellm.APIError, _REFUSAL_BODY), + (429, litellm.RateLimitError, _REFUSAL_BODY), + (500, litellm.InternalServerError, _REFUSAL_BODY), + (504, litellm.Timeout, _REFUSAL_BODY), + ), +) +def test_mapped_error_keeps_the_upstream_body_without_the_provider_credentials( + provider: str, status_code: int, mapped_error: type[Exception], body: str +) -> None: + client: Final = OpenAI( + api_key=_PROVIDER_KEY, + base_url=_UPSTREAM_BASE, + default_query={"key": _QUERY_SECRET}, + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport( + lambda _request: httpx.Response( + status_code, content=body, headers={"content-type": "application/json"} + ) + ) + ), + ) + + with pytest.raises(mapped_error) as raised: + litellm.completion( + model=f"{provider}/gpt-5.6", + messages=_MESSAGES, + client=client, + api_base=_UPSTREAM_BASE, + api_key=_PROVIDER_KEY, + num_retries=0, + ) + + error_response: Final = getattr(raised.value, "response") + assert error_response.status_code == status_code + assert error_response.text == body + assert error_response.json() == json.loads(body) + assert str(error_response.request.url) == f"{_UPSTREAM_BASE}/chat/completions" + for exposed in (error_response.request, getattr(raised.value, "request")): + assert _QUERY_SECRET not in str(exposed.url) + assert _PROVIDER_KEY not in str(exposed.headers) + assert exposed.content == b"" + + +def _time_out(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("no reply", request=request) + + +async def _sync_completion_timing_out(provider: str) -> None: + litellm.completion( + model=f"{provider}/gpt-5.6", + messages=_MESSAGES, + client=OpenAI( + api_key=_PROVIDER_KEY, + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(_time_out)), + ), + api_base=_UPSTREAM_BASE, + api_key=_PROVIDER_KEY, + num_retries=0, + ) + + +async def _async_stream_timing_out(provider: str) -> None: + stream: Final = await litellm.acompletion( + model=f"{provider}/gpt-5.6", + messages=_MESSAGES, + stream=True, + client=AsyncOpenAI( + api_key=_PROVIDER_KEY, + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(_time_out)), + ), + api_base=_UPSTREAM_BASE, + api_key=_PROVIDER_KEY, + num_retries=0, + ) + async for _ in stream: + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize("call", (_sync_completion_timing_out, _async_stream_timing_out)) +async def test_timeout_before_any_reply_has_no_upstream_response( + provider: str, call: Callable[[str], Awaitable[None]] +) -> None: + with pytest.raises(litellm.Timeout) as raised: + await call(provider) + + assert getattr(raised.value, "response", None) is None + + @pytest.mark.asyncio async def test_acompletion_streams_text_deltas_over_injected_transport( http_backend: ModuleType, sdk_provider: str, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index aeab7a3d348..c867db2fd02 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -2,6 +2,7 @@ import asyncio import contextvars import copy import datetime +import gzip import json from collections.abc import AsyncIterable, Mapping from types import MappingProxyType, SimpleNamespace @@ -13,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx +import openai import pytest import respx from fastapi import HTTPException, Request, Response, status @@ -11675,6 +11677,67 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" +_UPSTREAM_REJECTION_BODY: Final = b'{"error":{"message":"upstream said no","type":"upstream_rejection"}}' + + +def _upstream_rejection_wire(content_encoding: Literal["gzip"] | None) -> tuple[bytes, dict[str, str]]: + wire_body: Final = gzip.compress(_UPSTREAM_REJECTION_BODY) if content_encoding else _UPSTREAM_REJECTION_BODY + return wire_body, { + "x-request-id": "req_upstream", + "x-upstream-trace": "trace-7", + "content-type": "application/json", + "content-length": str(len(wire_body)), + **({"content-encoding": content_encoding} if content_encoding else {}), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status_code", "mapped_error"), + ( + (400, litellm.BadRequestError), + (429, litellm.RateLimitError), + (503, litellm.ServiceUnavailableError), + ), +) +@pytest.mark.parametrize("content_encoding", (None, "gzip")) +async def test_handle_llm_api_exception_forwards_exactly_the_upstream_headers_of_a_mapped_openai_error( + status_code: int, mapped_error: type[Exception], content_encoding: Literal["gzip"] | None +) -> None: + wire_body, upstream_headers = _upstream_rejection_wire(content_encoding) + transport: Final = httpx.MockTransport( + lambda _request: httpx.Response(status_code, content=wire_body, headers=upstream_headers) + ) + with pytest.raises(mapped_error) as mapped: + await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + num_retries=0, + client=openai.AsyncOpenAI( + api_key="transport-only", + base_url="https://upstream.example/v1", + max_retries=0, + http_client=httpx.AsyncClient(transport=transport), + ), + ) + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with pytest.raises(ProxyException) as forwarded: + await ProxyBaseLLMRequestProcessing(data={}).handle_llm_api_exception( + e=mapped.value, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert forwarded.value.code == str(status_code) + assert "upstream said no" in forwarded.value.message + assert {name: value for name, value in forwarded.value.headers.items() if name.startswith("llm_provider-")} == { + f"llm_provider-{name}": value for name, value in upstream_headers.items() + } + + @pytest.mark.asyncio async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_exception_header_preservation.py b/tests/unit/test_exception_header_preservation.py index 7acacf05226..c5a0868f36d 100644 --- a/tests/unit/test_exception_header_preservation.py +++ b/tests/unit/test_exception_header_preservation.py @@ -13,6 +13,7 @@ from typing import Final import httpx import pytest +import litellm from litellm.exceptions import ( BadRequestError, ContentPolicyViolationError, @@ -418,3 +419,21 @@ class TestBedrockErrorHeaders: headers = get_response_headers(dict(_response_headers)) assert headers.get("llm_provider-x-amzn-requestid") == "req-proxy-500" + + +@pytest.mark.parametrize( + "error_class", + (RateLimitError, ServiceUnavailableError, litellm.BadGatewayError, litellm.InternalServerError), +) +def test_error_built_from_an_unread_streaming_response_keeps_its_headers(error_class: type[Exception]) -> None: + unread: Final = httpx.Response( + 500, + headers={"x-request-id": "req-unread"}, + stream=httpx.ByteStream(b'{"error":"never read"}'), + ) + + error: Final = error_class(message="upstream failed", llm_provider="openai", model="gpt-5.6", response=unread) + + error_response: Final = getattr(error, "response") + assert error_response.text == "" + assert dict(error_response.headers) == {"x-request-id": "req-unread"} From dbcacc7ab7880d5115376cd6c22d22898c50d08f Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:34:57 -0700 Subject: [PATCH 24/34] fix(health): probe only models a partial wildcard serves (#49060) * fix(health): probe only models a partial wildcard serves A partial wildcard health check now probes the cheapest cost-map chat models the wildcard matches, keeps provider route prefixes such as Bedrock converse/ on the probe, and falls back only within the wildcard. A wildcard with no matching cost-map chat model reports an error naming model_info.health_check_model * fix(health): keep the provider-wide probe for an unmatched wildcard, skip retired models, and ignore Bedrock ARNs A partial wildcard with no served cost-map chat model falls back to the provider's cheapest chat models instead of failing, and fails with a message naming model_info.health_check_model only when the provider has none. The cheapest-model picker skips models past their cost-map deprecation_date, and a Bedrock ARN segment is never read as a route prefix. * fix(health): fall back to the provider's cheapest probe when a wildcard's own models fail A partial wildcard now probes up to three of its own cheapest chat models first and then the provider-wide cheapest ones, all from one cost-map scan, so stale or unpriced rows the wildcard matches cannot turn a healthy deployment red. Retired models are read by their resolved cost-map key, both deprecation checks use the UTC date, Bedrock only treats converse, invoke, and region names as route segments, and equal-price ties break by model id * fix(health): cap wildcard health probes at three and rank retired cost-map rows last A partial wildcard now probes two of its own cheapest cost-map chat models and then one provider-wide model, so a failing deployment makes no more calls than before. Rows past their cost-map deprecation date go last instead of being dropped, since some still serve, and the shared cheapest-model picker returns what it returned before. * fix(health): probe a wildcard's own models first, then the same three models main probes A partial wildcard's health check tries its two cheapest own cost-map models, then the provider's three cheapest models main already probes, so a row main shows healthy stays healthy. The deprecation check uses the server's local date, like the Baseten listing * fix(health): keep main's provider-wide probe order so equal-price ties match main --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../health_check_helpers.py | 80 +++- .../litellm_core_utils/llm_request_utils.py | 58 ++- .../test_health_check_helpers.py | 343 +++++++++++++++++- .../test_llm_request_utils.py | 84 ++++- .../proxy/test_health_check_max_tokens.py | 7 +- 5 files changed, 545 insertions(+), 27 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 940730eeb9f..e40299a218f 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,6 +3,8 @@ Helper functions for health check calls. """ import base64 +import itertools +import re from collections.abc import Awaitable, Callable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal @@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = ( ) ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16 +WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2 +WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3 +BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"}) EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( {"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})} ) @@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) +def _provider_base_model(custom_llm_provider: str, model: str) -> str | None: + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + try: + model_info: Final = ProviderConfigManager.get_provider_model_info( + model=None, provider=LlmProviders(custom_llm_provider) + ) + return None if model_info is None else model_info.get_base_model(model) + except (ValueError, NotImplementedError): + return None + + +def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool: + if custom_llm_provider == "bedrock": + from litellm.llms.bedrock.common_utils import get_all_bedrock_regions + + return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions() + base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}") + return base_model is not None and not base_model.startswith(f"{segment}/") + + +def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]: + from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + + provider_prefix: Final = f"{custom_llm_provider}/" + segments: Final = wildcard.removeprefix(provider_prefix).split("/") + route_segments: Final = tuple( + itertools.takewhile( + lambda segment_index: _is_route_segment( + custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :]) + ), + range(len(segments) - 1), + ) + ) + route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments) + model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :]))) + ranked_model_ids: Final = tuple( + (ranked_model.model.removeprefix(provider_prefix), ranked_model) + for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider) + ) + served_model_ids: Final = sorted( + (ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])), + key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]), + ) + served_probes: Final = itertools.islice( + dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids), + WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT, + ) + provider_probes: Final = ( + f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT] + ) + return tuple(dict.fromkeys((*served_probes, *provider_probes))) + + class HealthCheckHelpers: @staticmethod async def ahealth_check_wildcard_models( @@ -99,23 +160,16 @@ class HealthCheckHelpers: litellm_logging_obj: "Logging", ) -> dict: from litellm import acompletion - from litellm.litellm_core_utils.llm_request_utils import ( - pick_cheapest_chat_models_from_llm_provider, - ) - # this is a wildcard model, we need to pick a random model from the provider - cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3) - if len(cheapest_models) == 0: + probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider) + if len(probe_models) == 0: raise Exception( - f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" + f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has " + "no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves" ) - if len(cheapest_models) > 1: - fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list - else: - fallback_models = None - model_params["model"] = cheapest_models[0] + model_params["model"] = probe_models[0] model_params["litellm_logging_obj"] = litellm_logging_obj - model_params["fallbacks"] = fallback_models + model_params["fallbacks"] = list(probe_models[1:]) or None model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16 await acompletion(**model_params) return {} diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index db881aa16c6..9c2ec0b8984 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,6 +1,11 @@ +import datetime from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass from typing import Final, cast +from pydantic import TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict + import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider @@ -151,19 +156,52 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): Returns: list[str]: A list of the n cheapest chat models. """ - if custom_llm_provider not in litellm.models_by_provider: - return [] + return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]] + +@dataclass(frozen=True, slots=True) +class RankedChatModel: + model: str + route_rank: int + cost: float + is_past_deprecation_date: bool + + +def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]: known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) - ranked_rows: Final = ( - (route_rank, cost, model) - for model, cost in _chat_row_costs(known_models, custom_llm_provider) + ranked_models: Final = ( + RankedChatModel( + model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date + ) + for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider) if (route_rank := _route_rank(model, custom_llm_provider)) is not None ) - return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]] + return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost))) -def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: +class _CostMapDeprecation(TypedDict): + deprecation_date: NotRequired[ReadOnly[object]] + + +_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation) + + +def _is_past_deprecation_date(cost_map_key: str) -> bool: + deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python( + litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON + ).get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return ( + datetime.date.fromisoformat(deprecation_date) + <= datetime.datetime.now(datetime.timezone.utc).astimezone().date() + ) + except ValueError: + return False + + +def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]: for model in models: try: model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) @@ -171,7 +209,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator continue if model_info.get("mode") != "chat": continue - yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) + yield ( + model, + (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0), + _is_past_deprecation_date(model_info["key"]), + ) def _route_rank(model: str, custom_llm_provider: str) -> int | None: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 6979d72ff14..950a91beb81 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import responses import respx @@ -19,6 +20,7 @@ from litellm.litellm_core_utils.health_check_helpers import ( HealthCheckHelpers, default_health_check_mode, native_health_check_mode, + wildcard_health_check_models, ) from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth @@ -1449,6 +1451,339 @@ async def test_groq_wildcard_health_check_uses_a_concrete_model( assert response == {} +_WILDCARD_CATALOG: Final = MappingProxyType( + { + "databricks": ( + "databricks/databricks-claude-fixture-d", + "databricks/databricks-claude-fixture-c", + "databricks/databricks-claude-fixture-b", + "databricks/databricks-claude-fixture-a", + "databricks/databricks-cheap-chat", + "databricks/databricks-claude-fixture-retired", + ), + "bedrock": ("meta.llama-fixture", "anthropic.claude-fixture"), + "cloudflare": ("cloudflare/@cf/meta/fixture-chat",), + "vertex_ai": ("claude-fixture@default",), + } +) +_WILDCARD_CATALOG_COST: Final = MappingProxyType( + { + "databricks/databricks-claude-fixture-retired": 1e-10, + "databricks/databricks-cheap-chat": 1e-9, + "databricks/databricks-claude-fixture-a": 2e-9, + "databricks/databricks-claude-fixture-b": 3e-9, + "databricks/databricks-claude-fixture-c": 4e-9, + "databricks/databricks-claude-fixture-d": 5e-9, + "meta.llama-fixture": 1e-9, + "anthropic.claude-fixture": 2e-9, + "cloudflare/@cf/meta/fixture-chat": 1e-9, + "claude-fixture@default": 1e-9, + } +) +_DATABRICKS_FIXTURE_API_BASE: Final = "https://fixture.cloud.databricks.com/serving-endpoints" + + +@pytest.fixture +def wildcard_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + providers: Final = {model: provider for provider, models in _WILDCARD_CATALOG.items() for model in models} + litellm.register_model( + { + model: { + "litellm_provider": providers[model], + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": cost, + "output_cost_per_token": cost, + **({"deprecation_date": "2000-01-01"} if model.endswith("-retired") else {}), + } + for model, cost in _WILDCARD_CATALOG_COST.items() + } + ) + for provider, models in _WILDCARD_CATALOG.items(): + monkeypatch.setitem(litellm.models_by_provider, provider, list(models)) + + +_DATABRICKS_PROVIDER_PROBES: Final = ( + "databricks/databricks-claude-fixture-retired", + "databricks/databricks-cheap-chat", + "databricks/databricks-claude-fixture-a", +) +_BEDROCK_PROVIDER_PROBES: Final = ("bedrock/meta.llama-fixture", "bedrock/anthropic.claude-fixture") +_DATABRICKS_SERVED_CLAUDE_PROBES: Final = ( + "databricks/databricks-claude-fixture-a", + "databricks/databricks-claude-fixture-b", +) + + +@pytest.mark.parametrize( + ("custom_llm_provider", "wildcard", "expected"), + [ + ("databricks", "databricks-claude-*", (*_DATABRICKS_SERVED_CLAUDE_PROBES, *_DATABRICKS_PROVIDER_PROBES[:2])), + ( + "databricks", + "databricks/databricks-claude-*", + (*_DATABRICKS_SERVED_CLAUDE_PROBES, *_DATABRICKS_PROVIDER_PROBES[:2]), + ), + ("databricks", "*", (*_DATABRICKS_PROVIDER_PROBES[1:], _DATABRICKS_PROVIDER_PROBES[0])), + ("databricks", "databricks-claude-fixture-r*", _DATABRICKS_PROVIDER_PROBES), + ("databricks", "databricks-llama-*", _DATABRICKS_PROVIDER_PROBES), + ("databricks", "*-fixture", _DATABRICKS_PROVIDER_PROBES), + ("databricks", "team-endpoints/*", _DATABRICKS_PROVIDER_PROBES), + ("bedrock", "converse/anthropic.*", ("bedrock/converse/anthropic.claude-fixture", *_BEDROCK_PROVIDER_PROBES)), + ( + "bedrock", + "converse/*", + ( + "bedrock/converse/meta.llama-fixture", + "bedrock/converse/anthropic.claude-fixture", + *_BEDROCK_PROVIDER_PROBES, + ), + ), + ("bedrock", "invoke/anthropic.*", ("bedrock/invoke/anthropic.claude-fixture", *_BEDROCK_PROVIDER_PROBES)), + ("bedrock", "us-east-1/anthropic.*", ("bedrock/us-east-1/anthropic.claude-fixture", *_BEDROCK_PROVIDER_PROBES)), + ("bedrock", "agent/anthropic.*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "nova/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "nova-2/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "mantle/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "openai/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "chat_completions/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/*", _BEDROCK_PROVIDER_PROBES), + ( + "bedrock", + "converse/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/*", + _BEDROCK_PROVIDER_PROBES, + ), + ("cloudflare", "@cf/*", ("cloudflare/@cf/meta/fixture-chat",)), + ("vertex_ai", "claude-*", ("vertex_ai/claude-fixture@default",)), + ("vertex_ai", "meta/*", ("vertex_ai/claude-fixture@default",)), + ("fixture-gateway", "team/*", ()), + ], +) +@pytest.mark.usefixtures("wildcard_catalog") +def test_wildcard_health_check_probes_the_cheapest_served_chat_models_before_the_provider_cheapest( + custom_llm_provider: str, wildcard: str, expected: tuple[str, ...] +) -> None: + assert wildcard_health_check_models(wildcard=wildcard, custom_llm_provider=custom_llm_provider) == expected + + +def _fail_first_databricks_claude_probe(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model == "databricks-claude-fixture-a": + return httpx.Response(500, json={"error_code": "INTERNAL_ERROR", "message": "fixture outage"}) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-health", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "2"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_falls_back_within_the_wildcard_before_any_other_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_fail_first_databricks_claude_probe + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + probed_models: Final = [json.loads(call.request.content)["model"] for call in upstream.calls] + assert probed_models[0] == "databricks-claude-fixture-a" + assert probed_models[-1] == "databricks-claude-fixture-b" + assert all(model.startswith("databricks-claude-") for model in probed_models) + assert "databricks-claude-fixture-a" not in probed_models[1:] + assert response == {} + + +def _answer_only_the_retired_databricks_claude_model(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model != "databricks-claude-fixture-retired": + return httpx.Response(404, json={"error_code": "ENDPOINT_NOT_FOUND", "message": f"{model} does not exist"}) + return _fail_first_databricks_claude_probe(request) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_stays_healthy_while_any_model_the_provider_probes_answers( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_answer_only_the_retired_databricks_claude_model + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert [json.loads(call.request.content)["model"] for call in upstream.calls] == [ + "databricks-claude-fixture-a", + "databricks-claude-fixture-b", + "databricks-claude-fixture-retired", + ] + assert response == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_that_fails_on_every_probe_tries_its_own_models_then_the_provider_probes( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").respond( + 401, json={"error_code": "UNAUTHENTICATED", "message": "fixture bad token"} + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert [json.loads(call.request.content)["model"] for call in upstream.calls] == [ + "databricks-claude-fixture-a", + "databricks-claude-fixture-b", + "databricks-claude-fixture-retired", + "databricks-cheap-chat", + ] + assert "error" in response + + +_DATABRICKS_EQUAL_PRICE_MODELS: Final = ( + "databricks/databricks-free-d", + "databricks/databricks-free-c", + "databricks/databricks-free-b", + "databricks/databricks-free-a", +) + + +def _register_equal_price_databricks_models(monkeypatch: pytest.MonkeyPatch, listed_order: tuple[str, ...]) -> None: + litellm.register_model( + { + model: { + "litellm_provider": "databricks", + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + for model in listed_order + } + ) + monkeypatch.setitem(litellm.models_by_provider, "databricks", list(listed_order)) + + +@pytest.mark.parametrize("listed_order", [_DATABRICKS_EQUAL_PRICE_MODELS, _DATABRICKS_EQUAL_PRICE_MODELS[::-1]]) +def test_partial_wildcard_health_check_orders_its_own_equal_price_models_by_id_whatever_the_listing_order( + monkeypatch: pytest.MonkeyPatch, listed_order: tuple[str, ...] +) -> None: + _register_equal_price_databricks_models(monkeypatch, listed_order) + + assert wildcard_health_check_models(wildcard="databricks/databricks-free-*", custom_llm_provider="databricks")[ + :2 + ] == ("databricks/databricks-free-a", "databricks/databricks-free-b") + + +def _answer_only_the_first_listed_databricks_free_model(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model != "databricks-free-d": + return httpx.Response(404, json={"error_code": "ENDPOINT_NOT_FOUND", "message": f"{model} does not exist"}) + return _fail_first_databricks_claude_probe(request) + + +@pytest.mark.asyncio +async def test_wildcard_health_check_falls_back_to_equal_price_provider_models_in_the_provider_listing_order( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + _register_equal_price_databricks_models(monkeypatch, _DATABRICKS_EQUAL_PRICE_MODELS) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_answer_only_the_first_listed_databricks_free_model + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert [json.loads(call.request.content)["model"] for call in upstream.calls] == ["databricks-free-d"] + assert response == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_without_a_served_cost_map_model_probes_the_provider_cheapest_chat_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_fail_first_databricks_claude_probe + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-llama-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert json.loads(upstream.calls[0].request.content)["model"] == "databricks-claude-fixture-retired" + assert response == {} + + +@pytest.mark.asyncio +async def test_partial_wildcard_health_check_on_a_provider_without_cost_map_chat_models_names_health_check_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setitem(litellm.models_by_provider, "databricks", []) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-llama-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert "model_info.health_check_model" in response["error"], response["error"] + assert respx_mock.calls.call_count == 0 + + @pytest.mark.asyncio async def test_cohere_rerank_health_check_returns_a_successful_response( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter @@ -1632,7 +1967,12 @@ async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_ monkeypatch.setitem( litellm.model_cost, "fake-together-chat", - {"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9, "mode": "chat", "litellm_provider": "together_ai"}, + { + "input_cost_per_token": 1e-9, + "output_cost_per_token": 1e-9, + "mode": "chat", + "litellm_provider": "together_ai", + }, ) monkeypatch.setitem(litellm.models_by_provider, "together_ai", ("fake-together-chat",)) litellm.in_memory_llm_clients_cache.flush_cache() @@ -1647,4 +1987,3 @@ async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_ assert "error" not in result, result assert json.loads(together_route.calls.last.request.content)["model"] == "fake-together-chat" assert not other_hosts.called - diff --git a/tests/unit/litellm_core_utils/test_llm_request_utils.py b/tests/unit/litellm_core_utils/test_llm_request_utils.py index 0a2bd5cec1b..e7350fae395 100644 --- a/tests/unit/litellm_core_utils/test_llm_request_utils.py +++ b/tests/unit/litellm_core_utils/test_llm_request_utils.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.llm_request_utils import ( ensure_extra_body_is_safe, flatten_form_field_values, pick_cheapest_chat_models_from_llm_provider, + rank_chat_models_from_llm_provider, serialize_multipart_form_fields, ) @@ -149,6 +150,24 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP "deepseek/fake-cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"}, "openai/fake-free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, "deepseek/fake-cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"}, + "deepseek/fake-retired-chat": { + "input_cost_per_token": 1e-14, + "output_cost_per_token": 1e-14, + "mode": "chat", + "deprecation_date": "2000-01-01", + }, + "deepseek/fake-retiring-chat": { + "input_cost_per_token": 2e-12, + "output_cost_per_token": 2e-12, + "mode": "chat", + "deprecation_date": "2999-01-01", + }, + "deepseek/fake-misdated-chat": { + "input_cost_per_token": 2.5e-12, + "output_cost_per_token": 2.5e-12, + "mode": "chat", + "deprecation_date": "next spring", + }, "fake-free-chat-without-provider": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, } for name, row in rows.items(): @@ -156,12 +175,73 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows)) -def test_pick_cheapest_chat_models_prefers_provider_routed_ids_and_skips_ids_routed_elsewhere( +def test_pick_cheapest_chat_models_prefers_provider_routed_ids_skips_ids_routed_elsewhere_and_keeps_retired_ones( deepseek_rows_whose_cheapest_ids_route_elsewhere: None, ) -> None: assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [ + "deepseek/fake-retired-chat", "deepseek/fake-cheap-chat", + "deepseek/fake-retiring-chat", + "deepseek/fake-misdated-chat", "deepseek/fake-pricey-chat", "fake-free-chat-without-provider", ] - assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"] + assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-retired-chat"] + + +def test_rank_chat_models_flags_only_the_rows_whose_cost_map_deprecation_date_has_passed( + deepseek_rows_whose_cheapest_ids_route_elsewhere: None, +) -> None: + assert { + ranked_model.model: ranked_model.is_past_deprecation_date + for ranked_model in rank_chat_models_from_llm_provider(PICKER_PROVIDER) + } == { + "deepseek/fake-retired-chat": True, + "deepseek/fake-cheap-chat": False, + "deepseek/fake-retiring-chat": False, + "deepseek/fake-misdated-chat": False, + "deepseek/fake-pricey-chat": False, + "fake-free-chat-without-provider": False, + } + + +@pytest.mark.parametrize( + "listed_order", + [("deepseek/fake-tie-b", "deepseek/fake-tie-a"), ("deepseek/fake-tie-a", "deepseek/fake-tie-b")], +) +def test_pick_cheapest_chat_models_keeps_the_provider_listing_order_for_equal_price_models( + monkeypatch: pytest.MonkeyPatch, listed_order: tuple[str, str] +) -> None: + for name in listed_order: + monkeypatch.setitem( + litellm.model_cost, + name, + { + "input_cost_per_token": 1e-12, + "output_cost_per_token": 1e-12, + "mode": "chat", + "litellm_provider": PICKER_PROVIDER, + }, + ) + monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, listed_order) + + assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == list(listed_order) + + +def test_rank_chat_models_flags_a_retired_model_the_provider_lists_without_its_cost_map_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows: Final = { + "vertex_ai/fake-retired-vertex-chat": {"input_cost_per_token": 1e-14, "deprecation_date": "2000-01-01"}, + "vertex_ai/fake-live-vertex-chat": {"input_cost_per_token": 1e-12, "deprecation_date": "2999-01-01"}, + } + for name, row in rows.items(): + monkeypatch.setitem(litellm.model_cost, name, {**row, "mode": "chat", "litellm_provider": "vertex_ai"}) + monkeypatch.setitem( + litellm.models_by_provider, "vertex_ai", tuple(name.removeprefix("vertex_ai/") for name in rows) + ) + + assert [ + (ranked_model.model, ranked_model.is_past_deprecation_date) + for ranked_model in rank_chat_models_from_llm_provider("vertex_ai") + ] == [("fake-retired-vertex-chat", True), ("fake-live-vertex-chat", False)] diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index e48bf4edc09..df0429316ed 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -11,6 +11,7 @@ import respx import litellm from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers +from litellm.litellm_core_utils.llm_request_utils import RankedChatModel from litellm.proxy import health_check as hc_module from litellm.proxy.health_check import ( _is_strategy_router_deployment, @@ -85,8 +86,10 @@ async def test_ahealth_check_wildcard_models_respects_max_tokens(): """ with ( patch( - "litellm.litellm_core_utils.llm_request_utils.pick_cheapest_chat_models_from_llm_provider", - return_value=["gpt-4o-mini"], + "litellm.litellm_core_utils.llm_request_utils.rank_chat_models_from_llm_provider", + return_value=( + RankedChatModel(model="gpt-4o-mini", route_rank=0, cost=0.0, is_past_deprecation_date=False), + ), ), patch("litellm.acompletion", new_callable=AsyncMock), ): From 4a5c5665faec4efd3bd2dc31436c9f6f8b25cf2e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:49:55 -0700 Subject: [PATCH 25/34] fix(azure_ai)!: keep OpenAI prompt cache breakpoints on the Foundry chat completions path (#45299) * fix(azure_ai): keep OpenAI prompt cache breakpoints on the Foundry chat completions path * fix(azure_ai): keep only OpenAI content part types when preserving prompt cache breakpoints * fix(azure_ai): share the breakpoint check with the cache-mark step and keep only text parts A deployment's model_info opt-in is registered under its provider-prefixed cost-map key without a litellm_provider, so that prefix now names the provider the flag speaks for. * refactor(azure_ai): read text parts through a walrus and mark new test values Final * fix(azure_ai): read a router-registered breakpoint opt-in only for azure_ai and keep a message mark off dropped file parts * fix(azure_ai): read the breakpoint opt-in from each deployment's own model_info and base_model A router deployment without its own supports_prompt_cache_breakpoint flag now reads the cost map as loaded, so a sibling's opt-in on the same model string no longer turns its requests into 400s. The shared hook no longer reads router registered keys for azure_ai. * fix(azure_ai): send breakpoints in Foundry's shape under drop_params and fall back to the model name * test(integration): expect the injected breakpoint on the Foundry GPT-6 chat wire --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 33 +- litellm/llms/azure_ai/chat/transformation.py | 111 +++++- ...ect_prompt_cache_breakpoint_owned_proxy.py | 17 +- .../chat/test_azure_ai_transformation.py | 356 ++++++++++++++++++ 4 files changed, 499 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index fc1f1830129..45c82589d2d 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -23,6 +23,7 @@ from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, with_prompt_cache_breakpoint, @@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[ def _hosted_openai_dialect_flag( - model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None] + model: str, + custom_llm_provider: str | None, + resolve_provider: Callable[[str], str | None], + model_cost: Mapping[str, Mapping[str, object]] | None = None, ) -> bool | None: """ Explicit opt-in for an OpenAI-shaped deployment served by another provider. @@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag( """ import litellm - exact_entry: Final = litellm.model_cost.get(model) - exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None + cost_map: Final = ( + cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries + if model_cost is None + else model_cost + ) + exact_entry: Final = cost_map.get(model) + exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)): return None provider: Final = custom_llm_provider or resolve_provider(model) if provider is None or provider == "openai": return None - if isinstance(exact_entry, dict) and exact_provider == provider: + if isinstance(exact_entry, Mapping) and exact_provider == provider: return _hosted_entry_flag(exact_entry, lambda: provider) from litellm.utils import get_potential_model_names @@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag( names["stripped_model_name"], names["provider_prefixed_model_name"], ) - entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates) - flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict)) + entries: Final = (cost_map.get(candidate) for candidate in candidates) + flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping)) return next((flag for flag in flags if flag is not None), None) @@ -537,6 +546,18 @@ class AnthropicCacheControlHook(CustomPromptManagement): return False return prompt_cache_options is not None or targets_openai_api(api_base) + @staticmethod + def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool: + return ( + _hosted_openai_dialect_flag( + model, + custom_llm_provider, + AnthropicCacheControlHook._resolve_provider, + GetModelCostMap.loaded_model_cost_map(), + ) + is True + ) + @staticmethod def _resolve_provider(model: str) -> str | None: return _served_provider_for_model(model) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index d382d653a53..5381447f28f 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast import httpx from httpx import Response +from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.prompt_templates.common_utils import ( content_parts, convert_content_list_to_str, drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, is_pdf_file_part, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, ) from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( @@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig from litellm.llms.xai.chat.transformation import XAIChatConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField from litellm.utils import add_path_to_api_base, supports_tool_choice @@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool: return any(_is_list_only_content_part(part) for part in content_parts(message)) +_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + +def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + content: Final = message.get("content") + if not isinstance(content, list): + return () + try: + parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content) + except ValidationError: + return () + return tuple(part for part in parts if part.get("type") == "text") + + +def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues: + if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)): + return message + text_only: Final = {**message, "content": list(text_parts)} + return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only + + +_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None: + if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None: + return None + return PromptCacheBreakpoint(mode="explicit") + + +def _part_with_foundry_breakpoint(part: object) -> object: + if not isinstance(part, Mapping): + return part + fields: Final = _FIELDS.validate_python(part) + if "prompt_cache_breakpoint" not in fields: + return fields + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"])) + + +def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues: + renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message)) + parts: Final = content_parts(renamed) if drop_params else () + if not parts: + return renamed + foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]} + return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape + + +def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: + base_model: Final = litellm_params.get("base_model") + named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,) + model_info: Final = litellm_params.get("model_info") + if not isinstance(model_info, Mapping): + return any( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models + ) + deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint") + if isinstance(deployment_flag, bool): + return deployment_flag + return any( + AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai") + for name in named_models + ) + + class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator): def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract usage: Final = chunk.get("usage") @@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig): self, messages: list[AllMessageValues], model: str, + reads_breakpoints: bool | None = None, + drop_params: bool = False, ) -> list: """ - Azure AI Studio doesn't support content as a list. This handles: @@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig): Azure AI Foundry backends set additionalProperties=false and reject these with "Extra inputs are not permitted", which breaks multi-turn Anthropic-format clients that echo thinking blocks back as history. - 2. Transforms list content to a string. + 2. Transforms list content to a string, except on deployments that read OpenAI explicit + prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry + of its base_model or name, as loaded and without a sibling deployment's opt-in): only a + content part can carry the breakpoint (a message-level one is silently ignored), so those + keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the + way the openai provider renames it. Every other part is dropped, exactly as the string + conversion drops it, before the marks are renamed, so a message-level mark lands on the + last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only + shape Foundry takes, and a malformed one is dropped. 3. If message contains an image, audio, or PDF file part, send as is (user-intended) Operates on a deep copy so the caller's messages keep their thinking blocks and provider metadata, which a fallback to another provider still needs. """ - stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages)) + deployment_reads_breakpoints: Final = ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai") + if reads_breakpoints is None + else reads_breakpoints + ) + copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages)) + stripped_messages: Final = ( + [_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages] + if deployment_reads_breakpoints + else copied_messages + ) for message in stripped_messages: message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: @@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig): if _has_list_only_content_part(message): continue + if deployment_reads_breakpoints and (text_parts := _text_parts(message)): + message_dict["content"] = list(text_parts) + continue + texts = convert_content_list_to_str(message=message) if texts: message["content"] = texts @@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig): if extra_body and isinstance(extra_body, dict): optional_params.update(extra_body) optional_params.pop("max_retries", None) - return super().transform_request(model, messages, optional_params, litellm_params, headers) + return { + "model": model, + "messages": self._transform_messages( + messages, + model, + _deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)), + bool(litellm_params.get("drop_params") or litellm.drop_params), + ), + **optional_params, + } def transform_response( self, diff --git a/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py b/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py index 3df73d3b767..d470605b1e5 100644 --- a/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py +++ b/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py @@ -153,15 +153,16 @@ def _foundry_deployment(rig: _Rig, scenario: Scenario, *, bare: bool) -> str: return name -def _assert_foundry_chat_wire(received: Request, prompt: str, *, options: JsonValue | None) -> None: +def _assert_foundry_chat_wire(received: Request, user_content: JsonValue, *, options: JsonValue | None) -> None: body: Final = body_of(received) + marked_system: Final = [{"type": "text", "text": SYSTEM, "prompt_cache_breakpoint": EXPLICIT}] assert urlsplit(received.target).netloc == _FOUNDRY_HOST, received.target assert urlsplit(received.target).path == _FOUNDRY_CHAT_PATH, received.target - assert "prompt_cache_breakpoint" not in received.body.decode(), received.body assert body["model"] == _FOUNDRY_MODEL, received.body - assert body["messages"] == [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], ( - received.body - ) + assert body["messages"] == [ + {"role": "system", "content": marked_system}, + {"role": "user", "content": user_content}, + ], received.body assert body.get("prompt_cache_options") == options, received.body @@ -185,7 +186,7 @@ def test_b9_b12_foundry_gpt6_responses_reach_the_openai_dialect_on_the_foundry_h @pytest.mark.timeout(_CELL_TIMEOUT) -def test_b10_foundry_gpt6_messages_bridge_to_chat_without_a_breakpoint(rig: _Rig) -> None: +def test_b10_foundry_gpt6_messages_bridge_to_chat_keeps_the_injected_breakpoint(rig: _Rig) -> None: marker: Final = fresh_marker() with rig.gateway.scenario() as scenario: name: Final = _foundry_deployment(rig, scenario, bare=False) @@ -193,12 +194,12 @@ def test_b10_foundry_gpt6_messages_bridge_to_chat_without_a_breakpoint(rig: _Rig rig.gateway, rig.wire, "messages", request_body("messages", name, prompt_text(marker)) ) assert_answered(outcome, marker) - _assert_foundry_chat_wire(received, prompt_text(marker), options=IMPLICIT) + _assert_foundry_chat_wire(received, [{"type": "text", "text": prompt_text(marker)}], options=IMPLICIT) success_row(name, marker) @pytest.mark.timeout(_CELL_TIMEOUT) -def test_b11_foundry_gpt6_chat_stays_on_the_plain_chat_wire(rig: _Rig) -> None: +def test_b11_foundry_gpt6_chat_stays_on_the_chat_wire_with_the_injected_breakpoint(rig: _Rig) -> None: marker: Final = fresh_marker() with rig.gateway.scenario() as scenario: name: Final = _foundry_deployment(rig, scenario, bare=False) diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 6269fd367fe..145d45317c3 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,6 +1,7 @@ import json import traceback from collections.abc import Callable +from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch @@ -9,6 +10,7 @@ import respx from httpx import Response import litellm +from litellm import Router from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.azure_ai.azure_model_router.transformation import ( @@ -880,6 +882,360 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages(): } +def test_azure_ai_keeps_prompt_cache_breakpoint_for_a_model_that_supports_it(_local_model_cost_map): + """ + Foundry GPT-6 deployments take OpenAI explicit prompt caching, and the breakpoint only + counts on a content part, so a system message carrying one must reach the request body as + the content-part list it arrived in while the Anthropic-only fields are still stripped. + """ + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "stable instructions", + "prompt_cache_breakpoint": {"mode": "explicit"}, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request["messages"][1]["content"] == [{"type": "text", "text": "hi"}] + assert request["prompt_cache_options"] == {"mode": "explicit"} + + +def test_azure_ai_still_flattens_list_content_for_a_model_without_the_breakpoint_flag(_local_model_cost_map): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "stable ", "prompt_cache_breakpoint": {"mode": "explicit"}}, + {"type": "text", "text": "instructions"}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == "stable instructions" + assert request["messages"][1]["content"] == "hi" + assert not _find_key_anywhere(request["messages"], "prompt_cache_breakpoint") + + +def test_azure_ai_drops_thinking_parts_from_history_while_keeping_the_breakpoint(_local_model_cost_map): + """ + A multi-turn client can echo an assistant turn back with thinking parts inside its content + list. Foundry rejects those parts, and the string conversion always dropped them, so a model + that keeps content parts must still drop them while the text parts and the breakpoint stay. + """ + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + {"role": "user", "content": "read the file"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "The user wants me to read a file.", "signature": "sig"}, + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "text", "text": "Reading it now."}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "thanks"}]}, + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request["messages"][2]["content"] == [{"type": "text", "text": "Reading it now."}] + assert request["messages"][3]["content"] == [{"type": "text", "text": "thanks"}] + assert not _find_key_anywhere(request["messages"], "thinking") + assert not _find_key_anywhere(request["messages"], "data") + + + +def test_azure_ai_renames_a_cache_control_mark_to_a_breakpoint_for_a_model_that_reads_them(_local_model_cost_map): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "system", + "content": [{"type": "text", "text": "stable instructions", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}, + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request["messages"][1]["content"] == [ + {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert not _find_key_anywhere(request["messages"], "cache_control") + + +def test_azure_ai_drops_a_file_part_next_to_the_breakpoint_like_the_string_conversion(_local_model_cost_map): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize", "prompt_cache_breakpoint": {"mode": "explicit"}}, + {"type": "file", "file": {"file_id": "file-held-by-another-provider"}}, + ], + } + ] + + kept: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + flattened: Final = AzureAIStudioConfig().transform_request( + model="gpt-4o", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert kept["messages"][0]["content"] == [ + {"type": "text", "text": "summarize", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert flattened["messages"][0]["content"] == "summarize" + + +def test_azure_ai_sends_a_pdf_part_next_to_the_breakpoint_as_is(_local_model_cost_map): + content: Final = [ + {"type": "file", "file": {"filename": "document.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}, + {"type": "text", "text": "What is the codeword?", "prompt_cache_breakpoint": {"mode": "explicit"}}, + ] + + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[{"role": "user", "content": content}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": content}] + + +def test_azure_ai_puts_a_message_level_mark_on_the_last_text_part_when_a_file_part_follows(_local_model_cost_map): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "user", + "cache_control": {"type": "ephemeral"}, + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "file", "file": {"file_id": "file-held-by-another-provider"}}, + ], + } + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "summarize", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + + +@pytest.mark.parametrize( + ("registered", "keeps_parts"), + [ + ({"azure_ai/qa-foundry-alias": {"litellm_provider": "azure_ai", "supports_prompt_cache_breakpoint": True}}, True), + ({"qa-foundry-alias": {"litellm_provider": "openai", "supports_prompt_cache_breakpoint": True}}, False), + ], + ids=["azure_ai-entry-opts-in", "openai-entry-does-not-speak-for-azure_ai"], +) +def test_azure_ai_keeps_parts_only_when_the_azure_ai_entry_reads_breakpoints( + _local_model_cost_map, registered: dict[str, dict[str, object]], keeps_parts: bool +): + litellm.register_model(model_cost=registered, persist_across_reloads=False, add_to_provider_model_sets=False) + breakpoint_part: Final = {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + + request: Final = AzureAIStudioConfig().transform_request( + model="qa-foundry-alias", + messages=[{"role": "system", "content": [breakpoint_part]}], + optional_params={}, + litellm_params={"custom_llm_provider": "azure_ai", "drop_params": True}, + headers={}, + ) + + assert request["messages"][0]["content"] == ([breakpoint_part] if keeps_parts else "stable") + + +@pytest.mark.parametrize( + "registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")] +) +def test_azure_ai_deployment_reads_only_its_own_breakpoint_flag_not_a_siblings_sharing_its_model_string( + _local_model_cost_map, respx_mock: respx.MockRouter, registration_order: tuple[str, ...] +) -> None: + breakpoint_flags: Final = MappingProxyType( + { + "opted-in": {"supports_prompt_cache_breakpoint": True}, + "opted-out": {"supports_prompt_cache_breakpoint": False}, + } + ) + def deployment(host: str, model_info: dict[str, object]) -> dict[str, object]: + return { + "model_name": host, + "litellm_params": { + "model": "azure_ai/qa-shared-deployment", + "api_base": f"https://{host}.services.ai.azure.com/models", + "api_key": "azure-ai-test-key", + }, + "model_info": model_info, + } + + routes: Final = { + host: respx_mock.post(host=f"{host}.services.ai.azure.com", path="/models/chat/completions").mock( + return_value=Response( + 200, + json={ + "id": "chatcmpl-azure-ai", + "object": "chat.completion", + "created": 1700000000, + "model": "qa-shared-deployment", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ) + ) + for host in registration_order + } + router: Final = Router( + model_list=[ + deployment(host, {"id": f"{host}-id", **breakpoint_flags.get(host, {})}) for host in registration_order + ] + ) + marked_system: Final = { + "role": "system", + "content": [{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}], + } + + for host in registration_order: + router.completion(model=host, messages=[marked_system, {"role": "user", "content": "hi"}]) + + def sent_system_content(host: str) -> object: + return json.loads(routes[host].calls.last.request.content)["messages"][0]["content"] + + assert sent_system_content("opted-in") == [ + {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert sent_system_content("opted-out") == "stable" + assert sent_system_content("unflagged") == "stable" + + +@pytest.mark.parametrize( + "litellm_params", + [ + {"base_model": "azure_ai/gpt-6-astra"}, + {"base_model": "azure_ai/gpt-6-astra", "model_info": {"id": "my-astra-deployment-id"}}, + ], +) +def test_azure_ai_keeps_parts_for_a_custom_deployment_name_whose_base_model_reads_breakpoints( + _local_model_cost_map, litellm_params: dict[str, object] +): + breakpoint_part: Final = {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + + request: Final = AzureAIStudioConfig().transform_request( + model="my-astra-deployment", + messages=[{"role": "system", "content": [breakpoint_part]}], + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["messages"][0]["content"] == [breakpoint_part] + + +@pytest.mark.parametrize( + "litellm_params", + [ + {"base_model": "azure/cost-tracking-row"}, + {"base_model": "azure/cost-tracking-row", "model_info": {"id": "astra-deployment-id"}}, + ], +) +def test_azure_ai_falls_back_to_the_model_name_when_the_base_model_row_does_not_read_breakpoints( + _local_model_cost_map, litellm_params: dict[str, object] +): + breakpoint_part: Final = {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[{"role": "system", "content": [breakpoint_part]}], + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["messages"][0]["content"] == [breakpoint_part] + + +@pytest.mark.parametrize( + "sent_marker, litellm_params, wire_part", + [ + ( + {"mode": "explicit", "ttl": "30m"}, + {"drop_params": True}, + {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}, + ), + ({"type": "ephemeral"}, {"drop_params": True}, {"type": "text", "text": "stable"}), + ( + {"mode": "explicit", "ttl": "30m"}, + {}, + {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}, + ), + ], +) +def test_azure_ai_reshapes_breakpoints_for_foundry_only_under_drop_params( + _local_model_cost_map, + sent_marker: dict[str, object], + litellm_params: dict[str, object], + wire_part: dict[str, object], +): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "stable", "prompt_cache_breakpoint": sent_marker}]} + ], + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["messages"][0]["content"] == [wire_part] + + def test_azure_ai_reference_only_tool_result_is_sent_as_empty_text(): messages: Final = [ {"role": "user", "content": "load the WebFetch tool"}, From 953750ed41c69ea909fa4064a42317b04fa8a39b Mon Sep 17 00:00:00 2001 From: Devon Krisman <844724+dkrisman@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:06:57 -0400 Subject: [PATCH 26/34] fix(websearch-interception)!: stream native server-tool blocks to clients (#40967) * fix(websearch-interception): stream native server-tool blocks to clients Streaming native clients (Anthropic SDK, Claude Desktop, Claude Code) never received the server_tool_use and web_search_tool_result blocks the interception layer builds for them, so citation panels stayed empty and result counts read zero. Two gaps in the same flow: FakeAnthropicMessagesStreamIterator._create_content_block_chunks only knew text, thinking, redacted_thinking and tool_use; any other block type emitted a bare content_block_stop with no start and no payload, silently dropping the block from the fake stream. server_tool_use now streams as content_block_start plus input_json_delta and any other server-generated block rides whole in content_block_start, matching Anthropic's native streaming of these types. try_short_circuit_search only recognized a native client by finding a web_search_* tool in the tools list, but the pre-request and deployment hooks convert that tool to the LiteLLM standard shape before the short-circuit runs, recording the conversion in WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY. The flag now counts as native too, so the synthetic response carries the native blocks on the converted path instead of only when hook ordering happens to leave the tool untouched. Fixes #35333 * fix(anthropic): accept server_tool_use and web_search_tool_result blocks when costing a /v1/messages response The short-circuit now returns native web search blocks, and the cost path validated the served body against a content union that only allowed text, tool_use and thinking blocks, so non-streaming requests answered 500 * fix(anthropic): keep every server_tool_use field when re-streaming a /v1/messages reply * fix(anthropic): keep tool_use fields, redacted thinking data and text citations when re-streaming a /v1/messages reply * test(websearch-interception): build the cost test's logging object with a datetime start time * fix(websearch-interception): send search snippets once in the direct search answer * fix(guardrails): mask web search queries, titles and urls on /v1/messages and deliver stream rewrites past server_tool_use blocks Output guardrails now scan the native web search blocks the short-circuit emits, ahead of the reply text so guardrails that read the last text still scan the reply. A guardrail reply of a different length falls back to the reply-only write-back. Ended-stream tool call rewrites land on server_tool_use blocks too, the ended-stream scan key covers the result strings, and legacy-hook pipeline steps pass through the result strings their response view lacks. The wire test reads the streamed server_tool_use input from its input_json_delta. * fix(guardrails): deliver a legacy post-call hook's tool-call rewrites on streams instead of discarding the stream rewrite A legacy hook that masked a tool call's arguments on a buffered stream made the whole rewrite undeliverable, text included, so the original stream went out. With web search blocks in the stream that now hit any hook masking the search query. The adapter hands same-count tool-call rewrites to the translation's write-back, as a unified guardrail's already are, and still discards a rewrite that adds or drops a tool call. * fix(guardrails): show a legacy post-call hook the web search result blocks of a /v1/messages stream The legacy hook's view of a buffered Anthropic stream is rebuilt from the assembled ModelResponse, which keeps web_search_tool_result blocks only in provider-specific fields, so a hook masking result titles on a stream never saw them. The hook's view now carries those blocks, its rewrite of them goes back through the ended-stream write-back, and the adapter no longer needs to pass texts outside the hook's view through untouched. * test(guardrails): type the legacy hook stream test helpers * fix(guardrails): scan a streamed web search query as text like the non-streaming reply * test(guardrails): type the masking guardrail test helpers * fix(guardrails): keep the block id when scanning a streamed web search query --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../websearch_interception/handler.py | 15 +- .../websearch_interception/transformation.py | 10 +- .../chat/guardrail_translation/handler.py | 335 ++++++++++++++++-- .../messages/native/fake_stream_iterator.py | 56 +-- .../proxy/policy_engine/pipeline_executor.py | 36 +- litellm/types/llms/anthropic.py | 19 + .../test_websearch_interception_wire.py | 27 +- .../test_websearch_short_circuit.py | 122 +++++++ .../test_anthropic_guardrail_handler.py | 307 +++++++++++++++- .../native/test_fake_stream_iterator.py | 145 +++++++- .../policy_engine/test_pipeline_executor.py | 238 +++++++++++-- 11 files changed, 1191 insertions(+), 119 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 4739b3ee561..2bb89248be4 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger): (t for t in tools if is_anthropic_native_web_search_tool(t)), None, ) + emit_native_blocks: Final = native_tool is not None or ( + kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY)) + ) outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs) search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome) content: Final[list[dict[str, object]]] = [] - if native_tool is not None: + if emit_native_blocks: tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}" - tool_name: Final = native_tool.get("name") or "web_search" + tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search" content.append( { "type": "server_tool_use", @@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger): } ) content.append( - WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome) + WebSearchTransformation.build_web_search_outcome_block( + tool_use_id=tool_use_id, outcome=outcome, with_snippets=False + ) ) - # Keep the text block so non-native short-circuit callers (Claude Code, - # github_copilot, etc.) see the same payload they always have. content.append({"type": "text", "text": search_result_text}) response: Final[dict[str, object]] = { @@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug( "WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)", len(search_result_text), - native_tool is not None, + emit_native_blocks, ) return response diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 58ae8bd6e48..55564dd1134 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -412,6 +412,7 @@ class WebSearchTransformation: def build_web_search_tool_result_block( tool_use_id: str, search_response: SearchResponse | None, + with_snippets: bool = True, ) -> dict[str, object]: """ Build an Anthropic-native ``web_search_tool_result`` content block. @@ -445,6 +446,8 @@ class WebSearchTransformation: ``litellm.asearch()``. If None or empty, the block is still emitted with an empty result list (signals "search ran, no results" rather than "search did not run"). + with_snippets: False when a text block in the same reply already + carries the snippets. """ items: Final[list[dict[str, object]]] = [] if search_response is not None: @@ -460,7 +463,7 @@ class WebSearchTransformation: "title": title, "page_age": page_age, "encrypted_content": "", - "snippet": getattr(r, "snippet", "") or "", + **({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}), } ) return { @@ -481,12 +484,15 @@ class WebSearchTransformation: } @staticmethod - def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]: + def build_web_search_outcome_block( + tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True + ) -> dict[str, object]: match outcome: case SearchSucceeded(response=response): return WebSearchTransformation.build_web_search_tool_result_block( tool_use_id=tool_use_id, search_response=response, + with_snippets=with_snippets, ) case SearchFailed(error_code=error_code): return WebSearchTransformation.build_web_search_tool_result_error_block( diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index e351542da01..7f688ae733b 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -24,7 +24,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, @@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( unappliable_request_rewrite, ) from litellm.types.llms.anthropic import ( + ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AllAnthropicToolsValues, AnthropicMessagesRequest, ) @@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol): text: str +@runtime_checkable class _WritableMessage(Protocol): @overload def get(self, key: str, /) -> object | None: ... @@ -195,6 +197,140 @@ class _ToolCallShape: arguments: str +@dataclass(frozen=True, slots=True) +class _WebSearchString: + block_idx: int + result_idx: int | None + field: str + value: str + + +def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]: + choice: Final = response.choices[0] if response.choices else None + fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None + blocks: Final = ( + (as_str_mapping(item) for item in iter_items(fields.get("web_search_results"))) + if fields is not None and isinstance(fields.get("web_search_results"), list) + else () + ) + return tuple( + deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result" + ) + + +def _web_search_results(block: Mapping[str, object]) -> Iterable[object]: + return iter_items(block.get("content")) if isinstance(block.get("content"), list) else () + + +def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]: + fields: Final = as_str_mapping(result) + if fields is None or fields.get("type") != "web_search_result": + return () + values: Final = ((field, fields.get(field)) for field in ("title", "url")) + return tuple( + _WebSearchString(block_idx, result_idx, field, value) + for field, value in values + if isinstance(value, str) and value + ) + + +def _is_server_tool_use(block: Mapping[str, object]) -> bool: + block_id: Final = block.get("id") + match block.get("type"): + case "server_tool_use": + return True + case "tool_use": + return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) + case _: + return False + + +def _web_search_block_strings( + block_idx: int, block: Mapping[str, object], *, with_query: bool +) -> tuple[_WebSearchString, ...]: + """The strings a client reads off a native web search block: the ``server_tool_use`` query and each + ``web_search_tool_result`` result's title and url.""" + match block.get("type"): + case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block): + tool_input: Final = as_str_mapping(block.get("input")) + query: Final = None if tool_input is None else tool_input.get("query") + return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else () + case "web_search_tool_result": + return tuple( + chain.from_iterable( + _web_search_result_strings(block_idx, result_idx, result) + for result_idx, result in enumerate(_web_search_results(block)) + ) + ) + case _: + return () + + +def _rewritten_web_search_result( + result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str] +) -> object: + fields: Final = as_str_mapping(result) + if fields is None: + return result + return { + **fields, + **{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites}, + } + + +def _rewritten_web_search_block( + block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]] +) -> Mapping[str, object]: + by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites}) + if _is_server_tool_use(block): + query: Final = by_location.get((None, "query")) + tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({}) + return block if query is None else {**block, "input": {**tool_input, "query": query}} + if not isinstance(block.get("content"), list): + return block + return { + **block, + "content": [ + _rewritten_web_search_result(result_idx, result, by_location) + for result_idx, result in enumerate(_web_search_results(block)) + ], + } + + +def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None: + block: Final = as_str_mapping(content_block) + if block is None or not isinstance(content_block, _WritableMessage): + return + rewritten: Final = _rewritten_web_search_block(block, rewrites) + writable: Final = _as_writable(content_block) + for key in ("input", "content"): + if key in rewritten: + writable[key] = rewritten[key] + + +def _split_web_search_texts( + guardrailed: Sequence[str], web_search_count: int, reply_count: int +) -> tuple[Sequence[str], Sequence[str]]: + split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0 + return guardrailed[:split_at], guardrailed[split_at:] + + +def _web_search_rewrites_by_block( + strings: Sequence[_WebSearchString], guardrailed: Sequence[object] +) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]: + rewrites: Final = tuple( + (target, value) + for target, value in zip(strings, guardrailed) + if isinstance(value, str) and value != target.value + ) + return MappingProxyType( + { + block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx) + for block_idx in {target.block_idx for target, _ in rewrites} + } + ) + + def _is_client_tool_use(block: Mapping[str, object]) -> bool: return ( block.get("type") == "tool_use" @@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge _TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object]) _DECODED_SSE_DATA: Final = TypeAdapter(object) +_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"}) _RELEASED_TOOL_USE_STOP: Final = ( b"event: message_delta\n" b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, ' @@ -315,6 +452,51 @@ def _write_back_tool_use( block["name"] = shape.name +def _streamed_query_strings( + tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape] +) -> tuple[_WebSearchString, ...]: + if len(tool_call_blocks) != len(tool_calls): + return () + return tuple( + chain.from_iterable( + _web_search_block_strings( + index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True + ) + for (index, block), tool_call in zip(tool_call_blocks, tool_calls) + ) + ) + + +def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]: + return MappingProxyType( + { + target.block_idx: value + for target, value in zip(strings, guardrailed) + if isinstance(value, str) and value != target.value + } + ) + + +def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape: + arguments: Final = _rewritten_tool_use_input(tool_call.arguments) + if arguments is None: + return tool_call + return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query})) + + +def _with_query_rewrites( + tool_calls: tuple[_ToolCallShape, ...], + tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], + query_rewrites: Mapping[int, str], +) -> tuple[_ToolCallShape, ...]: + if not query_rewrites or len(tool_call_blocks) != len(tool_calls): + return tool_calls + return tuple( + _with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call + for (index, _), tool_call in zip(tool_call_blocks, tool_calls) + ) + + @dataclass(frozen=True, slots=True) class _SSEFieldRewrite: """One field of one nested section of a buffered SSE event, rewritten.""" @@ -402,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation): def post_call_hook_response(self, response: object) -> object: if not isinstance(response, ModelResponse): return response - return self.adapter.translate_openai_response_to_anthropic(response) + translated: Final = self.adapter.translate_openai_response_to_anthropic(response) + result_blocks: Final = _streamed_web_search_result_blocks(response) + if not result_blocks: + return translated + content: Final = translated.get("content") or () + return {**translated, "content": [*result_blocks, *content]} @staticmethod def _build_streaming_usage_response( @@ -1273,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation): task_mappings, tool_calls_to_check, ) + block_fields: Final = tuple(map(self._output_block_fields, response_content)) + web_search_strings: Final = tuple( + chain.from_iterable( + _web_search_block_strings(content_idx, fields[1], with_query=True) + for content_idx, fields in enumerate(block_fields) + if fields is not None + ) + ) # Step 2: Apply guardrail to all texts in batch - if texts_to_check or tool_calls_to_check: + if texts_to_check or tool_calls_to_check or web_search_strings: request_data = self._prepare_request_data( request_data, response, @@ -1284,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation): ) inputs: Final = self._build_guardrail_inputs( - texts_to_check, + [*(target.value for target in web_search_strings), *texts_to_check], images_to_check, tool_calls_to_check, response, @@ -1297,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + web_search_rewrites, reply_rewrites = _split_web_search_texts( + guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check) + ) # Step 3: Map guardrail responses back to original response structure await self._apply_guardrail_responses_to_output( response=response, - responses=guardrailed_texts, + responses=list(reply_rewrites), task_mappings=task_mappings, ) + self._write_back_web_search_strings( + response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites) + ) verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response) @@ -1350,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation): string_so_far = first_choice.message.content pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) guardrail_inputs: Final = GenericGuardrailAPIInputs() - if string_so_far: - guardrail_inputs["texts"] = [string_so_far] + leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else () + tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far) + streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls) + streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) + streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings) + texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts] + if texts_to_check: + guardrail_inputs["texts"] = texts_to_check if tool_calls_list: guardrail_inputs["tool_calls"] = tool_calls_list @@ -1374,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation): responses_so_far, request_data ) raise - guardrailed_texts: Final = _guardrailed_inputs.get("texts") + web_search_rewrites, reply_rewrites = _split_web_search_texts( + _guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts) + ) if ( deliver_ended_stream_rewrites and isinstance(string_so_far, str) and string_so_far - and guardrailed_texts - and guardrailed_texts[0] != string_so_far + and reply_rewrites + and reply_rewrites[0] != string_so_far ): self._write_ended_stream_text_rewrite( responses_so_far, - guardrailed_texts[0], + reply_rewrites[0], guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + if deliver_ended_stream_rewrites: + self._write_ended_stream_web_search_rewrites( + responses_so_far, + _web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites), + ) if deliver_ended_stream_rewrites: returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls") self._write_ended_stream_tool_call_rewrites( responses_so_far, pre_guardrail_tool_calls=pre_guardrail_tool_calls, - post_guardrail_tool_calls=_tool_call_shapes( - returned_tool_calls - if isinstance(returned_tool_calls, list) - and len(returned_tool_calls) == len(pre_guardrail_tool_calls) - else tool_calls_list or () + post_guardrail_tool_calls=_with_query_rewrites( + _tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(pre_guardrail_tool_calls) + else tool_calls_list or () + ), + tool_call_blocks, + _query_rewrites( + streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :] + ), ), guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) @@ -1451,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation): request_data["litellm_metadata"] = user_metadata return request_data + @staticmethod + def _write_back_web_search_strings( + response_content: Sequence[object], + rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]], + ) -> None: + for block_idx, rewrites in rewrites_by_block.items(): + _write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites) + @staticmethod def _get_response_content(response: object) -> Sequence[object]: """Extract content list from a dict or object response.""" @@ -1573,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation): ) -> None: """Deliver ended-stream guardrail tool-call rewrites by rewriting the buffered chunks in place: the rebuilt response lists tool calls in the - order of the stream's ``tool_use`` blocks, so the nth rewritten call lands - on the nth block, its first ``input_json_delta`` carrying the full rewritten + order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth + rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten arguments, every later one blanked, and ``content_block_start`` carrying the rewritten name. Blocks that do not line up with the rebuilt tool calls make the rewrite undeliverable, so the pipeline executor discards it and releases the original chunks.""" if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - block_indices: Final = tuple( - index - for item in responses_so_far - for event in cls._iter_sse_events(item) - if event.get("type") == "content_block_start" - and isinstance(block := event.get("content_block"), Mapping) - and block.get("type") == "tool_use" - and isinstance(index := event.get("index"), int) - ) + block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far)) if len(block_indices) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite raise UndeliverableStreamRewrite( guardrail_name, f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried " - f"{len(block_indices)} tool_use blocks", + f"{len(block_indices)} tool_use or server_tool_use blocks", ) rewrites_by_block: Final = MappingProxyType( { @@ -1636,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation): ), ) + @classmethod + def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]: + events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far) + result_blocks: Final = ( + (index, block) + for event in events + if event.get("type") == "content_block_start" + and isinstance(index := event.get("index"), int) + and (block := as_str_mapping(event.get("content_block"))) is not None + ) + return tuple( + chain.from_iterable( + _web_search_block_strings(index, block, with_query=False) for index, block in result_blocks + ) + ) + + @classmethod + def _write_ended_stream_web_search_rewrites( + cls, + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place + rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]], + ) -> None: + """Deliver ended-stream guardrail rewrites of web search result titles and urls into the + ``content_block_start`` that carries each whole ``web_search_tool_result`` block.""" + if not rewrites_by_block: + return + + def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None: + index: Final = event.get("index") + block: Final = as_str_mapping(event.get("content_block")) + rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None + if event.get("type") != "content_block_start" or block is None or rewrites is None: + return None + return _SSEFieldRewrite( + "content_block", "content", _rewritten_web_search_block(block, rewrites).get("content") + ) + + cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block) + + @classmethod + def _streamed_tool_call_blocks( + cls, responses_so_far: Sequence[object] + ) -> tuple[tuple[int, Mapping[str, object]], ...]: + events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far) + return tuple( + (index, block) + for event in events + if event.get("type") == "content_block_start" + and (block := as_str_mapping(event.get("content_block"))) is not None + and isinstance(block_type := block.get("type"), str) + and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES + and isinstance(index := event.get("index"), int) + ) + @classmethod def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]: """The id of each buffered ``tool_use`` block, by its content block index.""" @@ -1738,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation): def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: stream_ended: Final = self._check_streaming_has_ended(responses_so_far) tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far) + streamed_text: Final = self.get_streaming_string_so_far(responses_so_far) + result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else () return StreamingScanKey( - texts=(self.get_streaming_string_so_far(responses_so_far),), + texts=(*(target.value for target in result_strings), streamed_text), tool_calls=tool_use_fingerprints if stream_ended else (), stream_ended=stream_ended, tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended, diff --git a/litellm/llms/anthropic/messages/native/fake_stream_iterator.py b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py index 39bdfbbc53a..3a26737a8ac 100644 --- a/litellm/llms/anthropic/messages/native/fake_stream_iterator.py +++ b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py @@ -12,11 +12,20 @@ import json from collections.abc import Mapping, Sequence from typing import Final, cast +from pydantic import TypeAdapter + from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +_CITATIONS: Final = TypeAdapter(tuple[object, ...]) + + +def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]: + citations: Final = block.get("citations") + return _CITATIONS.validate_python(citations) if isinstance(citations, list) else () + class FakeAnthropicMessagesStreamIterator: """ @@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator: def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]: """Build SSE chunks for a single content block.""" - chunks: Final = [] + chunks: Final = list[bytes]() block_type: Final = block_dict.get("type") if block_type == "text": + citations: Final = _citations_of(block_dict) content_block_start = { "type": "content_block_start", "index": index, - "content_block": {"type": "text", "text": ""}, + "content_block": {"type": "text", "text": "", "citations": ()} + if citations + else {"type": "text", "text": ""}, } chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) + citation_deltas: Final = ( + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "citations_delta", "citation": citation}, + } + for citation in citations + ) + chunks.extend( + f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas + ) text: Final = block_dict.get("text", "") content_block_delta = { "type": "content_block_delta", @@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator: } chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()) - elif block_type == "redacted_thinking": - content_block_start = { + elif block_type in ("tool_use", "server_tool_use"): + empty_input: Final[Mapping[str, object]] = {} + tool_use_start: Final = { "type": "content_block_start", "index": index, - "content_block": {"type": "redacted_thinking"}, + "content_block": {**block_dict, "input": empty_input}, } - chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) - - elif block_type == "tool_use": - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": { - "type": "tool_use", - "id": block_dict.get("id"), - "name": block_dict.get("name"), - "input": {}, - }, - } - chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) - input_data: Final = block_dict.get("input", {}) - content_block_delta = { + chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode()) + tool_use_delta: Final = { "type": "content_block_delta", "index": index, "delta": { "type": "input_json_delta", - "partial_json": json.dumps(input_data), + "partial_json": json.dumps(block_dict.get("input", {})), }, } - chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()) + chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode()) else: passthrough_start: Final = { diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 4ad29bd3478..47572dd7ceb 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -98,15 +98,11 @@ def _count_change(sent: tuple[object, ...] | None, returned: tuple[object, ...] return (len(sent), len(returned)) -def _tool_call_mismatch_reason( - sent: tuple[tuple[object, object], ...] | None, returned: tuple[tuple[object, object], ...] | None -) -> str | None: - if sent == returned: - return None +def _tool_call_count_mismatch_reason(sent: Sequence[object] | None, returned: Sequence[object] | None) -> str | None: sent_count: Final = len(sent or ()) returned_count: Final = len(returned or ()) if sent_count == returned_count: - return "the legacy hook changed a tool call's name or arguments, which this path cannot write back" + return None return f"the legacy hook returned {returned_count} tool calls for a stream that carried {sent_count}" @@ -205,11 +201,10 @@ class _LegacyHookStreamAdapter(CustomGuardrail): non-streaming hooks, an exception it raises ends the stream through the executor's fail/error classification, and the response it hands back, or the one it changed in place and returned ``None`` for, is re-scanned by the same translation so its texts reach the - client through the translation's ended-stream write-back. A - replacement whose scanned texts do not line up with the originals, or whose tool calls - differ from them, is undeliverable, so the executor releases the original chunks. A stream - that carried no text to scan, such as a tool-only Anthropic message, stays deliverable as - long as the hook left the tool calls alone.""" + client through the translation's ended-stream write-back. Tool calls the hook rewrote go to the translation's + tool-call write-back like a unified guardrail's. A replacement whose scanned texts or tool + calls do not line up with the originals by count is undeliverable, so the executor releases + the original chunks.""" def __init__( self, @@ -258,14 +253,19 @@ class _LegacyHookStreamAdapter(CustomGuardrail): guardrail_name, f"the legacy hook returned {returned_text_count} texts for a stream that carried {sent_text_count}", ) - tool_call_mismatch: Final = _tool_call_mismatch_reason( - _tool_call_shapes(inputs.get("tool_calls")), _tool_call_shapes(rescanned.get("tool_calls")) + sent_tool_calls: Final = inputs.get("tool_calls") + rewritten_tool_calls: Final = rescanned.get("tool_calls") + tool_call_count_mismatch: Final = _tool_call_count_mismatch_reason(sent_tool_calls, rewritten_tool_calls) + if tool_call_count_mismatch is not None: + raise UndeliverableStreamRewrite(guardrail_name, tool_call_count_mismatch) + texts_update: Final[GenericGuardrailAPIInputs] = {"texts": rewritten} if rewritten else {} + tool_calls_update: Final[GenericGuardrailAPIInputs] = ( + {"tool_calls": rewritten_tool_calls} + if rewritten_tool_calls is not None + and _rewrote(_tool_call_shapes(sent_tool_calls), _tool_call_shapes(rewritten_tool_calls)) + else {} ) - if tool_call_mismatch is not None: - raise UndeliverableStreamRewrite(guardrail_name, tool_call_mismatch) - if not rewritten: - return inputs - rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} + rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, **texts_update, **tool_calls_update} return rewritten_inputs async def _rescan( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 50c4a2340a3..a648376c542 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -713,6 +713,23 @@ class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel): data: str +class AnthropicResponseContentBlockServerToolUse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["server_tool_use"] + id: str + name: str + input: Mapping[str, object] + + +class AnthropicResponseContentBlockWebSearchToolResult(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["web_search_tool_result"] + tool_use_id: str + content: Sequence[Mapping[str, object]] | Mapping[str, object] + + class AnthropicResponseUsageBlock(LiteLLMBaseModel): model_config = ConfigDict(extra="allow") @@ -744,6 +761,8 @@ class AnthropicResponse(LiteLLMBaseModel): | AnthropicResponseContentBlockToolUse | AnthropicResponseContentBlockThinking | AnthropicResponseContentBlockRedactedThinking + | AnthropicResponseContentBlockServerToolUse + | AnthropicResponseContentBlockWebSearchToolResult ] """Content generated by the model.""" diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py index 5ec860910e5..1bd9f18253e 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py @@ -29,6 +29,24 @@ def sse_events(text: str) -> tuple[tuple[str, dict[str, object]], ...]: ) +def streamed_blocks(events: tuple[tuple[str, dict[str, object]], ...]) -> tuple[dict[str, object], ...]: + input_json: Final = tuple( + (event["index"], event["delta"]["partial_json"]) + for name, event in events + if name == "content_block_delta" and event["delta"]["type"] == "input_json_delta" + ) + return tuple( + { + **event["content_block"], + "input": json.loads("".join(part for index, part in input_json if index == event["index"])), + } + if any(index == event["index"] for index, _ in input_json) + else event["content_block"] + for name, event in events + if name == "content_block_start" + ) + + @pytest.mark.covers("other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results") def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks( gateway: Gateway, tmp_path: Path @@ -136,7 +154,7 @@ def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_sni for earlier, earlier_event in events[:position] if earlier == "content_block_start" }, response.text - started: Final = tuple(event["content_block"] for name, event in events if name == "content_block_start") + started: Final = streamed_blocks(events) search_ids: Final = tuple(block["id"] for block in started if block["type"] == "server_tool_use") assert search_ids and all(search_id.startswith("srvtoolu_") for search_id in search_ids), response.text assert started[-1] == {"type": "text", "text": ""}, response.text @@ -162,7 +180,12 @@ def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_sni ) ), response.text assert ( - "".join(event["delta"]["text"] for name, event in events if name == "content_block_delta") == "turn-1" + "".join( + event["delta"]["text"] + for name, event in events + if name == "content_block_delta" and event["delta"]["type"] == "text_delta" + ) + == "turn-1" ), response.text assert [event["delta"]["stop_reason"] for name, event in events if name == "message_delta"] == [ "end_turn" diff --git a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py index ef050d96dc5..10cb6ff12e9 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py @@ -5,7 +5,9 @@ Tests the short-circuit path that detects web-search-only /v1/messages requests and executes the search directly without routing through the backend LLM. """ +import datetime import json +from collections.abc import Iterator, Mapping, Sequence from typing import Final from unittest.mock import AsyncMock, patch @@ -16,6 +18,7 @@ import litellm from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) +from litellm.messages.dispatch import MessagesResult # --------------------------------------------------------------------------- # Detection tests @@ -73,6 +76,42 @@ class TestTryShortCircuitSearch: assert "Result" in text_block["text"] mock_search.assert_called_once_with("Search for Claude Code releases") + @pytest.mark.asyncio + async def test_emits_native_blocks_when_tools_already_converted(self, perplexity_search: respx.Route): + """Pre-request/deployment hooks convert the native tool to the LiteLLM + standard shape before the short-circuit runs and record that in + WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY; the flag alone must be enough for + native blocks to be emitted.""" + from litellm.integrations.websearch_interception.handler import ( + WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY, + ) + + logger: Final = WebSearchInterceptionLogger(enabled_providers=["hosted_vllm"]) + + result: Final = await logger.try_short_circuit_search( + model="hosted_vllm/some-model", + messages=[{"role": "user", "content": "Search for something"}], + tools=[ + { + "name": "litellm_web_search", + "description": "Search the web for information.", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + } + ], + custom_llm_provider="hosted_vllm", + kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + assert perplexity_search.call_count == 1 + assert result is not None + assert [b["type"] for b in result["content"]] == ["server_tool_use", "web_search_tool_result", "text"] + assert result["content"][0]["name"] == "web_search" + assert [r["url"] for r in result["content"][1]["content"]] == ["https://example.com/releases"] + @pytest.mark.asyncio async def test_does_not_short_circuit_mixed_tools(self): """Mix of web_search and other tools → NOT short-circuited""" @@ -468,6 +507,89 @@ class TestShortCircuitEntryPoint: assert text_block["text"] == "results" +class TestShortCircuitThroughMessagesEntryPoint: + """The pre-request hook renames the native tool before the short-circuit + runs, so only a request through anthropic_messages shows whether native + clients still get their server_tool_use and web_search_tool_result blocks.""" + + NATIVE_TOOL: Final = {"type": "web_search_20250305", "name": "web_search"} + + @pytest.fixture(autouse=True) + def _intercept_web_search(self, perplexity_search: respx.Route) -> Iterator[respx.Route]: + with patch("litellm.callbacks", [WebSearchInterceptionLogger(enabled_providers=["bedrock"])]): + yield perplexity_search + + @staticmethod + async def _send(*, tools: Sequence[Mapping[str, object]], stream: bool, **kwargs: object) -> MessagesResult: + return await litellm.anthropic_messages( + model="bedrock/converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": "latest python release"}], + tools=tools, + max_tokens=64, + stream=stream, + **kwargs, + ) + + @pytest.mark.asyncio + async def test_native_tool_gets_native_blocks(self, perplexity_search: respx.Route): + response: Final = await self._send(tools=[self.NATIVE_TOOL], stream=False) + + assert perplexity_search.call_count == 1 + assert [block["type"] for block in response["content"]] == [ + "server_tool_use", + "web_search_tool_result", + "text", + ] + server_tool_use, tool_result, text = response["content"] + assert server_tool_use["name"] == "web_search" + assert server_tool_use["input"] == {"query": "latest python release"} + assert tool_result["tool_use_id"] == server_tool_use["id"] + assert [result["url"] for result in tool_result["content"]] == ["https://example.com/releases"] + assert "Release notes" in text["text"] + assert "Release notes" not in json.dumps(tool_result) + + @pytest.mark.asyncio + async def test_native_blocks_reply_can_be_costed(self): + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = Logging( + model="bedrock/converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": "latest python release"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="short-circuit-cost", + function_id="short-circuit-cost", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + response: Final = await self._send(tools=[self.NATIVE_TOOL], stream=False, litellm_logging_obj=logging_obj) + + assert logging_obj.response_cost_calculator(result=response) == 0.0 + + @pytest.mark.asyncio + async def test_native_tool_streams_native_blocks(self): + stream: Final = await self._send(tools=[self.NATIVE_TOOL], stream=True) + body: Final = b"".join([chunk async for chunk in stream]) + + starts: Final = [ + json.loads(line.removeprefix(b"data: "))["content_block"] + for line in body.splitlines() + if line.startswith(b"data: ") and b'"content_block_start"' in line + ] + assert [block["type"] for block in starts] == ["server_tool_use", "web_search_tool_result", "text"] + assert starts[1]["tool_use_id"] == starts[0]["id"] + + @pytest.mark.asyncio + async def test_litellm_standard_tool_stays_text_only(self): + from litellm.integrations.websearch_interception.tools import ( + get_litellm_web_search_tool, + ) + + response: Final = await self._send(tools=[get_litellm_web_search_tool()], stream=False) + + assert [block["type"] for block in response["content"]] == ["text"] + + class TestShortCircuitSafeguardResults: """A short-circuited reply never reaches the backend, so the proxy answers Claude Code's safeguards itself.""" diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 141ad8a0481..d815e1eedd3 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,6 +6,7 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json +import re from typing import Any, Literal, Optional, Final from unittest.mock import MagicMock, patch @@ -552,13 +553,25 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert chunks == original @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_use_rewrite_with_server_tool_use_block_fails_closed(self): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - + async def test_deliver_ended_stream_tool_use_rewrite_lands_on_server_tool_use_and_tool_use_blocks(self): handler = AnthropicMessagesHandler() server_tool_use = [ - ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}}), - ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}}), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}, + }, + ), ("content_block_stop", {"type": "content_block_stop", "index": 0}), ] tool_use = self._ended_tool_use_sse_chunks() @@ -568,6 +581,26 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: + [chunk.replace(b'"index": 0', b'"index": 1') for chunk in tool_use[1:]] ) + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', '{"fruit": "[MASKED]"}', "", ""] + raw = b"".join(chunks).decode() + assert '"id": "srvtoolu_1"' in raw and '"id": "toolu_1"' in raw + assert "persim" not in raw and '"query"' not in raw + + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_use_rewrite_on_an_unindexed_tool_block_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = AnthropicMessagesHandler() + tool_use = self._ended_tool_use_sse_chunks() + chunks = tool_use[:1] + [tool_use[1].replace(b'"index": 0', b'"index": "0"')] + tool_use[2:] + with pytest.raises(UndeliverableStreamRewrite): await handler.process_output_streaming_response( responses_so_far=chunks, @@ -576,6 +609,270 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: deliver_ended_stream_rewrites=True, ) + @staticmethod + def _python_masking_guardrail() -> CustomGuardrail: + class MaskPython(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + for tool_call in inputs.get("tool_calls", []): + if isinstance(tool_call, ChatCompletionMessageToolCall): + tool_call.function.arguments = re.sub( + "python", "[MASKED]", tool_call.function.arguments, flags=re.I + ) + return { + **inputs, + "texts": [re.sub("python", "[MASKED]", text, flags=re.I) for text in inputs.get("texts", [])], + } + + return MaskPython(guardrail_name="test") + + @staticmethod + def _python_texts_masking_guardrail() -> CustomGuardrail: + class MaskPythonTexts(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + return { + **inputs, + "texts": [re.sub("python", "[MASKED]", text, flags=re.I) for text in inputs.get("texts", [])], + } + + return MaskPythonTexts(guardrail_name="test") + + @staticmethod + def _web_search_result_block() -> dict[str, object]: + return { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [ + { + "type": "web_search_result", + "url": "https://docs.python.org/3.14/", + "title": "Python 3.14 docs", + "encrypted_content": "", + "page_age": None, + }, + { + "type": "web_search_result", + "url": "https://peps.example/745", + "title": "PEP 745", + "encrypted_content": "", + "page_age": None, + }, + ], + } + + @classmethod + def _web_search_stream_chunks(cls, server_tool_block_type: str = "server_tool_use") -> list[bytes]: + events = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + ), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": server_tool_block_type, + "id": "srvtoolu_1", + "name": "web_search", + "input": {}, + }, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "python 3.14 release date"}'}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "content_block_start", + {"type": "content_block_start", "index": 1, "content_block": cls._web_search_result_block()}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ( + "content_block_start", + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 2, + "delta": {"type": "text_delta", "text": "Python 3.14 shipped"}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 2}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 2}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + @pytest.mark.parametrize("server_tool_block_type", ["server_tool_use", "tool_use"]) + @pytest.mark.parametrize( + "guardrail_factory", + [_python_masking_guardrail, _python_texts_masking_guardrail], + ids=["texts_and_tool_calls", "texts_only"], + ) + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_mask_the_web_search_query_results_and_text( + self, guardrail_factory, server_tool_block_type + ): + chunks = self._web_search_stream_chunks(server_tool_block_type) + + await AnthropicMessagesHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=guardrail_factory(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + raw = b"".join(chunks).decode() + assert "python" not in raw.lower() + result_block = next( + json.loads(line[len("data:") :])["content_block"] + for line in raw.split("\n") + if line.startswith("data:") and '"web_search_tool_result"' in line + ) + assert [(result["title"], result["url"]) for result in result_block["content"]] == [ + ("[MASKED] 3.14 docs", "https://docs.[MASKED].org/3.14/"), + ("PEP 745", "https://peps.example/745"), + ] + assert '"id": "srvtoolu_1"' in raw and '"tool_use_id": "srvtoolu_1"' in raw + + @staticmethod + def _last_text_redacting_guardrail() -> CustomGuardrail: + class RedactLastText(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return {**inputs, "texts": ["[REDACTED] " + inputs["texts"][-1]]} + + return RedactLastText(guardrail_name="test") + + @pytest.mark.asyncio + async def test_output_response_keeps_the_reply_as_the_last_text_a_guardrail_scans(self): + response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "python 3.14 release date"}, + }, + self._web_search_result_block(), + {"type": "text", "text": "Python 3.14 shipped"}, + ], + "stop_reason": "end_turn", + } + + await AnthropicMessagesHandler().process_output_response( + response=response, + guardrail_to_apply=self._last_text_redacting_guardrail(), + litellm_logging_obj=MagicMock(), + ) + + assert response["content"][0]["input"] == {"query": "python 3.14 release date"} + assert response["content"][1] == self._web_search_result_block() + assert response["content"][2] == {"type": "text", "text": "[REDACTED] Python 3.14 shipped"} + + @pytest.mark.asyncio + async def test_deliver_ended_stream_keeps_the_reply_as_the_last_text_a_guardrail_scans(self): + chunks = self._web_search_stream_chunks() + + await AnthropicMessagesHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._last_text_redacting_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + raw = b"".join(chunks).decode() + assert "[REDACTED] Python 3.14 shipped" in raw + assert raw.count("[REDACTED]") == 1 + assert json.dumps(self._web_search_result_block()) in raw + + def test_ended_stream_scan_key_differs_from_a_mid_stream_key_by_the_web_search_results(self): + handler = AnthropicMessagesHandler() + chunks = self._web_search_stream_chunks() + + mid_stream_key = handler.get_streaming_scan_key(chunks[:-2]) + ended_key = handler.get_streaming_scan_key(chunks) + + assert mid_stream_key.texts == ("Python 3.14 shipped",) + assert ended_key != mid_stream_key + assert ended_key.texts[-1] == "Python 3.14 shipped" + assert {"Python 3.14 docs", "https://docs.python.org/3.14/"} <= set(ended_key.texts) + + @pytest.mark.asyncio + async def test_output_response_masks_the_web_search_query_results_and_text(self): + response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "python 3.14 release date"}, + }, + self._web_search_result_block(), + {"type": "text", "text": "Python 3.14 shipped"}, + ], + "stop_reason": "end_turn", + } + + await AnthropicMessagesHandler().process_output_response( + response=response, + guardrail_to_apply=self._python_masking_guardrail(), + litellm_logging_obj=MagicMock(), + ) + + assert "python" not in json.dumps(response).lower() + assert response["content"][0]["input"] == {"query": "[MASKED] 3.14 release date"} + assert [(result["title"], result["url"]) for result in response["content"][1]["content"]] == [ + ("[MASKED] 3.14 docs", "https://docs.[MASKED].org/3.14/"), + ("PEP 745", "https://peps.example/745"), + ] + assert response["content"][2] == {"type": "text", "text": "[MASKED] 3.14 shipped"} + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py index 233145f2706..ece5bb39225 100644 --- a/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py @@ -1,6 +1,10 @@ import json -from collections.abc import Iterable, Mapping -from typing import Final +from collections.abc import Iterable, Mapping, Sequence +from typing import Final, cast + +import anthropic +import httpx +import pytest from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, @@ -91,3 +95,140 @@ def test_message_delta_omits_absent_optional_fields(): "usage": {"input_tokens": 10, "output_tokens": 2}, } assert _message_delta(response) == {"stop_reason": "end_turn", "stop_sequence": None} + + +_WEB_SEARCH_RESULT_BLOCK: Final = { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [ + { + "type": "web_search_result", + "url": "https://example.com/a", + "title": "Example A", + "encrypted_content": "abc123", + } + ], +} + + +def _content_events(content: Sequence[Mapping[str, object]]) -> tuple[tuple[str, dict[str, object]], ...]: + response: Final = { + "id": "msg_ws", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": list(content), + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + return tuple( + event for event in _decoded_events(FakeAnthropicMessagesStreamIterator(response=response)) + if event[0].startswith("content_block_") + ) + + +def test_server_tool_use_streams_empty_start_then_input_json_delta(): + """A real Anthropic stream opens server_tool_use with an empty input and + sends the query as an input_json_delta, which SDK accumulators read.""" + caller: Final = {"type": "code_execution_20260120", "tool_id": "srvtoolu_00"} + events: Final = _content_events( + [ + { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "example search"}, + "caller": caller, + } + ] + ) + assert [event_type for event_type, _ in events] == [ + "content_block_start", + "content_block_delta", + "content_block_stop", + ] + assert events[0][1]["content_block"] == { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {}, + "caller": caller, + } + delta: Final = events[1][1]["delta"] + assert isinstance(delta, dict) + assert delta["type"] == "input_json_delta" + assert json.loads(delta["partial_json"]) == {"query": "example search"} + + +def test_web_search_tool_result_rides_whole_in_content_block_start(): + events: Final = _content_events([_WEB_SEARCH_RESULT_BLOCK]) + assert [event_type for event_type, _ in events] == ["content_block_start", "content_block_stop"] + assert events[0][1]["content_block"] == _WEB_SEARCH_RESULT_BLOCK + + +def test_native_search_blocks_keep_indices_aligned_with_text(): + events: Final = _content_events( + [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "q"}}, + _WEB_SEARCH_RESULT_BLOCK, + {"type": "text", "text": "Here is what I found."}, + ] + ) + starts: Final = tuple(payload for event_type, payload in events if event_type == "content_block_start") + stops: Final = tuple(payload["index"] for event_type, payload in events if event_type == "content_block_stop") + assert [(start["index"], cast(dict[str, object], start["content_block"])["type"]) for start in starts] == [ + (0, "server_tool_use"), + (1, "web_search_tool_result"), + (2, "text"), + ] + assert stops == (0, 1, 2) + + +_CALLER: Final = {"type": "code_execution_20260120", "tool_id": "srvtoolu_00"} + +_REPLAYED_BLOCKS: Final = ( + {"type": "thinking", "thinking": "Look up the release page.", "signature": "sig_01"}, + {"type": "redacted_thinking", "data": "EncryptedThinking=="}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}, "caller": _CALLER}, + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_fetch", "input": {"url": "https://example.com/a"}, "caller": _CALLER}, + _WEB_SEARCH_RESULT_BLOCK, + { + "type": "text", + "text": "Example A is the newest release.", + "citations": [ + { + "type": "web_search_result_location", + "cited_text": "Example A was released today.", + "url": "https://example.com/a", + "title": "Example A", + "encrypted_index": "idx_01", + } + ], + }, +) + + +def _sdk_rebuilt_content(block: Mapping[str, object]) -> list[dict[str, object]]: + response: Final = { + "id": "msg_replay", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [block], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + body: Final = b"".join(FakeAnthropicMessagesStreamIterator(response=response)) + transport: Final = httpx.MockTransport( + lambda _request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + client: Final = anthropic.Anthropic(api_key="sk-ant-test", http_client=httpx.Client(transport=transport)) + with client.messages.stream( + model="claude-sonnet-5-5", max_tokens=64, messages=[{"role": "user", "content": "hi"}] + ) as stream: + return [content_block.model_dump(exclude_none=True) for content_block in stream.get_final_message().content] + + +@pytest.mark.parametrize("block", _REPLAYED_BLOCKS, ids=lambda block: str(block["type"])) +def test_anthropic_sdk_rebuilds_every_block_field_from_the_stream(block: Mapping[str, object]): + assert _sdk_rebuilt_content(block) == [block] diff --git a/tests/unit/proxy/policy_engine/test_pipeline_executor.py b/tests/unit/proxy/policy_engine/test_pipeline_executor.py index c42878bd4f7..da855753b38 100644 --- a/tests/unit/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/unit/proxy/policy_engine/test_pipeline_executor.py @@ -5,9 +5,12 @@ Uses mock guardrails to validate pipeline execution without external services. """ import copy +import json import logging import pickle -from typing import Literal +from collections.abc import Callable, Iterable, Mapping, Sequence +from itertools import chain +from typing import Final, Literal from unittest.mock import MagicMock import pytest @@ -15,6 +18,7 @@ import pytest import litellm from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.utils import UnappliableRequestRewrite from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import ( @@ -1726,21 +1730,6 @@ async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up assert chunks == [_chunk()] -@pytest.mark.asyncio -async def test_streaming_step_discards_legacy_rewrite_that_changes_a_tool_call(monkeypatch, caplog): - masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} - guardrail = _LegacyHookGuardrail( - replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[masked_tool_call]) - ) - chunks = [_chunk()] - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) - - _assert_passed_with_discard_warning(result, caplog, "the legacy hook changed a tool call's name or arguments") - assert chunks == [_chunk()] - - @pytest.mark.asyncio async def test_streaming_step_discards_legacy_rewrite_that_drops_the_tool_calls(monkeypatch, caplog): guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[])) @@ -1772,21 +1761,6 @@ async def test_streaming_step_passes_a_tool_only_stream_the_legacy_hook_left_alo assert not any("discarded" in record.getMessage() for record in caplog.records) -@pytest.mark.asyncio -async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only_stream(monkeypatch, caplog): - masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} - guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement(tool_calls=[masked_tool_call])) - chunks = [_tool_only_chunk()] - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - result = await _run_legacy_streaming_step( - monkeypatch, guardrail, chunks, translation=_ToolOnlyLegacyScanningTranslation() - ) - - _assert_passed_with_discard_warning(result, caplog, "the legacy hook changed a tool call's name or arguments") - assert chunks == [_tool_only_chunk()] - - class _NoHooksGuardrail(CustomGuardrail): pass @@ -1825,6 +1799,208 @@ async def test_streaming_step_discards_a_legacy_rewrite_the_translation_cannot_r assert chunks == [_chunk()] +_AnthropicHookResponse = dict[str, list[dict[str, object]]] + + +class _AnthropicTextRewritingLegacyGuardrail(_LegacyHookGuardrail): + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: _AnthropicHookResponse + ) -> None: + self.calls.append({"response": copy.deepcopy(response)}) + for block in response["content"]: + if block["type"] == "text" and isinstance(text := block.get("text"), str): + block["text"] = "[REWRITTEN] " + text + + +def _anthropic_web_search_stream() -> list[bytes]: + result_block: Final = { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [ + {"type": "web_search_result", "url": "https://example.test/a", "title": "A", "encrypted_content": ""} + ], + } + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "m", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "a"}'}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": result_block}, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "hello world"}}, + {"type": "content_block_stop", "index": 2}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + return _anthropic_sse(events) + + +def _anthropic_sse(events: Iterable[Mapping[str, object]]) -> list[bytes]: + return [f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events] + + +def _anthropic_tool_only_stream() -> list[bytes]: + return _anthropic_sse( + ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "m", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"ssn"'}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": ': "123"}'}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + ) + + +def _streamed_events(chunks: Sequence[bytes]) -> list[dict[str, object]]: + return [ + json.loads(line[len("data: ") :]) + for line in b"".join(chunks).decode().splitlines() + if line.startswith("data: ") + ] + + +def _streamed_block_deltas(chunks: Sequence[bytes], index: int, field: str) -> str: + deltas: Final = ( + event["delta"] + for event in _streamed_events(chunks) + if event.get("index") == index and event["type"] == "content_block_delta" + ) + return "".join(str(delta[field]) for delta in deltas if isinstance(delta, dict)) + + +def _streamed_web_search_results(chunks: Sequence[bytes]) -> list[dict[str, str]]: + blocks: Final = ( + event["content_block"] for event in _streamed_events(chunks) if event["type"] == "content_block_start" + ) + results: Final = ( + block["content"] for block in blocks if isinstance(block, dict) and block["type"] == "web_search_tool_result" + ) + return list(chain.from_iterable(results)) + + +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_hook_mask_of_web_search_result_titles( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + chunks: Final = _anthropic_web_search_stream() + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result: Final = await _run_legacy_streaming_step( + monkeypatch, _AnthropicMaskingLegacyGuardrail(), chunks, translation=AnthropicMessagesHandler() + ) + + assert [step.outcome for step in result.step_results] == ["pass"] + assert [(item["title"], item["url"]) for item in _streamed_web_search_results(chunks)] == [ + ("[MASKED]", "https://example.test/a") + ] + assert _streamed_block_deltas(chunks, 2, "text") == "[MASKED] world" + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +class _AnthropicMaskingLegacyGuardrail(_LegacyHookGuardrail): + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: _AnthropicHookResponse + ) -> None: + for block in response["content"]: + if block["type"] == "text" and isinstance(text := block.get("text"), str): + block["text"] = text.replace("hello", "[MASKED]") + if block["type"] == "tool_use" and isinstance(tool_input := block.get("input"), dict): + block["input"] = {key: "[MASKED]" for key in tool_input} + if block["type"] == "web_search_tool_result" and isinstance(results := block.get("content"), list): + block["content"] = [{**result, "title": "[MASKED]"} for result in results] + + +@pytest.mark.parametrize( + "stream, masked_key", + [(_anthropic_web_search_stream, "query"), (_anthropic_tool_only_stream, "ssn")], + ids=["web_search_query", "tool_only"], +) +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_hook_mask_of_a_streamed_tool_input( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + stream: Callable[[], list[bytes]], + masked_key: str, +) -> None: + chunks: Final = stream() + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result: Final = await _run_legacy_streaming_step( + monkeypatch, _AnthropicMaskingLegacyGuardrail(), chunks, translation=AnthropicMessagesHandler() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert json.loads(_streamed_block_deltas(chunks, 0, "partial_json")) == {masked_key: "[MASKED]"} + assert "hello" not in b"".join(chunks).decode() + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_text_rewrite_on_a_web_search_stream( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + guardrail: Final = _AnthropicTextRewritingLegacyGuardrail() + chunks: Final = _anthropic_web_search_stream() + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result: Final = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=AnthropicMessagesHandler() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + raw: Final = b"".join(chunks).decode() + assert "[REWRITTEN] hello world" in raw + assert '"title": "A"' in raw and '"url": "https://example.test/a"' in raw + assert not any("discarded" in record.getMessage() for record in caplog.records) + + @pytest.mark.asyncio async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) From 56388fd13de12f13e7f49a43e0050ad590baa94d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:11:55 -0700 Subject: [PATCH 27/34] fix(count_tokens): count a history that carries a compaction block (#44784) * fix(count_tokens): count a compaction block instead of answering 500 A message history that replays Anthropic's signed compaction block made /v1/messages/count_tokens answer HTTP 500: neither the Anthropic nor the Vertex AI partner count-tokens request carried the compact-2026-09-04 beta, the provider rejected the block with a 400, the proxy fell back to the local tokenizer, and that raised on the unknown block type. Both provider count paths now add the compaction beta whenever the history carries a compaction block, and the local tokenizer counts the block's summary text like a text block, skipping the signature. * fix(token_counter): skip the tokenizer for empty thinking and compaction text * fix(count_tokens): forward context_management on Anthropic counts and count chat compaction blocks locally Vertex AI compaction support for inference and counting moves to its own PR, so the Vertex count handler is back to main here * fix(anthropic): map list-form context_management to edits for count_tokens and inference betas Count requests forwarded the OpenAI list form of context_management unchanged, while inference maps it to Anthropic edits, and the inference beta helper read the list as no edits. Both now map it once. A replayed compaction block with an empty summary no longer reaches the tokenizer. * fix(anthropic): forward count context_management only with edits, map Vertex list-form betas, and return provider context_management * fix(anthropic): drop an unmappable context_management list under drop_params --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 43 ++++-- litellm/llms/anthropic/common_utils.py | 74 ++++++++-- .../llms/anthropic/count_tokens/handler.py | 4 +- .../anthropic/count_tokens/token_counter.py | 1 + .../anthropic/count_tokens/transformation.py | 31 ++++- .../messages/native/transformation.py | 26 +--- .../anthropic/messages/transformation.py | 25 +--- .../proxy/anthropic_endpoints/endpoints.py | 18 ++- .../litellm_core_utils/test_token_counter.py | 91 ++++++++++++ .../test_anthropic_messages_transformation.py | 62 ++++++++- ...t_anthropic_count_tokens_transformation.py | 131 +++++++++++++++++- .../llms/anthropic/test_count_tokens_oauth.py | 2 +- .../vertex_ai/count_tokens/test_context.py | 3 +- ...artner_models_anthropic_messages_config.py | 23 +++ .../anthropic_endpoints/test_endpoints.py | 68 ++++++++- 15 files changed, 520 insertions(+), 82 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ea3e645ffdf..800a0939cec 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -17,6 +17,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from pydantic import TypeAdapter from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar @@ -69,6 +70,7 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, AnthropicSearchResultParam, + CompactionBlock, ) from litellm.types.llms.openai import ( AllMessageValues, @@ -937,6 +939,8 @@ def _count_messages( default_token_count, params.pdf_pricing, ) + elif key == "provider_specific_fields": + num_tokens += sum(map(params.count_function, _compaction_summaries(value))) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_search_results_text, @@ -951,6 +955,25 @@ def _count_messages( return num_tokens +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]: + fields: Final = as_str_mapping(provider_specific_fields) + blocks: Final = fields.get("compaction_blocks") if fields is not None else None + block_fields: Final = ( + tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else () + ) + return tuple( + content + for block in block_fields + if block is not None + and block.get("type") == "compaction" + and isinstance(content := block.get("content"), str) + and content + ) + + def _count_extra( count_function: TokenCounterFunction, tools: list[ChatCompletionToolParam] | None, @@ -1648,6 +1671,11 @@ def _count_anthropic_content( return tokens +_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType( + {"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"} +) + + def _count_content_list( count_function: TokenCounterFunction, content_list: str @@ -1659,6 +1687,7 @@ def _count_content_list( | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam | AnthropicSearchResultParam + | CompactionBlock ], use_default_image_token_count: bool, default_token_count: int | None, @@ -1670,8 +1699,10 @@ def _count_content_list( for c in content_list: if isinstance(c, str): num_tokens += count_function(c) - elif c["type"] == "text": - num_tokens += count_function(str(c.get("text", ""))) + elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE: + block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "") + if block_text: + num_tokens += count_function(block_text) elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) @@ -1704,12 +1735,6 @@ def _count_content_list( default_token_count, pdf_pricing, ) - elif c["type"] in ("thinking", "redacted_thinking"): - # Claude extended thinking content block - # Count the thinking text and skip the opaque blobs (signature, redacted data) - thinking_text = str(c.get("thinking", "")) - if thinking_text: - num_tokens += count_function(thinking_text) elif c["type"] == "tool_reference": # Anthropic tool-search reference block: a lightweight pointer to # a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}. @@ -1727,7 +1752,7 @@ def _count_content_list( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"tool_reference, search_result)." + f"compaction, tool_reference, search_result)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0bc519187f4..82977f97854 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -8,12 +8,13 @@ import re from collections.abc import Container, Mapping, MutableMapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone +from itertools import chain from types import MappingProxyType from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar from urllib.parse import quote, urlparse import httpx -from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError +from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -24,6 +25,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, is_encrypted_reasoning_block, @@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DECODED_JSON: Final = TypeAdapter(object) +_JSON_VALUE: Final = TypeAdapter(JsonValue) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool: return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages") +def _message_blocks(message: object) -> Sequence[object]: + fields: Final = as_str_mapping(message) + blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None + return blocks or () + + +def _is_signed_compaction_block(block: object) -> bool: + fields: Final = as_str_mapping(block) + if fields is None or fields.get("type") != "compaction": + return False + signature: Final = fields.get("signature") + return isinstance(signature, str) and bool(signature) + + +def carries_signed_compaction_block(messages: Sequence[object]) -> bool: + return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages)))) + + def requires_native_compaction_beta( custom_llm_provider: str, optional_params: Mapping[str, object], messages: Sequence[object], ) -> bool: return custom_llm_provider == "anthropic" and ( - optional_params.get("compaction") is not None - or any( - isinstance(block, Mapping) - and block.get("type") == "compaction" - and isinstance(block.get("signature"), str) - and bool(block.get("signature")) - for message in messages - if isinstance(message, Mapping) - for content in (message.get("content"),) - if isinstance(content, (list, tuple)) - for block in content + optional_params.get("compaction") is not None or carries_signed_compaction_block(messages) + ) + + +def anthropic_context_management(context_management: JsonValue) -> JsonValue: + if not isinstance(context_management, list): + return context_management + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic( + [entry for entry in context_management if isinstance(entry, dict)] + ) + return context_management if edits is None else _JSON_VALUE.validate_python(edits) + + +def _context_management_edits(spec: JsonValue) -> Sequence[object]: + mapping: Final = as_str_mapping(spec) + return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or () + + +def context_management_with_edits(context_management: JsonValue) -> JsonValue: + spec: Final = anthropic_context_management(context_management) + return spec if _context_management_edits(spec) else None + + +def context_management_betas(context_management: JsonValue) -> tuple[str, ...]: + edits: Final = _context_management_edits(anthropic_context_management(context_management)) + edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None) + return tuple( + beta + for beta, used in ( + (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types), + ( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + any(edit_type != "compact_20260112" for edit_type in edit_types), + ), ) + if used ) @@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index f8bf7232d9c..7a7d1d776d3 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, drop_params: bool = False, ) -> dict[str, JsonValue]: """ @@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + context_management=context_management, drop_params=drop_params, ) @@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers - headers: Final = self.get_count_tokens_headers(auth_header) + headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management) # Use LiteLLM's async httpx client async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 91fc77773d1..99b1cd7f241 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + context_management=context_management.spec if context_management is not None else None, drop_params=drops_unsupported_params(litellm_params.get("drop_params")), ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index e60ffe67c8d..1c8f0c206cd 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( + carries_signed_compaction_block, + context_management_betas, + context_management_with_edits, is_claude_model_target, merge_anthropic_beta_headers, tools_without_custom_field, ) from litellm.llms.anthropic.wif import resolve_anthropic_base +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -376,6 +380,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, drop_params: bool = False, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ @@ -408,6 +413,7 @@ class AnthropicCountTokensConfig: drop_custom=drop_params, ), ), + ("context_management", context_management_with_edits(context_management)), ) if value is not None } @@ -419,18 +425,33 @@ class AnthropicCountTokensConfig: ) ) - def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]: + def get_count_tokens_headers( + self, + auth_header: Mapping[str, str], + messages: Sequence[Mapping[str, JsonValue]], + context_management: JsonValue = None, + ) -> dict[str, str]: """The count-tokens headers around a resolved Anthropic auth header (``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta - merges with the token-counting beta instead of replacing it.""" + merges with the token-counting beta instead of replacing it. A history that carries a + signed compaction block also needs the compaction beta, or Anthropic rejects the block + as an unknown content type, and ``context_management`` edits need the betas inference sends.""" + compaction_betas: Final = ( + (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,) + if carries_signed_compaction_block(messages) + else () + ) + request_betas: Final = ( + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + *compaction_betas, + *context_management_betas(context_management), + ) return { "Content-Type": "application/json", "anthropic-version": "2023-06-01", **auth_header, - "anthropic-beta": merge_anthropic_beta_headers( - auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION - ), + "anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas), } def validate_request( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index c086618e829..ef855eff180 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -43,6 +43,7 @@ from ...common_utils import ( OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, + context_management_betas, is_claude_model_target, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, @@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) if transformed_context_management is not None: anthropic_messages_optional_request_params["context_management"] = transformed_context_management + elif drops_unsupported_params(litellm_params.drop_params): + anthropic_messages_optional_request_params.pop("context_management") ####### get required params for all anthropic messages requests ###### # Lazy %s: the f-string previously stringified the entire messages @@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", ()) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for structured outputs. Anthropic's newer request shape nests # the schema under output_config.format; the older top-level diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index bfa94396ea7..da82766c2b9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", []) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for web search tool for tool in tools: diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1a2e610d5c4..0e204cf9211 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,7 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from pydantic import JsonValue, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -54,6 +54,7 @@ from litellm.proxy.litellm_pre_call_utils import ( should_auto_drop_params_for_agentic_cli, ) from litellm.router import Router +from litellm.types.utils import TokenCountResponse router: Final = APIRouter() @@ -334,6 +335,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: } +_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + +def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]: + original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response) + context_management: Final = original.get("context_management") if original is not None else None + return {} if context_management is None else {"context_management": context_management} + + @router.post( "/v1/messages/count_tokens", tags=["[beta] Anthropic Messages Token Counting"], @@ -413,7 +423,11 @@ async def count_tokens( log_llm_api_exception(e, litellm_call_id) return _anthropic_error_json_response(proxy_exception_from_route_error(e), request) else: - return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)} + return { + "input_tokens": token_response.total_tokens, + **_provider_context_management(token_response), + **count_source_fields(token_response), + } @router.post( diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index c2f8e12a726..697ac76b4e5 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1962,6 +1962,97 @@ def test_token_counter_with_redacted_thinking_content(): assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) + +def test_token_counter_with_compaction_block(): + """ + A replayed on-demand compaction block (Anthropic's signed summary of the earlier turns) counts its + summary text like a text block and nothing for its signature. It used to raise, which turned every + /v1/messages/count_tokens call on a compacted history into a 500 once the provider count fell back here. + """ + model = "anthropic/claude-sonnet-5-5" + summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": [{"type": "text", "text": "Now do the same for Ingredient."}]} + signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40} + + as_compaction = [{"role": "assistant", "content": [signed_block]}, follow_up] + unsigned = [{"role": "assistant", "content": [{"type": "compaction", "content": summary}]}, follow_up] + as_text = [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up] + + assert token_counter(model=model, messages=as_compaction) == token_counter(model=model, messages=as_text) + assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text) + + +def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text(): + """ + Chat clients replay a compaction block in the assistant message's provider_specific_fields, and inference + sends it to Anthropic as a content block, so the local count includes its summary text and skips its signature. + """ + model = "anthropic/claude-sonnet-5-5" + summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40} + + replayed = [ + {"role": "assistant", "content": None, "provider_specific_fields": {"compaction_blocks": [signed_block]}}, + follow_up, + ] + as_text = [{"role": "assistant", "content": summary}, follow_up] + + assert token_counter(model=model, messages=replayed) == token_counter(model=model, messages=as_text) + + +class _BosEncoder: + def encode_batch_fast(self, texts: list[str]) -> list[list[int]]: + return [[0] * (len(text.split()) + 1) for text in texts] + + +def test_token_counter_charges_nothing_for_a_chat_compaction_block_with_no_summary(): + """ + A tokenizer that prepends a BOS token charges one token for the empty string, so a replayed + compaction block with an empty summary must not reach it, like an empty thinking block. + """ + model = "anthropic/claude-sonnet-5-5" + tokenizer = {"type": "huggingface_tokenizer", "tokenizer": _BosEncoder()} + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + replayed = [ + { + "role": "assistant", + "content": None, + "provider_specific_fields": {"compaction_blocks": [{"type": "compaction", "content": ""}]}, + }, + follow_up, + ] + bare = [{"role": "assistant", "content": None}, follow_up] + + assert token_counter(model=model, messages=replayed, custom_tokenizer=tokenizer) == token_counter( + model=model, messages=bare, custom_tokenizer=tokenizer + ) + + +def test_count_content_list_skips_the_tokenizer_for_empty_block_text(): + """ + An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction + block with no summary carry no tokens, even under a tokenizer that charges for the empty string + (the llama-2 tokenizer prepends its BOS token to every encode call). Counting them through the + tokenizer added one token per block on such models. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + def charges_for_empty_text(text: str) -> int: + return len(text.split()) + 1 + + empty_blocks = [ + {"type": "thinking", "thinking": "", "signature": "EqcLCkYICxgCKkCrqu6lP..."}, + {"type": "thinking", "thinking": None, "signature": "EqcLCkYICxgCKkCrqu6lP..."}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIw"}, + {"type": "compaction", "content": ""}, + {"type": "text", "text": ""}, + ] + + assert _count_content_list(charges_for_empty_text, empty_blocks, False, None) == 0 + assert _count_content_list(charges_for_empty_text, [{"type": "thinking", "thinking": "two words"}], False, None) == 3 + + def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 1c004d8c36e..8fa9d534a8c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -24,7 +24,7 @@ from litellm.llms.tencent.messages.transformation import TencentAnthropicMessage from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) -from litellm.types.llms.anthropic import AnthropicFinishReason +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicFinishReason from litellm.types.router import GenericLiteLLMParams from tests.unit.llms.anthropic.test_anthropic_wif import ( ScriptedPoster, @@ -321,6 +321,66 @@ def _derived_beta_header(config: AnthropicMessagesConfig) -> dict: ) +_COMPACT_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value +_CLEARING_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + + +@pytest.mark.parametrize( + ("context_management", "betas"), + ( + pytest.param({"edits": [{"type": "compact_20260112"}]}, {_COMPACT_EDIT_BETA}, id="compact"), + pytest.param({"edits": [{"type": "clear_tool_uses_20250919"}]}, {_CLEARING_EDIT_BETA}, id="clearing"), + pytest.param( + {"edits": [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}]}, + {_COMPACT_EDIT_BETA, _CLEARING_EDIT_BETA}, + id="both", + ), + pytest.param( + [{"type": "compaction", "compact_threshold": 150000}], {_COMPACT_EDIT_BETA}, id="openai_list_form" + ), + ), +) +def test_context_management_edits_add_the_beta_each_edit_needs( + context_management: JsonValue, betas: set[str], local_beta_headers_config: None +) -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( + headers={}, optional_params={"context_management": context_management}, messages=[] + ) + + assert set(headers["anthropic-beta"].split(",")) == betas + + +_UNMAPPABLE_LIST_FORM: Final = [{"type": "clear_tool_uses_20250919"}] + + +@pytest.mark.parametrize("config_class", [AnthropicMessagesConfig, VertexAIPartnerModelsAnthropicMessagesConfig]) +@pytest.mark.parametrize( + ("context_management", "drop_params", "forwarded"), + ( + pytest.param(_UNMAPPABLE_LIST_FORM, True, None, id="unmappable_dropped_under_drop_params"), + pytest.param(_UNMAPPABLE_LIST_FORM, False, _UNMAPPABLE_LIST_FORM, id="unmappable_kept_for_the_provider_400"), + pytest.param( + [{"type": "compaction"}], + True, + {"edits": [{"type": "compact_20260112"}]}, + id="mappable_still_mapped_under_drop_params", + ), + ), +) +def test_context_management_the_list_form_cannot_map_is_dropped_only_under_drop_params( + config_class: type[AnthropicMessagesConfig], context_management: JsonValue, drop_params: bool, forwarded: JsonValue +) -> None: + request: Final = config_class().transform_anthropic_messages_request( + model="claude-haiku-5-5", + messages=[{"role": "user", "content": "Hi"}], + anthropic_messages_optional_request_params={"max_tokens": 50, "context_management": context_management}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request.get("context_management") == forwarded + + @pytest.mark.parametrize("config", _classifier_configs(), ids=type) def test_routes_whose_platform_runs_the_classifier_keep_safeguards_and_the_beta( config: AnthropicMessagesConfig, local_beta_headers_config: None diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index a3e723eaca2..aa7fa2727b4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -26,7 +26,24 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse +from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + +COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value +COMPACTED_HISTORY = [ + { + "role": "assistant", + "content": [ + { + "type": "compaction", + "content": "The user is building a recipe app and asked for one-sentence class descriptions.", + "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40, + } + ], + }, + {"role": "user", "content": "Now do the same for Ingredient."}, +] @pytest.mark.parametrize( @@ -890,3 +907,115 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat inference["tool_choice"], inference["tools"], ) + + +def test_headers_add_the_compaction_beta_only_for_a_compacted_history(): + config = AnthropicCountTokensConfig() + auth_header = {"x-api-key": "sk-ant-api03-test-key"} + + with_block = config.get_count_tokens_headers(auth_header, COMPACTED_HISTORY)["anthropic-beta"].split(",") + plain = config.get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])["anthropic-beta"] + + assert COMPACTION_BETA in with_block + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in with_block + assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION + + +@pytest.mark.asyncio +async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_transport_clients): + with respx.mock: + route = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 404}) + ) + result = await AnthropicCountTokensHandler().handle_count_tokens_request( + model="claude-sonnet-5-5", + messages=COMPACTED_HISTORY, + auth_header={"x-api-key": "sk-ant-api03-test-key"}, + api_base="https://gateway.example", + ) + + assert result == {"input_tokens": 404} + betas = route.calls.last.request.headers["anthropic-beta"].split(",") + assert COMPACTION_BETA in betas + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in betas + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize( + ("context_management", "forwarded", "sent_beta", "unsent_beta"), + ( + pytest.param( + {"edits": [{"type": "compact_20260112"}]}, + {"edits": [{"type": "compact_20260112"}]}, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + id="compact", + ), + pytest.param( + {"edits": [{"type": "clear_tool_uses_20250919"}]}, + {"edits": [{"type": "clear_tool_uses_20250919"}]}, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + id="clearing", + ), + pytest.param( + [{"type": "compaction", "compact_threshold": 150000}], + {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]}, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + id="openai_list_form", + ), + ), +) +async def test_counter_forwards_context_management_with_the_beta_inference_sends( + context_management: JsonValue, forwarded: JsonValue, sent_beta: str, unsent_beta: str +) -> None: + """Anthropic counts an unsigned compaction block only under the compact_20260112 edit that inference sends, + and a clearing edit changes the count, so the count carries the request's context_management as inference + sends it, the OpenAI list form mapped to edits, and its beta.""" + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 259}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Recipe app summary."}]}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = sent.headers["anthropic-beta"].split(",") + assert result is not None and result.total_tokens == 259 + assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded + assert (sent_beta in betas, unsent_beta in betas) == (True, False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list")) +async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None: + """Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count + leaves it out and adds no context-management beta.""" + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 11}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "user", "content": "Hello"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(",")) + assert result is not None and result.total_tokens == 11 + assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content) + assert not betas & { + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } diff --git a/tests/unit/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py index b8a7c001b37..3e3b408d825 100644 --- a/tests/unit/llms/anthropic/test_count_tokens_oauth.py +++ b/tests/unit/llms/anthropic/test_count_tokens_oauth.py @@ -48,7 +48,7 @@ FEDERATED_DEPLOYMENT = { def count_tokens_headers_for(api_key: str) -> dict[str, str]: auth_header = AnthropicModelInfo.get_auth_header(api_key=api_key) assert auth_header is not None - return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header) + return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}]) @pytest.fixture diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index 38eb87debfd..67733884e12 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -666,7 +666,8 @@ async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_whe *_agent_turns(2, first_round=3), ) - view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact)) + with patch.object(litellm, "token_counter", side_effect=RuntimeError("tokenizer unavailable")): + view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact)) assert isinstance(view, InferenceRequestView) assert _tool_outputs_kept(view) == (True, True) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 1e7c3d52814..7f16957ddc4 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -526,6 +526,29 @@ def test_both_compact_and_context_management_headers_added(): ) +def test_openai_list_form_context_management_sends_the_compact_beta(): + config: Final = VertexAIPartnerModelsAnthropicMessagesConfig() + + with ( + patch.object(config, "_ensure_access_token", return_value=("token", "test-project")), + patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"), + ): + updated_headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]}, + litellm_params={ + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + }, + api_base=None, + ) + + assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value] + + def test_validate_environment_always_refreshes_token_ignoring_stale_bearer(): """Regression: stale Authorization in shared deployment extra_headers must not skip token refresh on /v1/messages — _ensure_access_token is always called.""" diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index f0317af915f..604fc84a096 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -6,6 +6,7 @@ import json import logging import unittest from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -587,6 +588,48 @@ class TestErrorLogCarriesCallId: assert call_id in record.getMessage() + +class TestCountTokensCompactedHistory: + """A history carrying Anthropic's signed compaction block used to fail /v1/messages/count_tokens once the count + ran on the local tokenizer, which rejected the block type.""" + + @pytest.mark.asyncio + async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self): + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + summary = "The user is building a recipe app and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + compacted = { + "model": "claude-sonnet-5-5", + "messages": [ + { + "role": "assistant", + "content": [{"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}], + }, + follow_up, + ], + } + as_text = { + "model": "claude-sonnet-5-5", + "messages": [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up], + } + request = MagicMock() + request.headers = {} + + async def count(body): + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "llm_router", None), # test-quality-ok: module global; with no router the count runs on the local tokenizer, the path that raised + ): + return await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + counted = await count(compacted) + + assert counted["input_tokens"] > 0 + assert counted == await count(as_text) + def _claude_router() -> Router: return Router( model_list=[ @@ -759,7 +802,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e def _count_on_claude( - monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str] + monkeypatch: pytest.MonkeyPatch, + peer: Callable[[httpx.Request], httpx.Response], + headers: Mapping[str, str], + extra_body: Mapping[str, object] = MappingProxyType({}), ) -> httpx.Response: from fastapi import FastAPI @@ -799,7 +845,7 @@ def _count_on_claude( app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth() return TestClient(app).post( "/v1/messages/count_tokens", - json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]}, + json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body}, headers=dict(headers), ) @@ -827,6 +873,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( ) +def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None: + applied: Final = {"original_input_tokens": 640} + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response( + 200, json={"input_tokens": 576, "context_management": applied}, request=request + ), + {}, + {"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, + ) + + assert response.json() == { + "input_tokens": 576, + "context_management": applied, + "litellm_count_source": "provider", + }, response.text + + def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error( monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] ) -> None: From cfb77ecc3ad72d9f3602dbda22d78d7e84739289 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:14:49 -0700 Subject: [PATCH 28/34] fix(proxy): list only routable models for a wildcard alias over a provider org (#49152) * fix(proxy): list only routable models for a wildcard alias over a provider org A wildcard alias over an org-scoped deployment (together_ai/* over together_ai/nosuchorg/*) listed the provider's whole catalog on /v1/models and /model/info, and every listed id routed to a model the provider 404s. A skipped deployment path segment now counts as a routing prefix only when the routed model resolves to a catalog id through the get_model_info lookup order, so responses/, Bedrock converse/, and region paths list as before. Together's model list now also carries the cost map's together_ai rows, skipping the together-ai-* pricing tiers, so an alias over a real org lists the models Together serves today. * fix(proxy): keep flat catalog ids under routes the cost map can't parse, drop retired Together rows * fix(proxy): check skipped path segments only on Together so Hugging Face provider routes keep listing --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 5 ++ litellm/constants.py | 2 +- litellm/proxy/auth/model_checks.py | 34 ++++++++- litellm/responses/main.py | 8 +- litellm/utils.py | 46 ++++++------ tests/unit/proxy/auth/test_model_checks.py | 87 +++++++++++++++++++++- 6 files changed, 150 insertions(+), 32 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index d59edcaff89..5f14c605cd0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -872,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) + elif value.get("litellm_provider") == "together_ai": + if _is_past_deprecation_date(value): + together_ai_models.discard(key) + elif not key.startswith("together-ai-"): + together_ai_models.add(key) elif value.get("litellm_provider") == "baseten": if _is_past_deprecation_date(value): baseten_models.discard(key) diff --git a/litellm/constants.py b/litellm/constants.py index cfee777f36a..e7c9c5dcbfb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1274,7 +1274,7 @@ empower_models: Final = set( ] ) -together_ai_models: Final[set] = set( +together_ai_models: Final[set[str]] = set( [ # llama llms - chat "togethercomputer/llama-2-70b-chat", diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c40ee190e12..b21315529eb 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -14,14 +14,16 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository +from litellm.responses.main import strip_responses_routing_prefix from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models +from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) +_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value}) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -331,6 +333,25 @@ def _routes_back_to( ) +def _resolves_to_catalog_model(model: str, provider: str, provider_model_ids: frozenset[str]) -> bool: + provider_id_prefix: Final = f"{provider}/" + resolved_model: Final = ( + f"{provider_id_prefix}{strip_responses_routing_prefix(model.removeprefix(provider_id_prefix))}" + ) + return any( + name.removeprefix(provider_id_prefix) in provider_model_ids + for name in potential_model_names_in_lookup_order(resolved_model, provider) + ) + + +def _routes_to_catalog_model( + routes_to_model: Callable[[str, str], bool], provider: str, provider_model_ids: frozenset[str] +) -> Callable[[str, str], bool]: + return lambda listed_id, model: ( + routes_to_model(listed_id, model) and _resolves_to_catalog_model(model, provider, provider_model_ids) + ) + + def _ids_routing_to_catalog_models( name_pattern: str, deployment_pattern: str, @@ -344,8 +365,15 @@ def _ids_routing_to_catalog_models( return () provider_id_prefix: Final = f"{provider}/" provider_model_ids: Final = tuple(catalog_id.removeprefix(provider_id_prefix) for catalog_id in catalog_ids) + deployment_model_pattern_without_provider: Final = deployment_pattern.removeprefix(provider_id_prefix) catalog_pattern: Final = _without_routing_segments( - deployment_pattern.removeprefix(provider_id_prefix), provider, name_pattern, provider_model_ids + deployment_model_pattern_without_provider, provider, name_pattern, provider_model_ids + ) + routing_prefix: Final = deployment_model_pattern_without_provider.removesuffix(catalog_pattern) + reaches_catalog_model: Final = ( + _routes_to_catalog_model(routes_to_model, provider, frozenset(provider_model_ids)) + if routing_prefix and provider in _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID + else routes_to_model ) deployment_model_pattern: Final = f"{provider_id_prefix}{catalog_pattern}" deployment_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex(deployment_model_pattern)) @@ -369,7 +397,7 @@ def _ids_routing_to_catalog_models( for listed_id, catalog_id in listed_ids if listed_id and _routes_back_to( - listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model + listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, reaches_catalog_model ) ) ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 98356e9a5d4..063ef9d91ea 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -759,7 +759,7 @@ def _dispatched_model_name( from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider) - return _strip_responses_routing_prefix(provider_model) + return strip_responses_routing_prefix(provider_model) def _will_bridge_to_chat_completions( @@ -1245,7 +1245,7 @@ def _pop_use_chat_completions_api_kw(kwargs: dict[str, object]) -> bool: _RESPONSES_ROUTING_PREFIX: Final = "responses/" -def _strip_responses_routing_prefix(model: str) -> str: +def strip_responses_routing_prefix(model: str) -> str: if not model.startswith(_RESPONSES_ROUTING_PREFIX): return model return model[len(_RESPONSES_ROUTING_PREFIX) :] @@ -1273,7 +1273,7 @@ def _resolve_model_provider_for_responses( litellm_params.api_key = dynamic_api_key if dynamic_api_base is not None: litellm_params.api_base = dynamic_api_base - return _strip_responses_routing_prefix(provider_model), resolved_provider + return strip_responses_routing_prefix(provider_model), resolved_provider def _apply_managed_file_id_mapping( @@ -2893,7 +2893,7 @@ async def _aresponses_websocket( api_key=api_key, litellm_params=litellm_params, ) - resolved_model: Final = _strip_responses_routing_prefix(provider_model) + resolved_model: Final = strip_responses_routing_prefix(provider_model) litellm_params_dict["data_residency"] = infer_openai_data_residency( _custom_llm_provider, diff --git a/litellm/utils.py b/litellm/utils.py index 41a3fddf3a3..e85e089289a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6708,6 +6708,26 @@ def _first_registered_match( return next(matches, (None, None)) +def _model_cost_lookup_order( + model: str, potential_model_names: PotentialModelNamesAndCustomLLMProvider +) -> tuple[str, ...]: + return ( + potential_model_names["combined_model_name"], + model, + potential_model_names["region_free_combined_model_name"], + potential_model_names["split_model"], + potential_model_names["combined_stripped_model_name"], + potential_model_names["stripped_model_name"], + potential_model_names["provider_prefixed_model_name"], + ) + + +def potential_model_names_in_lookup_order(model: str, custom_llm_provider: str) -> tuple[str, ...]: + return _model_cost_lookup_order( + model, get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider) + ) + + def _get_model_info_from_generalization( model: str, potential_model_names: PotentialModelNamesAndCustomLLMProvider, @@ -6726,15 +6746,7 @@ def _get_model_info_from_generalization( still had a priced exact name to try. O(number of rules); only call after the exact lookups have missed. """ - candidates: Final = ( - potential_model_names["combined_model_name"], - model, - potential_model_names["region_free_combined_model_name"], - potential_model_names["split_model"], - potential_model_names["combined_stripped_model_name"], - potential_model_names["stripped_model_name"], - potential_model_names["provider_prefixed_model_name"], - ) + candidates: Final = _model_cost_lookup_order(model, potential_model_names) if any(get_model_cost_key(candidate) is not None for candidate in candidates): return None for candidate in candidates: @@ -6973,11 +6985,6 @@ def get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model) ) - combined_model_name: Final = potential_model_names["combined_model_name"] - stripped_model_name: Final = potential_model_names["stripped_model_name"] - combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"] - provider_prefixed_model_name: Final = potential_model_names["provider_prefixed_model_name"] - split_model: Final = potential_model_names["split_model"] custom_llm_provider = potential_model_names["custom_llm_provider"] model_cost_custom_llm_provider: Final = custom_llm_provider ######################### @@ -7044,16 +7051,9 @@ def get_model_info_helper( and strip it. Tried last so no model that already resolves through 1-6 can change. """ - lookup_order: Final = ( - combined_model_name, - model, - potential_model_names["region_free_combined_model_name"], - split_model, - combined_stripped_model_name, - stripped_model_name, - provider_prefixed_model_name, + lookup: Final = _first_registered_match( + _model_cost_lookup_order(model, potential_model_names), model_cost_custom_llm_provider ) - lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider) key: str | None = lookup[0] _model_info: dict[str, Any] | None = lookup[1] diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 3f0f97e63b0..c67610c24a1 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1057,7 +1057,9 @@ def test_wildcard_lists_each_model_once_under_its_full_owner_id(provider: str): import litellm from litellm.proxy.auth.model_checks import get_known_models_from_wildcard - full_ids: Final = {f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]} + full_ids: Final = { + f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider] + } assert any(model.count("/") > 1 for model in full_ids) assert sorted(get_known_models_from_wildcard(f"{provider}/*")) == sorted(full_ids) @@ -1372,12 +1374,57 @@ def test_partial_wildcard_lists_catalog_ids_stored_with_the_provider_prefix( assert sorted(listed) == expected +@pytest.mark.parametrize( + ("provider", "routing_segment"), [("bedrock", "converse"), ("bedrock", "us-east-1"), ("openai", "responses")] +) +def test_routing_segment_lists_the_same_catalog_ids_as_the_bare_provider_wildcard( + provider: str, routing_segment: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + bare_listing: Final = get_known_models_from_wildcard(f"{provider}/*", LiteLLM_Params(model=f"{provider}/*")) + routed_listing: Final = get_known_models_from_wildcard( + f"{provider}/*", LiteLLM_Params(model=f"{provider}/{routing_segment}/*") + ) + + assert bare_listing + assert sorted(routed_listing) == sorted(bare_listing) + + +def test_provider_wildcard_lists_cost_map_models_but_not_pricing_tiers() -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + model_id: Final = "together_ai/partial-wildcard-org/model-t" + pricing_tier: Final = "together-ai-partial-wildcard-tier" + with _registered_chat_models({model_id: "together_ai", pricing_tier: "together_ai"}): + listed: Final = get_known_models_from_wildcard("together_ai/*") + + assert model_id in listed + assert [listed_id for listed_id in listed if pricing_tier in listed_id] == [] + + +def test_provider_wildcard_stops_listing_a_cost_map_model_once_it_is_retired() -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + model_id: Final = "together_ai/partial-wildcard-org/model-retired" + with _registered_chat_models({model_id: "together_ai"}): + listed_while_served: Final = get_known_models_from_wildcard("together_ai/*") + litellm.add_known_models({model_id: {"litellm_provider": "together_ai", "deprecation_date": "2000-01-01"}}) + listed_once_retired: Final = get_known_models_from_wildcard("together_ai/*") + + assert model_id in listed_while_served + assert model_id not in listed_once_retired + + _ROUTE_CF_ID: Final = "cloudflare/@cf/partial-route-org/model-a-instruct" _ROUTE_CF_SUFFIXED_ID: Final = "cloudflare/@cf/partial-route-org/model-b-instruct-v2" _ROUTE_HF_ID: Final = "cloudflare/@hf/partial-route-org/model-c" _ROUTE_GEMINI_ID: Final = "gemini-partial-route-9-flash" _ROUTE_OPENAI_ID: Final = "gpt-partial-route-9-mini" _ROUTE_BEDROCK_ID: Final = "anthropic.partial-route-9-v1:0" +_ROUTE_TOGETHER_ID: Final = "together_ai/partial-route-org/model-t" _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} @@ -1456,6 +1503,29 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} set(), id="same-name-namespace-the-catalog-lacks", ), + pytest.param( + "together/*", "together_ai/partial-route-missing-org/*", None, set(), id="alias-over-org-the-catalog-lacks" + ), + pytest.param( + "openai/*", + "openai/chat_completions/*", + None, + {f"openai/chat_completions/{_ROUTE_OPENAI_ID}"}, + id="chat-completions-bridge-route", + ), + pytest.param( + "bedrock/*", "bedrock/llama/*", None, {f"bedrock/llama/{_ROUTE_BEDROCK_ID}"}, id="invoke-format-route" + ), + pytest.param( + "together/*", + "together_ai/partial-route-org/*", + None, + {_ROUTE_TOGETHER_ID}, + id="alias-over-org-a-cost-map-catalog-carries", + ), + pytest.param( + "bedrock/*", "bedrock/us-east-1/*", None, {f"bedrock/us-east-1/{_ROUTE_BEDROCK_ID}"}, id="region-route" + ), pytest.param( "*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all" ), @@ -1501,6 +1571,7 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( _ROUTE_GEMINI_ID: "gemini", _ROUTE_OPENAI_ID: "openai", _ROUTE_BEDROCK_ID: "bedrock", + _ROUTE_TOGETHER_ID: "together_ai", } ): v1_models: Final = get_complete_model_list( @@ -1535,6 +1606,20 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( assert listed_model_info_names == sorted(reachable) +def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + plain: Final = get_known_models_from_wildcard("huggingface/*", LiteLLM_Params(model="huggingface/*")) + routed: Final = get_known_models_from_wildcard( + "huggingface/*", LiteLLM_Params(model="huggingface/featherless-ai/*") + ) + + assert plain + assert [model for model in plain if "/" not in model.removeprefix("huggingface/")] == [] + assert sorted(routed) == sorted(plain) + + def test_catch_all_lists_no_id_a_more_specific_sibling_wildcard_routes_elsewhere() -> None: from litellm import Router from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list From e81dd348a425c180c7a632c1eecea7a37682ad96 Mon Sep 17 00:00:00 2001 From: Filip Sinitsin Date: Fri, 30 Oct 2026 07:18:15 +0000 Subject: [PATCH 29/34] fix(proxy)!: log and bill Responses-style bodies sent to /cursor/chat/completions (#30277) * fix(proxy): restore spend logging for /cursor/chat/completions (#30126) The /cursor/chat/completions route processes requests through the Responses API pipeline, but _get_metadata_variable_name did not treat it as a litellm_metadata route. Proxy auth metadata therefore landed in the user-facing metadata param, so the spend tracking callback saw user_api_key=None and silently skipped the SpendLogs write. Add the route to LITELLM_METADATA_ROUTES and cover it with regression tests. * test(proxy): pin provider metadata and key billing on /cursor/chat/completions A Responses-format body on the Cursor route now sends OpenAI only the metadata the client sent and writes a spend row that bills the key. The chat-body branch, which this fix also moves onto litellm_metadata, still bills the key with its team and request tags. The scripted chat reply gets a unique id per test, since a shared id let only the first spend row per database land * fix(proxy): pick the Cursor route's metadata field from the body shape A Responses-style body on /cursor/chat/completions keeps proxy metadata in litellm_metadata, the same as /v1/responses, while a Chat Completions body keeps metadata, the same as /v1/chat/completions. Adding the whole route to the internal-metadata list let a client's chat-body tags replace the key's tags in spend logs and skip tag budgets. * fix(proxy): keep get_metadata_variable_name callable with only the request * fix(proxy): check client tags against tag budgets and read agent limits from litellm_metadata on Responses bodies A Responses-style body on /cursor/chat/completions now keeps its proxy fields in litellm_metadata, so the pre-call tag budget check and the v3 agent rate limiter stopped seeing the client's metadata.tags and the x-litellm-agent-id value. The pre-auth tag check now merges the caller's metadata.tags into the selected metadata field, sharing one helper with request setup, and the limiter reads agent_id from either field * fix(proxy): check client tags sent in a JSON-string metadata against tag budgets on Responses bodies * fix(proxy): answer 422, not 500, when a budget rejects a Responses body whose metadata is a JSON string * fix(proxy): keep key guardrails, metadata type errors and failure-row client details on Responses-style bodies --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 32 ++-- litellm/proxy/auth/auth_checks.py | 10 +- litellm/proxy/common_request_processing.py | 16 +- .../hooks/parallel_request_limiter_v3.py | 3 +- .../proxy/hooks/proxy_track_cost_callback.py | 37 +++- litellm/proxy/litellm_pre_call_utils.py | 88 ++++++++-- .../proxy/response_api_endpoints/endpoints.py | 12 +- litellm/proxy/utils.py | 10 +- .../providers/test_cursor_responses_wire.py | 163 +++++++++++++++--- .../integrations/test_custom_guardrail.py | 14 ++ .../hooks/test_parallel_request_limiter_v3.py | 5 +- .../hooks/test_proxy_track_cost_callback.py | 50 ++++++ .../response_api_endpoints/test_endpoints.py | 2 +- .../proxy/test_common_request_processing.py | 34 ++++ .../unit/proxy/test_litellm_pre_call_utils.py | 68 ++++++++ .../test_post_call_failure_hook.py | 29 ++++ 16 files changed, 482 insertions(+), 91 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 9e2dc902d55..a9fb5c29bad 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -9,6 +9,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args import httpx +from pydantic import TypeAdapter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None = ) +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]: + if not isinstance(metadata, dict): + return [] + return metadata.get("guardrails") or [] + + class CustomGuardrail(CustomLogger): # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path. use_native_during_call_hook: ClassVar[bool] = False @@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger): return True raise - def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]: + def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]: """ - Returns the guardrail(s) to be run from the metadata or root + Returns the guardrail(s) to be run from the root, else from both metadata fields together """ if "guardrails" in data: return data["guardrails"] - # Check both metadata locations. For regular endpoints move_guardrails_to_metadata - # writes to "metadata"; for thread/assistant endpoints it writes to - # "litellm_metadata". We check the one that actually contains the "guardrails" - # key so that a non-empty litellm_metadata without guardrails does not shadow - # the merged list stored in metadata (which would cause team guardrails to be - # silently skipped while default_on=True policy guardrails still fire). - for meta_key in ("metadata", "litellm_metadata"): - meta = data.get(meta_key) or {} - if isinstance(meta, dict) and "guardrails" in meta: - return meta.get("guardrails") or [] - return [] + request_fields: Final = _REQUEST_FIELDS.validate_python(data) + return [ + *_guardrails_listed_in(request_fields.get("metadata")), + *_guardrails_listed_in(request_fields.get("litellm_metadata")), + ] def _guardrail_is_in_requested_guardrails( self, - requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]], + requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]], ) -> bool: for _guardrail in requested_guardrails: if isinstance(_guardrail, dict): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 1ef5b1072ed..b8d9530d2eb 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -446,7 +446,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request, from litellm.proxy.litellm_pre_call_utils import ( _trace_id_from_traceparent, # pyright: ignore[reportPrivateUsage] # reuse W3C traceparent parser get_chain_id_from_headers, - metadata_variable_name_for_route, + metadata_variable_name_for_request, ) if get_chain_id_from_headers(dict(request.headers)): @@ -460,7 +460,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request, traceparent: Final[str | None] = request.headers.get("traceparent") if isinstance(traceparent, str) and _trace_id_from_traceparent(traceparent): return True - metadata_variable_name: Final = metadata_variable_name_for_route(get_request_route(request)) + metadata_variable_name: Final = metadata_variable_name_for_request(get_request_route(request), request_body) selected_metadata: Final[object] = request_body.get(metadata_variable_name) if isinstance(selected_metadata, dict): selected_metadata_dict: Final = _TRACE_ID_METADATA_ADAPTER.validate_python(selected_metadata) @@ -1367,11 +1367,15 @@ async def common_checks( if not skip_all_budget_checks: # Key metadata.tags are injected into request_body here so the tag budget # check can read them; this mutation must run before the gathered checks. - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request if valid_token is not None: for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_body): LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route) + LiteLLMProxyRequestSetup.merge_caller_body_tags( + request_data=tag_target, + metadata_variable_name=metadata_variable_name_for_request(route, tag_target), + ) LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(request_data=tag_target, user_api_key_dict=valid_token) LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_body) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7cd752c2297..a79d8fe29db 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2401,9 +2401,10 @@ class ProxyBaseLLMRequestProcessing: version=version, proxy_config=proxy_config, ) + request_fields: Final = _REQUEST_FIELDS.validate_python(self.data) + metadata_variable_name: Final = get_metadata_variable_name(request, request_fields) if logging_auth is not None: - metadata_variable_name: Final = get_metadata_variable_name(request) - request_metadata: Final = self.data.get(metadata_variable_name) + request_metadata: Final = request_fields.get(metadata_variable_name) if isinstance(request_metadata, dict): self.data[metadata_variable_name] = { **TypeAdapter(dict[str, object]).validate_python(request_metadata), @@ -2431,12 +2432,11 @@ class ProxyBaseLLMRequestProcessing: # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved if queue_time_seconds is not None: - _metadata_variable_name: Final = get_metadata_variable_name(request) - if _metadata_variable_name not in self.data: - self.data[_metadata_variable_name] = {} - if not isinstance(self.data[_metadata_variable_name], dict): - self.data[_metadata_variable_name] = {} - self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds + if metadata_variable_name not in self.data: + self.data[metadata_variable_name] = {} + if not isinstance(self.data[metadata_variable_name], dict): + self.data[metadata_variable_name] = {} + self.data[metadata_variable_name]["queue_time_seconds"] = queue_time_seconds if isinstance(model, str): reject_url_valued_destination("model", model) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 14b08a09116..15282679622 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -3312,7 +3312,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if key_agent_id: return key_agent_id metadata: Final = data.get("metadata") or {} - return metadata.get("agent_id") + litellm_metadata: Final = data.get("litellm_metadata") or {} + return metadata.get("agent_id") or litellm_metadata.get("agent_id") def _get_session_id_from_data(self, data: dict) -> str | None: """Extract session_id from request metadata or litellm_session_id.""" diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7d8b2e8329c..9c4a4bc775b 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -37,7 +37,7 @@ from litellm.proxy.db.db_spend_update_writer import ( get_llm_router, ) from litellm.proxy.hooks.search_free_tier import apply_search_free_tier -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope from litellm.proxy.spend_tracking.spend_event import ( @@ -137,19 +137,34 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( _OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping) -def _proxy_stamped_used_client_oauth_token( - request_data: Mapping[str, object], request_route: str | None -) -> bool | None: +def _proxy_stamped_metadata(request_data: Mapping[str, object], request_route: str | None) -> Mapping[str, object]: proxy_bucket: Final = ( get_metadata_variable_name_from_kwargs(request_data) if request_route is None - else metadata_variable_name_for_route(request_route) + else metadata_variable_name_for_request(request_route, request_data) ) - proxy_metadata: Final = request_data.get(proxy_bucket) - stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None + return _str_keyed_fields(request_data.get(proxy_bucket)) + + +def _proxy_stamped_used_client_oauth_token( + request_data: Mapping[str, object], request_route: str | None +) -> bool | None: + stamped: Final = _proxy_stamped_metadata(request_data, request_route).get("used_client_oauth_token") return stamped if isinstance(stamped, bool) else None +_PROXY_STAMPED_REQUEST_FIELDS: Final = ("requester_ip_address", "user_agent") + + +def _proxy_stamped_request_fields( + request_data: Mapping[str, object], request_route: str | None +) -> Mapping[str, object]: + if request_route is None: + return {} + proxy_metadata: Final = _proxy_stamped_metadata(request_data, request_route) + return {field: proxy_metadata[field] for field in _PROXY_STAMPED_REQUEST_FIELDS if field in proxy_metadata} + + def _str_keyed_fields(value: object) -> Mapping[str, object]: return value if is_str_keyed_mapping(value) else {} @@ -455,7 +470,13 @@ class _ProxyDBLogger(CustomLogger): _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route) existing_metadata: Final[dict] = request_data.get("metadata", None) or {} - existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata}) + existing_metadata.update( + { + **_routed_metadata_fallback(request_fields), + **_proxy_stamped_request_fields(request_fields, request_route), + **_metadata, + } + ) litellm_metadata_bucket: Final = request_data.get("litellm_metadata") litellm_metadata: Final[ObjectMapping] = ( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a87fc815402..c1e541d20a9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -379,6 +379,8 @@ LITELLM_METADATA_ROUTES: Final = ( "interactions", ) +CURSOR_CHAT_COMPLETIONS_ROUTE: Final = "/cursor/chat/completions" + LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( { "mask_input", @@ -636,7 +638,7 @@ def _has_tag_budget(tag: "LiteLLM_TagTable | None") -> bool: return tag.litellm_budget_table.max_budget is not None -def _normalized_metadata_slot( +def normalized_metadata_slot( request_data: MutableMapping[str, object], metadata_variable_name: str ) -> dict[str, object]: """Return the request's metadata slot as a dict, normalising it in place first. @@ -653,6 +655,12 @@ def _normalized_metadata_slot( return normalized +def _holds_metadata_object(value: object) -> bool: + if value is None or isinstance(value, dict): + return True + return isinstance(value, str) and isinstance(_DECODED_JSON.validate_python(safe_json_loads(value)), dict) + + def _metadata_object_or_error(field: str, value: object) -> Mapping[str, object] | ProxyException: if value is None: return {} @@ -863,7 +871,7 @@ def _strip_router_reserved_metadata( ) -def get_metadata_variable_name(request: Request) -> str: +def get_metadata_variable_name(request: Request, data: Mapping[str, object] | None = None) -> str: """ Helper to return what the "metadata" field should be called in the request data @@ -874,7 +882,10 @@ def get_metadata_variable_name(request: Request) -> str: # Inline imports — auth_utils/route_checks participate in a proxy import cycle. from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 - return metadata_variable_name_for_route(get_request_route(request)) + route: Final = get_request_route(request) + if data is None: + return metadata_variable_name_for_route(route) + return metadata_variable_name_for_request(route, data) _get_metadata_variable_name: Final = get_metadata_variable_name @@ -890,6 +901,25 @@ def metadata_variable_name_for_route(route: str) -> Literal["metadata", "litellm return "metadata" +def is_chat_completions_body(data: Mapping[str, object]) -> bool: + messages: Final = data.get("messages") + if isinstance(messages, list) and messages: + return True + return "messages" in data and "input" not in data + + +def is_cursor_responses_request(route: str, body: Mapping[str, object]) -> bool: + return CURSOR_CHAT_COMPLETIONS_ROUTE in route and not is_chat_completions_body(body) + + +def metadata_variable_name_for_request( + route: str, body: Mapping[str, object] +) -> Literal["metadata", "litellm_metadata"]: + if is_cursor_responses_request(route, body): + return "litellm_metadata" + return metadata_variable_name_for_route(route) + + def _promoted_trace_control_fields( requester_metadata: Mapping[str, object], litellm_metadata: Mapping[str, object], @@ -2309,7 +2339,9 @@ class LiteLLMProxyRequestSetup: ``_tag_max_budget_check`` resolve to ``litellm_metadata``, leaving header tags invisible to per-tag budget enforcement. """ - if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES): + if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES) or is_cursor_responses_request( + route, request_data + ): request_data.setdefault("litellm_metadata", {}) @staticmethod @@ -2380,7 +2412,7 @@ class LiteLLMProxyRequestSetup: return tags: Final = _STRING_TAGS_ADAPTER.validate_python(raw_tags) slot: Final = metadata_key if metadata_key is not None else get_metadata_variable_name_from_kwargs(request_data) - metadata: Final = _normalized_metadata_slot(request_data, slot) + metadata: Final = normalized_metadata_slot(request_data, slot) existing: Final = metadata.get("tags") if existing and not isinstance(existing, list): return @@ -2393,6 +2425,34 @@ class LiteLLMProxyRequestSetup: tags=tuple(tag for tag in tags if tag not in sent_tags) ) + @staticmethod + def merge_caller_body_tags( + request_data: MutableMapping[str, object], # mutable-ok: the request body the tag merges write into + metadata_variable_name: str | None = None, + ) -> None: + slot: Final = ( + metadata_variable_name + if metadata_variable_name is not None + else get_metadata_variable_name_from_kwargs(request_data) + ) + if slot == "metadata" or not _holds_metadata_object(request_data.get(slot)): + return + raw_caller_metadata: Final = request_data.get("metadata") + caller_metadata: Final = ( + _DECODED_JSON.validate_python(safe_json_loads(raw_caller_metadata)) + if isinstance(raw_caller_metadata, str) + else raw_caller_metadata + ) + caller_tags: Final = caller_metadata.get("tags") if isinstance(caller_metadata, dict) else None + if not isinstance(caller_tags, list) or not caller_tags: + return + metadata: Final = normalized_metadata_slot(request_data, slot) + existing_tags: Final = metadata.get("tags") + metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags( + request_tags=existing_tags if isinstance(existing_tags, list) else None, + tags_to_add=caller_tags, + ) + @staticmethod def apply_key_tags_pre_auth( request_data: dict, @@ -2408,7 +2468,7 @@ class LiteLLMProxyRequestSetup: return _metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_data) - metadata: Final = _normalized_metadata_slot(request_data, _metadata_variable_name) + metadata: Final = normalized_metadata_slot(request_data, _metadata_variable_name) existing_tags: Final = metadata.get("tags") metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags( @@ -2460,7 +2520,7 @@ class LiteLLMProxyRequestSetup: # from (litellm_metadata vs metadata) so the merged tags are visible # to _tag_max_budget_check. _metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_data) - metadata: Final = _normalized_metadata_slot(request_data, _metadata_variable_name) + metadata: Final = normalized_metadata_slot(request_data, _metadata_variable_name) existing_tags: Final = metadata.get("tags") metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags( @@ -2637,7 +2697,7 @@ async def add_litellm_data_to_request( } safe_add_api_version_from_query_params(data, request) - _metadata_variable_name: Final = get_metadata_variable_name(request) + _metadata_variable_name: Final = get_metadata_variable_name(request, _REQUEST_FIELDS.validate_python(data)) if data.get(_metadata_variable_name, None) is None: data[_metadata_variable_name] = {} @@ -2974,17 +3034,7 @@ async def add_litellm_data_to_request( tags_to_add=tags, ) - _caller_body_metadata: Final = data.get("metadata") if _metadata_variable_name != "metadata" else None - _caller_body_tags: Final = ( - _caller_body_metadata.get("tags") - if isinstance(_caller_body_metadata, dict) and isinstance(_caller_body_metadata.get("tags"), list) - else None - ) - if _caller_body_tags: - data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup.merge_tags( # rebind-ok: matches file idiom - request_tags=data[_metadata_variable_name].get("tags"), - tags_to_add=_caller_body_tags, - ) + LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data, metadata_variable_name=_metadata_variable_name) # caller_tags: the tags the request's headers and body carry (x-litellm-tags, body "tags", and body # "metadata.tags" / "litellm_metadata.tags" read before the mapped tags are added below), diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9d0d89f681e..ef59deb2e5e 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -79,6 +79,7 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + is_chat_completions_body, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( request_tags_from_request_data, @@ -213,13 +214,6 @@ def _normalize_tool_dialect( return {**data, **{key: value for key, value in replaceable if key in data}} -def _is_chat_completions_body(data: Mapping[str, object]) -> bool: - messages: Final = data.get("messages") - if isinstance(messages, list) and messages: - return True - return "messages" in data and "input" not in data - - _CURSOR_THINKING_SEPARATOR: Final = "-thinking-" _CURSOR_FAST_SUFFIX: Final = "-fast" _CURSOR_THINKING_LEVELS: Final[frozenset[str]] = frozenset(get_args(REASONING_EFFORT)) @@ -261,7 +255,7 @@ def _resolve_cursor_model_variant( resolved: Final = {**data, "model": variant.base_model} if variant.reasoning_effort is None: return resolved - if _is_chat_completions_body(data): + if is_chat_completions_body(data): if "reasoning_effort" in data: return resolved return {**resolved, "reasoning_effort": variant.reasoning_effort} @@ -646,7 +640,7 @@ async def cursor_chat_completions( raw_body: Final = await read_request_body(request=request) - if _is_chat_completions_body(raw_body): + if is_chat_completions_body(raw_body): # Genuine chat completions body (Cursor sends these for models whose BYOK it # already fixed); delegate so behavior matches /chat/completions exactly. # Keyed on messages CONTENT, not key presence: Cursor can send a null or diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e125b95a68..b1055bb9124 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -250,7 +250,11 @@ from litellm.proxy.hooks.sensitive_data_routing import ( # noqa: F401, RUF100 PROXY_SensitiveDataRoutingHandler, _PROXY_SensitiveDataRoutingHandler, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_guardrails_from_auth_metadata +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + add_guardrails_from_auth_metadata, + normalized_metadata_slot, +) from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at from litellm.proxy.policy_engine.pipeline_executor import ( PipelineExecutor, @@ -3973,9 +3977,7 @@ class ProxyLogging: ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) - if "metadata" not in request_data: - request_data["metadata"] = {} - request_data["metadata"].update(user_api_key_logged_metadata) + normalized_metadata_slot(request_data, "metadata").update(user_api_key_logged_metadata) if litellm_logging_obj is not None: ## UPDATE LOGGING INPUT diff --git a/tests/integration/providers/test_cursor_responses_wire.py b/tests/integration/providers/test_cursor_responses_wire.py index d77d35f2ff8..1bd79b5b575 100644 --- a/tests/integration/providers/test_cursor_responses_wire.py +++ b/tests/integration/providers/test_cursor_responses_wire.py @@ -3,11 +3,14 @@ from __future__ import annotations import json import os import uuid +from collections.abc import Mapping +from hashlib import sha256 from importlib.resources import files from typing import Final +import httpx import pytest -from integration._support.client import Gateway, Scenario +from integration._support.client import Gateway, Scenario, eventually, list_value, object_value from integration._support.responses_vendor import same_response from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.worker_readiness import wait_for_worker_readiness @@ -38,6 +41,7 @@ _CHAT_SUFFIXES: Final[tuple[tuple[str, str], ...]] = (("", ""),) + tuple( (f"-thinking-{level}", level) for level in _CHAT_THINKING_LEVELS ) _PROMPT: Final = "Inspect the requested change." +_REQUEST_TAG: Final = "cursor-integration" _INPUT: Final = [{"role": "user", "content": [{"type": "input_text", "text": _PROMPT}]}] _CUSTOM_TOOL: Final = { "type": "custom", @@ -107,16 +111,30 @@ def _stream(identity: str) -> tuple[bytes, ...]: ) + (b"data: [DONE]\n\n",) -def _chat_completion(stream: bool) -> bytes | tuple[bytes, ...]: +def _chat_completion(stream: bool, identity: str) -> bytes | tuple[bytes, ...]: if stream: - return ( - b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Patch inspected."},"finish_reason":null}]}\n\n', - b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', - b"data: [DONE]\n\n", - ) + return tuple( + f"data: {json.dumps(chunk)}\n\n".encode() + for chunk in ( + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _MODEL, + "choices": [{"index": 0, "delta": {"content": "Patch inspected."}, "finish_reason": None}], + }, + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _MODEL, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + ) + (b"data: [DONE]\n\n",) return json.dumps( { - "id": "chat_cursor_1", + "id": identity, "object": "chat.completion", "created": 1, "model": _MODEL, @@ -133,11 +151,12 @@ def _chat_completion(stream: bool) -> bytes | tuple[bytes, ...]: ).encode() -def _deployment(scenario: Scenario, wire: Wire) -> str: +def _deployment(scenario: Scenario, wire: Wire, **pricing: JsonValue) -> str: model: Final = scenario.model( model=f"openai/{_MODEL}", api_key=_API_KEY, api_base=f"{wire.url}/v1", + **pricing, ) wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS) return model @@ -167,7 +186,7 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( assert request.method == "POST" assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) if request.target == "/v1/chat/completions": - completion: Final = _chat_completion(stream) + completion: Final = _chat_completion(stream, raw_id) return Reply( content_type="text/event-stream" if stream else "application/json", chunks=completion if isinstance(completion, tuple) else None, @@ -175,7 +194,7 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( ) assert request.target == "/v1/responses", request.target body: Final = _JSON_OBJECT.validate_json(request.body) - assert {key: value for key, value in body.items() if key != "metadata"} == { + assert body == { "model": _MODEL, "input": _INPUT, **reasoning, @@ -259,41 +278,69 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( assert [request.target for request in drain_contract_requests(wire)] == ["/v1/responses"] -def test_cursor_responses_body_carries_no_proxy_metadata(gateway: Gateway) -> None: - pytest.skip( - "BUG: /cursor/chat/completions forwards proxy-internal metadata (key hash, team, user, client ip, user agent) " - "to the provider in the Responses body" - ) +@pytest.mark.parametrize( + "client_metadata", ({}, {"cursor_conversation": "integration"}), ids=("no_client_metadata", "client_metadata") +) +@pytest.mark.parametrize("stream", (False, True)) +def test_cursor_responses_body_sends_only_client_metadata_and_bills_the_key( + gateway: Gateway, + stream: bool, + client_metadata: Mapping[str, JsonValue], +) -> None: raw_id: Final = f"resp_cursor_metadata_{uuid.uuid4().hex}" + alias: Final = f"integration-cursor-{uuid.uuid4().hex}" + sent_metadata: Final = {"metadata": client_metadata} if client_metadata else {} def respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return model_discovery_reply(_MODEL) assert request.method == "POST" and request.target == "/v1/responses", request.target + if stream: + return Reply(content_type="text/event-stream", chunks=_stream(raw_id)) return Reply(body=json.dumps(_response(raw_id)).encode()) with wire_server(respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire) + key: Final = scenario.key(key_alias=alias) + hashed_key: Final = sha256(key.encode()).hexdigest() response: Final = gateway.request( "POST", "/cursor/chat/completions", { - "model": f"{model}-thinking-high", + "model": model, "input": _INPUT, "messages": [], "tools": [_CUSTOM_TOOL], + "stream": stream, + **sent_metadata, }, + key=key, ) assert response.status_code == 200, response.text requests: Final = drain_contract_requests(wire) assert len(requests) == 1, requests - body: Final = _JSON_OBJECT.validate_json(requests[0].body) - assert body == { + assert _JSON_OBJECT.validate_json(requests[0].body) == { "model": _MODEL, "input": _INPUT, - "reasoning": {"effort": "high"}, "tools": [_CUSTOM_TOOL], + "stream": stream, + **sent_metadata, }, requests[0].body + (row,) = eventually( + lambda: gateway.rows( + 'SELECT call_type, spend, metadata FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (hashed_key,) + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert row["call_type"] == "aresponses", row + assert object_value(row["metadata"])["user_api_key_alias"] == alias, row + assert isinstance(row["spend"], float) and row["spend"] > 0, row + eventually( + lambda: gateway.rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed_key,)), + lambda tokens: len(tokens) == 1 and tokens[0]["spend"] == pytest.approx(row["spend"]), + seconds=70, + ) def test_cursor_stream_reports_usage_when_the_client_asks_for_it(gateway: Gateway) -> None: @@ -341,10 +388,12 @@ def test_cursor_messages_body_uses_chat_completions_wire( suffix: str, reasoning_effort: str, ) -> None: + chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}" + def respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return model_discovery_reply(_MODEL) - completion: Final = _chat_completion(stream) + completion: Final = _chat_completion(stream, chat_id) return Reply( content_type="text/event-stream" if stream else "application/json", chunks=completion if isinstance(completion, tuple) else None, @@ -367,6 +416,7 @@ def test_cursor_messages_body_uses_chat_completions_wire( "stream": stream, }, key=key, + headers={"x-litellm-tags": _REQUEST_TAG}, ) assert response.status_code == 200, response.text if stream: @@ -393,7 +443,7 @@ def test_cursor_messages_body_uses_chat_completions_wire( payload: Final = _JSON_OBJECT.validate_json(response.content) identity: Final = payload.get("id") created: Final = payload.get("created") - assert isinstance(identity, str) and same_response(identity, "chat_cursor_1"), response.text + assert isinstance(identity, str) and same_response(identity, chat_id), response.text assert isinstance(created, int) and created > 0, response.text assert payload == { "id": identity, @@ -456,3 +506,72 @@ def test_cursor_messages_body_uses_chat_completions_wire( **({"stream_options": {"include_usage": True}} if stream else {}), **({"reasoning_effort": reasoning_effort} if reasoning_effort else {}), }, requests[0].body + (row,) = eventually( + lambda: gateway.rows( + 'SELECT call_type, team_id, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert (row["call_type"], row["team_id"]) == ("acompletion", team), row + assert _REQUEST_TAG in list_value(row["request_tags"]), row + + +@pytest.mark.parametrize( + ("body_shape", "metadata_as_json_string"), (("messages", False), ("input", False), ("input", True)) +) +def test_cursor_body_keeps_client_and_key_tags_and_enforces_the_client_tag_budget( + gateway: Gateway, body_shape: str, metadata_as_json_string: bool +) -> None: + chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}" + response_id: Final = f"resp_cursor_tags_{uuid.uuid4().hex}" + budget_tag: Final = f"cursor-tag-budget-{uuid.uuid4().hex}" + key_tag: Final = f"cursor-key-tag-{uuid.uuid4().hex}" + client_metadata: Final = {"tags": [budget_tag]} + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.target == "/v1/responses": + return Reply(body=json.dumps(_response(response_id)).encode()) + completion: Final = _chat_completion(False, chat_id) + return Reply(body=completion if isinstance(completion, bytes) else b"") + + def delete_tag() -> None: + gateway.post("/tag/delete", {"name": budget_tag}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, input_cost_per_token=0.01, output_cost_per_token=0.01) + key: Final = scenario.key(models=[model], metadata={"tags": [key_tag]}) + gateway.post("/tag/new", {"name": budget_tag, "max_budget": 0.0001}) + scenario.cleanups.callback(delete_tag) + + def tagged_chat(text: str) -> httpx.Response: + return gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": model, + body_shape: [{"role": "user", "content": text}], + "metadata": json.dumps(client_metadata) if metadata_as_json_string else client_metadata, + }, + key=key, + ) + + first: Final = tagged_chat("first tagged chat") + assert first.status_code == 200, first.text + (row,) = eventually( + lambda: gateway.rows( + 'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert {budget_tag, key_tag} <= set(list_value(row["request_tags"])), row + blocked: Final = eventually( + lambda: tagged_chat("tag budget probe"), lambda response: response.status_code != 200, seconds=70 + ) + assert blocked.status_code == 422, blocked.text + assert "budget" in blocked.text.lower(), blocked.text diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 614ff97553a..0483888533e 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -3699,6 +3699,20 @@ def test_get_guardrail_from_metadata() -> None: ) == [{"test-guardrail": {"extra_body": {"key": "value"}}}] + +@pytest.mark.parametrize( + "caller_metadata", + [{"guardrails": []}, {"guardrails": ["not-a-guardrail"]}, {"guardrails": None}], +) +def test_caller_metadata_guardrails_never_hide_the_proxy_guardrails_in_litellm_metadata(caller_metadata: dict) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="key-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=False + ) + request_data: Final = {"metadata": caller_metadata, "litellm_metadata": {"guardrails": ["key-guardrail"]}} + + assert guardrail.should_run_guardrail(request_data, GuardrailEventHooks.pre_call) is True + + def test_guardrail_is_in_requested_guardrails() -> None: guardrail: Final = CustomGuardrail(guardrail_name="test-guardrail") diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index b66e7f3d091..3aad2bda9bf 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -2591,7 +2591,8 @@ async def test_agent_session_rate_limit_skipped_without_session_id(): @pytest.mark.asyncio -async def test_agent_rate_limit_from_metadata_agent_id(): +@pytest.mark.parametrize("metadata_field", ("metadata", "litellm_metadata")) +async def test_agent_rate_limit_from_metadata_agent_id(metadata_field): """ Test that agent rate limits work when agent_id comes from request metadata (header) rather than from the API key. @@ -2639,7 +2640,7 @@ async def test_agent_rate_limit_from_metadata_agent_id(): cache=local_cache, data={ "model": "gpt-4", - "metadata": {"agent_id": _agent_id}, + metadata_field: {"agent_id": _agent_id}, }, call_type="", ) diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index dc46b5faf4e..8d4f8cd87ad 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -623,6 +623,56 @@ async def test_async_post_call_failure_hook_reads_used_client_oauth_token_from_t assert json.loads(payload["metadata"])["used_client_oauth_token"] is expected + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_route, body", + [ + ( + "/cursor/chat/completions", + { + "input": "Hello", + "metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"}, + "litellm_metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"}, + }, + ), + ( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"}, + "litellm_metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"}, + }, + ), + ], +) +async def test_async_post_call_failure_hook_logs_the_proxy_stamped_ip_and_user_agent(request_route: str, body: dict): + request_data = { + "model": "gpt-4o", + "proxy_server_request": {"request_id": "test_request_id"}, + **body, + } + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await ProxyDBLogger().async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("invalid api key"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", request_route=request_route), + ) + + payload = get_logging_payload( + kwargs=mock_update_database.call_args[1]["kwargs"], + response_obj={}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert payload["requester_ip_address"] == "127.0.0.1" + assert json.loads(payload["metadata"])["user_agent"] == "Cursor/1.0" + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_bills_guardrail_cost_on_blocked_request(): """LIT-5651: a request blocked by a guardrail never reaches the LLM, but the diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index a783ac70881..1e4dded801e 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -3938,7 +3938,7 @@ class TestCursorInputArmFlattening: class TestChatCompletionsBodyDetection: def test_routing_matrix(self): - from litellm.proxy.response_api_endpoints.endpoints import _is_chat_completions_body + from litellm.proxy.litellm_pre_call_utils import is_chat_completions_body as _is_chat_completions_body assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}]}) is True assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}], "input": []}) is True diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index c867db2fd02..2d08609c93b 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9711,6 +9711,40 @@ async def test_saved_weights_override_caller_input_and_preserve_key_precedence( assert logging.pre_call_hook.call_args.kwargs["data"].get("_router_weights") == expected +def _hook_data(**kwargs: object) -> object: + return kwargs["data"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt, proxy_field, client_field", [ + ({"input": "hi"}, "litellm_metadata", "metadata"), + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), +]) +async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks( + prompt: dict[str, str | list[dict[str, str]]], + proxy_field: str, + client_field: str, +) -> None: + from litellm.proxy import proxy_server + + logging: Final = MagicMock(spec=ProxyLogging) + logging.pre_call_hook = AsyncMock(side_effect=_hook_data) + processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt}) + data, _ = await processor.common_processing_pre_call_logic( + request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}), + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash"), + proxy_logging_obj=logging, + proxy_config=proxy_server.ProxyConfig(), + route_type="acompletion", + logging_auth=ProxyUserAPIKeyAuth(api_key="logging-hash", key_alias="logging-alias"), + ) + assert data[proxy_field]["user_api_key_alias"] == "logging-alias" + assert "queue_time_seconds" in data[proxy_field] + assert "user_api_key_alias" not in data.get(client_field, {}) + assert "queue_time_seconds" not in data.get(client_field, {}) + assert data["metadata"]["client"] == "cursor" + class TestPerRequestModelGroupAlias: """``router_settings.model_group_alias`` on a key or team has to be resolved by the proxy: the Router resolves aliases from its own shared instance diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 1ce61a8e9b3..2edb032b048 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -121,6 +121,23 @@ class TestGetMetadataVariableName: request = self._make_request("/chat/completions") assert get_metadata_variable_name(request) == "metadata" + @pytest.mark.parametrize( + "body, expected", + [ + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata"), + ({"messages": []}, "metadata"), + ({"input": [{"role": "user", "content": "hi"}]}, "litellm_metadata"), + ({"input": "hi", "messages": []}, "litellm_metadata"), + ], + ) + def test_cursor_route_picks_the_metadata_field_by_body_shape(self, body, expected): + request = self._make_request("/cursor/chat/completions") + assert get_metadata_variable_name(request, body) == expected + + def test_cursor_route_without_a_body_keeps_the_route_answer(self): + request = self._make_request("/cursor/chat/completions") + assert get_metadata_variable_name(request) == "metadata" + def test_returns_metadata_for_completions(self): request = self._make_request("/v1/completions") assert get_metadata_variable_name(request) == "metadata" @@ -681,6 +698,45 @@ async def test_add_litellm_data_to_request_rejects_non_object_metadata(field, va assert field not in data +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/responses", "/cursor/chat/completions"]) +@pytest.mark.parametrize("value, received_type", [("junk", "a string"), (5, "an integer"), ([1], "an array")]) +async def test_invalid_litellm_metadata_answers_400_when_metadata_carries_tags(route, value, received_type): + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.utils import ProxyLogging + + data = {"model": "gpt-4o", "input": "hi", "metadata": {"tags": ["client-tag"]}, "litellm_metadata": value} + request = _batches_request_mock() + await common_checks( + request_body=data, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=None), + valid_token=UserAPIKeyAuth(api_key="hashed-key"), + request=request, + ) + + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data=data, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert exc_info.value.code == "400" + assert exc_info.value.message == ( + f"Invalid type for 'litellm_metadata': expected an object, but got {received_type} instead." + ) + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_removes_every_invalid_metadata_field_before_raising(): """When both fields are invalid, the raise for the first must not leave the @@ -11098,3 +11154,15 @@ def test_mapped_baggage_tags_keep_non_string_client_tags_as_sent(): data: Final[dict[str, object]] = {"metadata": {"tags": [1, {"team": "x"}, "client-tag", "client-tag"]}} LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) assert data["metadata"]["tags"] == [1, {"team": "x"}, "client-tag", "client-tag", "tenant:a"] + + +@pytest.mark.parametrize("caller_metadata", ({"tags": ["client-tag"]}, json.dumps({"tags": ["client-tag"]}))) +def test_caller_tags_reach_litellm_metadata_whether_metadata_is_a_dict_or_a_json_string(caller_metadata): + data: Final[dict[str, object]] = { + "input": "hi", + "metadata": caller_metadata, + "litellm_metadata": {"tags": ["key-tag"]}, + } + LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data) + assert set(data["litellm_metadata"]["tags"]) == {"key-tag", "client-tag"} + assert data["metadata"] == caller_metadata diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index f880d3548f7..4386d5aec94 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -4,6 +4,7 @@ and ``_handle_logging_proxy_only_error``.""" from __future__ import annotations import asyncio +import json from collections.abc import Callable from datetime import datetime from types import MappingProxyType @@ -1192,3 +1193,31 @@ async def test_handle_logging_proxy_only_path_flags_requests_that_died_before_re blocked_by_guardrail.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is REJECTED_BEFORE_REQUEST_SETUP, blocked_by_guardrail["litellm_call_id"], ) == (True, "client-supplied", False, "call-1") + + +@pytest.mark.asyncio +async def test_auth_failure_on_a_responses_body_with_a_json_string_metadata_is_logged_against_the_key( + proxy_logging, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "input": "hi", "metadata": json.dumps({"tags": ["checkout"]})}, + original_exception=litellm.BudgetExceededError(current_cost=1.0, max_budget=0.5), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-hashed", key_alias="checkout-key", request_route="/v1/responses"), + error_type=ProxyErrorTypes.auth_error, + route="/v1/responses", + ) + + assert len(recorded) == 1 + assert recorded[0]["standard_logging_object"]["metadata"]["user_api_key_alias"] == "checkout-key" From 73aaf9e3daec8bb174accdf56ecade90b34c27c8 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:16:16 -0700 Subject: [PATCH 30/34] fix(mistral): forward prompt_cache_key to the Mistral API (#47432) * fix(mistral): forward prompt_cache_key to the Mistral API * fix(mistral): send prompt_cache_key only to Mistral's API and accept reasoning_effort on devstral and code-agent models * fix(mistral): keep the upstream api_base out of the prompt_cache_key rejection message * fix(mistral): honor allowed_openai_params for prompt_cache_key and gate codestral async and env hosts * test(mistral): annotate the new adapter prompt_cache_key tests * fix(mistral): drop or reject a prompt_cache_key the Mistral API cannot take A non-ASCII prompt_cache_key gets a 500 from api.mistral.ai, and a non-string one a 422, so both now drop under drop_params and answer 400 without it, like any value the provider cannot take. The /v1/messages adapter derives no key from a non-ASCII Claude Code session for Mistral, and the 400 names the caller's provider, so a codestral deployment's error says codestral. * fix(mistral): send a reasoning_effort object as its level and keep UnsupportedParamsError on sync calls * fix(mistral): keep async codestral file uploads working by not rewriting the caller's file blocks --- .../litellm_core_utils/get_litellm_params.py | 1 + .../anthropic/messages/adapters/handler.py | 2 + .../messages/adapters/transformation.py | 30 +- litellm/llms/mistral/chat/transformation.py | 153 ++++++- litellm/llms/openai/openai.py | 5 +- litellm/main.py | 1 + litellm/types/llms/mistral.py | 19 + .../adapters/test_handler_prompt_cache_key.py | 68 ++++ .../test_mistral_chat_transformation.py | 372 +++++++++++++++--- 9 files changed, 576 insertions(+), 75 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8792fd0f58a..29f96d61976 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -85,6 +85,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "github_copilot_user_session", "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 704bec9056f..b3b84f721c0 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -647,12 +647,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["output_config"] = extra_kwargs["output_config"] custom_llm_provider: Final = extra_kwargs.get("custom_llm_provider") + api_base: Final = extra_kwargs.get("api_base") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), ) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index ddda759384f..8ea6293bbe6 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -24,6 +24,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64 TOOL_NAME_HASH_LENGTH: Final = 8 TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55 PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"}) +PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: Final = frozenset({"mistral", "codestral"}) _COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue]) @@ -255,7 +256,12 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None, drop_params: bool = True + self, + kwargs, + *, + custom_llm_provider: str | None = None, + api_base: str | None = None, + drop_params: bool = True, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -289,6 +295,7 @@ class AnthropicAdapter: ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, + api_base=api_base, drop_params=drop_params, ) @@ -1001,7 +1008,9 @@ class LiteLLMAnthropicMessagesAdapter: ) @staticmethod - def _supports_prompt_cache_key(model: str | None, custom_llm_provider: str | None) -> bool: + def _supports_prompt_cache_key( + prompt_cache_key: str, model: str | None, custom_llm_provider: str | None, api_base: str | None + ) -> bool: if not model or not custom_llm_provider: return False if custom_llm_provider in PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: @@ -1009,7 +1018,17 @@ class LiteLLMAnthropicMessagesAdapter: supported_params: Final = litellm.get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) - return "prompt_cache_key" in (supported_params or ()) + if "prompt_cache_key" not in (supported_params or ()): + return False + if custom_llm_provider not in PROVIDERS_SERVED_BY_THE_MISTRAL_CHAT_CONFIG: + return True + _, _, _, resolved_api_base = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + ) + mistral_config: Final = litellm.MistralConfig() + return mistral_config.takes_prompt_cache_key_value( + prompt_cache_key + ) and mistral_config.accepts_prompt_cache_key_at(resolved_api_base) @staticmethod def _target_declares_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: @@ -1042,6 +1061,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs: ChatCompletionRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, ) -> None: """Translate metadata fields from Anthropic request to OpenAI request.""" if "metadata" in anthropic_message_request: @@ -1050,7 +1070,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["user"] = metadata["user_id"] prompt_cache_key: Final = prompt_cache_key_from_user_id(metadata["user_id"]) if prompt_cache_key is not None and self._supports_prompt_cache_key( - anthropic_message_request.get("model"), custom_llm_provider + prompt_cache_key, anthropic_message_request.get("model"), custom_llm_provider, api_base ): new_kwargs["prompt_cache_key"] = prompt_cache_key @@ -1244,6 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, ) -> tuple[ChatCompletionRequest, dict[str, str]]: @@ -1282,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request=anthropic_message_request, new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) ## CONVERT TOOL CHOICE self._translate_tool_choice_to_openai( diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 2fb9402ed03..15e348453e0 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -9,11 +9,16 @@ Docs - https://docs.mistral.ai/api/ from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence from itertools import accumulate, count from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload +from urllib.parse import urlparse import httpx +from pydantic import TypeAdapter from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, @@ -30,7 +35,15 @@ from litellm.router_utils.reasoning_effort_capability import ( nearest_declared_reasoning_effort, ) from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.mistral import MistralTextBlock, MistralThinkingBlock, MistralToolCallMessage +from litellm.types.llms.mistral import ( + MistralCacheKeyParams, + MistralFilePart, + MistralReasoningEffortParams, + MistralRequestRouting, + MistralTextBlock, + MistralThinkingBlock, + MistralToolCallMessage, +) from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionTextObject from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object, supports_reasoning @@ -57,6 +70,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) +_MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) +_MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) +_MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams) +_MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart) + + +def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]: + requested: Final = as_str_mapping(reasoning_effort) + if requested is None: + return reasoning_effort, None + return requested.get("effort"), requested.get("summary") def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list @@ -88,6 +112,8 @@ class MistralConfig(OpenAIGPTConfig): - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message. """ + _is_base_class = False + temperature: int | None = None top_p: int | None = None max_tokens: int | None = None @@ -131,6 +157,7 @@ class MistralConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> list[str]: is_magistral: Final = "magistral" in model.lower() accepts_reasoning_effort: Final = self._accepts_reasoning(model) + accepts_prompt_cache_key: Final = self.custom_llm_provider == "mistral" return [ "stream", "temperature", @@ -145,6 +172,7 @@ class MistralConfig(OpenAIGPTConfig): "parallel_tool_calls", *(("thinking",) if is_magistral else ()), *(("reasoning_effort",) if accepts_reasoning_effort else ()), + *(("prompt_cache_key",) if accepts_prompt_cache_key else ()), ] def _map_tool_choice(self, tool_choice: str) -> str: @@ -211,15 +239,84 @@ class MistralConfig(OpenAIGPTConfig): optional_params["extra_body"] = {"random_seed": value} if param == "response_format": optional_params["response_format"] = value - if param == "reasoning_effort" and "magistral" not in model.lower(): - optional_params["reasoning_effort"] = _accepted_reasoning_effort(model, value, self.custom_llm_provider) + if ( + param == "reasoning_effort" + and "magistral" not in model.lower() + and ( + reasoning_level := self._reasoning_level_to_send( + model, _MISTRAL_REASONING_EFFORT_PARAMS.validate_python(non_default_params), drop_params + ) + ) + is not None + ): + optional_params["reasoning_effort"] = reasoning_level if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True - if param == "parallel_tool_calls": - optional_params["parallel_tool_calls"] = value + if param in ("parallel_tool_calls", "prompt_cache_key"): + optional_params[param] = value return optional_params + def _reasoning_level_to_send( + self, model: str, requested: MistralReasoningEffortParams, drop_params: bool + ) -> str | None: + level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort")) + sendable: Final = ( + _accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None + ) + if sendable is not None and summary is None: + return sendable + if drop_params or litellm.drop_params is True: + return sendable + raise UnsupportedParamsError( + message=( + f"{self.custom_llm_provider} does not support parameters: ['reasoning_effort'], for model={model}, " + "since the Mistral API takes it only as a level such as 'high', with no reasoning summary. " + "To drop it, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=self.custom_llm_provider, + model=model, + ) + + def accepts_prompt_cache_key_at(self, api_base: str | None) -> bool: + resolved_api_base, _ = self._get_openai_compatible_provider_info(api_base, None) + hostname: Final = urlparse(resolved_api_base).hostname or "" + return hostname == "mistral.ai" or hostname.endswith(".mistral.ai") + + @staticmethod + def takes_prompt_cache_key_value(prompt_cache_key: object) -> bool: + return isinstance(prompt_cache_key, str) and prompt_cache_key.isascii() + + def _prompt_cache_key_rejection(self, prompt_cache_key: object, api_base: str | None) -> str | None: + if not self.accepts_prompt_cache_key_at(api_base): + return "since only the Mistral API (*.mistral.ai) takes it" + if not self.takes_prompt_cache_key_value(prompt_cache_key): + return "since the Mistral API takes it only as a string of ASCII characters" + return None + + def _prompt_cache_key_withheld(self, model: str, prompt_cache_key: object, routing: MistralRequestRouting) -> bool: + requested_api_base: Final = routing.get("api_base") + api_base: Final = requested_api_base if isinstance(requested_api_base, str) else None + allowed_openai_params: Final = routing.get("allowed_openai_params") + if isinstance(allowed_openai_params, (list, tuple)) and "prompt_cache_key" in allowed_openai_params: + return False + rejection: Final = self._prompt_cache_key_rejection(prompt_cache_key, api_base) + if rejection is None: + return False + if litellm.drop_params is True or routing.get("drop_params") is True: + return True + requested_provider: Final = routing.get("custom_llm_provider") + provider: Final = requested_provider if isinstance(requested_provider, str) else self.custom_llm_provider + raise UnsupportedParamsError( + message=( + f"{provider} does not support parameters: ['prompt_cache_key'], for model={model}, {rejection}. " + "To send it anyway, pass `allowed_openai_params=['prompt_cache_key']`. To drop it, set " + "`litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n" + ), + llm_provider=provider, + model=model, + ) + def _get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = ( @@ -359,24 +456,31 @@ class MistralConfig(OpenAIGPTConfig): messages = self._handle_message_with_file(messages) return messages + @staticmethod + def _with_mistral_file_id(block: Mapping[str, object]) -> Mapping[str, object]: + file_part: Final = block.get("file") if block.get("type") == "file" else None + file_id: Final = ( + _MISTRAL_FILE_PART.validate_python(file_part).get("file_id") if isinstance(file_part, Mapping) else None + ) + if not file_id: + return block + return {**{key: value for key, value in block.items() if key != "file"}, "file_id": file_id} + + @classmethod + def _with_mistral_file_ids(cls, message: AllMessageValues) -> AllMessageValues: + content: Final = message.get("content") + if not isinstance(content, list): + return message + blocks: Final[tuple[Mapping[str, object], ...]] = tuple(content) + return cast( # cast-ok: only file blocks change, to the file_id shape the Mistral API takes + AllMessageValues, {**message, "content": [cls._with_mistral_file_id(block) for block in blocks]} + ) + def _handle_message_with_file(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: """ Mistral API supports only 'file_id' in message content with type 'file'. """ - for m in messages: - _content_block = m.get("content") - if _content_block and isinstance(_content_block, list): - if any(c.get("type") == "file" for c in _content_block): - # If file content is present, we get file_id from 'file' attribute of content block - # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it. - file_contents = [c for c in _content_block if c.get("type") == "file"] - for file_content in file_contents: - file_id = file_content.get("file", {}).get("file_id") - if file_id: - # Replace 'file' with 'file_id' - file_content["file_id"] = file_id # pyright: ignore[reportGeneralTypeIssues] # legacy in-place rewrite of the block shape - file_content.pop("file", None) - return messages + return [self._with_mistral_file_ids(m) for m in messages] def _add_reasoning_system_prompt_if_needed( self, messages: list[AllMessageValues], optional_params: dict @@ -794,7 +898,16 @@ class MistralConfig(OpenAIGPTConfig): if "magistral" in model.lower() and optional_params.get("_add_reasoning_prompt", False): messages = self._add_reasoning_system_prompt_if_needed(messages, optional_params) - upstream_params: Final = {key: value for key, value in optional_params.items() if key != "client_metadata"} + cache_key_params: Final = _MISTRAL_CACHE_KEY_PARAMS.validate_python(optional_params) + withheld_params: Final = ( + ("client_metadata", "prompt_cache_key") + if "prompt_cache_key" in cache_key_params + and self._prompt_cache_key_withheld( + model, cache_key_params["prompt_cache_key"], _MISTRAL_REQUEST_ROUTING.validate_python(litellm_params) + ) + else ("client_metadata",) + ) + upstream_params: Final = {key: value for key, value in optional_params.items() if key not in withheld_params} # Call parent transform_request which handles _transform_messages return super().transform_request( diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index e0a23d6c99f..bef99fd0719 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -27,6 +27,7 @@ import litellm from litellm import LlmProviders from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES +from litellm.exceptions import UnsupportedParamsError from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -971,8 +972,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): litellm.remove_index_from_tool_calls(messages=messages) else: raise e - except OpenAIError as e: - raise e + except (OpenAIError, UnsupportedParamsError): + raise except Exception as e: status_code: Final = getattr(e, "status_code", 500) error_headers = getattr(e, "headers", None) diff --git a/litellm/main.py b/litellm/main.py index d469aa629fe..8d6e07d6a8f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6199,6 +6199,7 @@ def completion( PROVIDER_AFFINITY_HEADER_KWARG_KEY, "fireworks_forward_user_id", "responses_api_version", + "allowed_openai_params", ) if key in kwargs }, diff --git a/litellm/types/llms/mistral.py b/litellm/types/llms/mistral.py index cd61f32f430..68a9d26827b 100644 --- a/litellm/types/llms/mistral.py +++ b/litellm/types/llms/mistral.py @@ -23,3 +23,22 @@ class MistralThinkingBlock(TypedDict): type: Literal["thinking"] thinking: list[MistralTextBlock] closed: NotRequired[ReadOnly[bool]] + + +class MistralRequestRouting(TypedDict, total=False): + api_base: ReadOnly[object] + drop_params: ReadOnly[object] + allowed_openai_params: ReadOnly[object] + custom_llm_provider: ReadOnly[object] + + +class MistralCacheKeyParams(TypedDict, total=False): + prompt_cache_key: ReadOnly[object] + + +class MistralReasoningEffortParams(TypedDict, total=False): + reasoning_effort: ReadOnly[object] + + +class MistralFilePart(TypedDict, total=False): + file_id: ReadOnly[object] diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index ed6cd6134ff..77bdacc938f 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -1,6 +1,7 @@ import json import os import sys +from typing import Final import pytest @@ -70,3 +71,70 @@ def test_prepare_completion_kwargs_keeps_prompt_cache_key_through_responses_rero ) assert completion_kwargs["model"] == "openai/responses/gpt-5.6-luna" assert completion_kwargs["prompt_cache_key"] == "session-abc" + + +@pytest.mark.parametrize( + ("api_base", "azure_api_base_env", "derives_key"), + [ + (None, None, True), + ("https://api.mistral.ai/v1", None, True), + ("https://my-resource.openai.azure.com/openai/v1", None, False), + (None, "https://my-resource.services.ai.azure.com/openai", False), + ], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_only_for_a_mistral_hosted_endpoint( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, azure_api_base_env: str | None, derives_key: bool +) -> None: + if azure_api_base_env is None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + else: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", azure_api_base_env) + + completion_kwargs: Final = _prepare( + "mistral/mistral-small-latest", {"custom_llm_provider": "mistral", "api_base": api_base} + ) + + assert completion_kwargs["user"] == CLAUDE_CODE_USER_ID + assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +@pytest.mark.parametrize( + ("codestral_api_base_env", "derives_key"), + [(None, True), ("https://my-resource.services.ai.azure.com/openai/v1", False)], +) +def test_prepare_completion_kwargs_derives_prompt_cache_key_for_codestral_only_on_its_mistral_host( + monkeypatch: pytest.MonkeyPatch, codestral_api_base_env: str | None, derives_key: bool +) -> None: + monkeypatch.delenv("MISTRAL_AZURE_API_BASE", raising=False) + if codestral_api_base_env is None: + monkeypatch.delenv("CODESTRAL_API_BASE", raising=False) + else: + monkeypatch.setenv("CODESTRAL_API_BASE", codestral_api_base_env) + + completion_kwargs: Final = _prepare("codestral/codestral-latest", {"custom_llm_provider": "codestral"}) + + assert ("prompt_cache_key" in completion_kwargs) is derives_key + + +def test_prepare_completion_kwargs_derives_no_prompt_cache_key_mistral_cannot_take_from_a_non_ascii_session() -> None: + non_ascii_user_id: Final = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "sitzung-ü"}) + + mistral_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="mistral/mistral-small-latest", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "mistral"}, + )[0] + openai_kwargs: Final = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + metadata={"user_id": non_ascii_user_id}, + thinking=None, + extra_kwargs={"custom_llm_provider": "openai"}, + )[0] + + assert "prompt_cache_key" not in mistral_kwargs + assert openai_kwargs["prompt_cache_key"] == "sitzung-ü" diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b76e1707805..c340f77c317 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,7 +1,13 @@ +import json +from collections.abc import Mapping from typing import Final, List, cast from unittest.mock import MagicMock, patch +from urllib.parse import urlparse +import httpx import pytest +import respx +from openai import AsyncOpenAI from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.llms.mistral.chat.transformation import ( @@ -45,21 +51,15 @@ class TestMistralReasoningSupport: mistral_config = MistralConfig() # Test magistral model supports reasoning parameters - supported_params = mistral_config.get_supported_openai_params( - "mistral/magistral-medium-2506" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/magistral-medium-2506") assert "reasoning_effort" in supported_params assert "thinking" in supported_params - supported_params_reasoning = mistral_config.get_supported_openai_params( - "mistral/mistral-medium-latest" - ) + supported_params_reasoning = mistral_config.get_supported_openai_params("mistral/mistral-medium-latest") assert "reasoning_effort" in supported_params_reasoning assert "thinking" not in supported_params_reasoning - supported_params_normal = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params_normal = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "reasoning_effort" not in supported_params_normal assert "thinking" not in supported_params_normal @@ -117,6 +117,36 @@ class TestMistralReasoningSupport: ) assert optional_params["reasoning_effort"] == sent + @pytest.mark.parametrize( + ("requested", "drop_params", "sent"), + [ + ({"effort": "medium"}, False, {"reasoning_effort": "high"}), + ({"effort": "high", "summary": "auto"}, True, {"reasoning_effort": "high"}), + ({"summary": "auto"}, True, {}), + ], + ) + def test_reasoning_effort_object_is_sent_as_its_level(self, local_model_cost_map, requested, drop_params, sent): + import litellm + + optional_params = litellm.get_optional_params( + model="mistral-small-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + drop_params=drop_params, + ) + assert {key: optional_params[key] for key in ("reasoning_effort",) if key in optional_params} == sent + + @pytest.mark.parametrize("requested", [{"effort": "high", "summary": "auto"}, {"summary": "auto"}]) + def test_reasoning_effort_object_mistral_cannot_take_is_rejected(self, local_model_cost_map, requested): + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match=r"\['reasoning_effort'\]"): + litellm.get_optional_params( + model="devstral-latest", + custom_llm_provider="mistral", + reasoning_effort=requested, + ) + def test_reasoning_effort_is_forwarded_verbatim_when_the_map_declares_no_levels( self, local_model_cost_map, monkeypatch ): @@ -194,9 +224,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should add a new system message at the beginning assert len(result) == 2 @@ -218,9 +246,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message assert len(result) == 2 @@ -251,9 +277,7 @@ class TestMistralReasoningSupport: ] optional_params = {"_add_reasoning_prompt": True} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should modify existing system message preserving list format assert len(result) == 2 @@ -266,10 +290,7 @@ class TestMistralReasoningSupport: # Original content should be preserved assert "You are a helpful assistant." in result[0]["content"][1]["text"] - assert ( - "You always provide detailed explanations." - in result[0]["content"][2]["text"] - ) + assert "You always provide detailed explanations." in result[0]["content"][2]["text"] assert result[1]["role"] == "user" @@ -287,9 +308,7 @@ class TestMistralReasoningSupport: ] string_params = {"_add_reasoning_prompt": True} - string_result = mistral_config._add_reasoning_system_prompt_if_needed( - string_messages, string_params - ) + string_result = mistral_config._add_reasoning_system_prompt_if_needed(string_messages, string_params) assert isinstance(string_result[0]["content"], str) assert "" in string_result[0]["content"] assert "You are helpful." in string_result[0]["content"] @@ -304,9 +323,7 @@ class TestMistralReasoningSupport: ] list_params = {"_add_reasoning_prompt": True} - list_result = mistral_config._add_reasoning_system_prompt_if_needed( - list_messages, list_params - ) + list_result = mistral_config._add_reasoning_system_prompt_if_needed(list_messages, list_params) assert isinstance(list_result[0]["content"], list) assert list_result[0]["content"][0]["type"] == "text" assert "" in list_result[0]["content"][0]["text"] @@ -319,9 +336,7 @@ class TestMistralReasoningSupport: messages = [{"role": "user", "content": "What is 2+2?"}] optional_params = {} - result = mistral_config._add_reasoning_system_prompt_if_needed( - messages, optional_params - ) + result = mistral_config._add_reasoning_system_prompt_if_needed(messages, optional_params) # Should return messages unchanged assert result == messages @@ -444,9 +459,7 @@ class TestMistralReasoningSupport: def test_mistral_streaming_chunk_preserves_thinking_blocks(): """Ensure streaming chunks keep magistral reasoning content.""" - iterator = MistralChatResponseIterator( - streaming_response=iter([]), sync_stream=True, json_mode=False - ) + iterator = MistralChatResponseIterator(streaming_response=iter([]), sync_stream=True, json_mode=False) streamed_chunk = { "id": "chunk-1", @@ -523,9 +536,7 @@ class TestMistralParallelToolCalls: def test_get_supported_openai_params_includes_parallel_tool_calls(self): """Test that parallel_tool_calls is in supported parameters.""" mistral_config = MistralConfig() - supported_params = mistral_config.get_supported_openai_params( - "mistral/mistral-large-latest" - ) + supported_params = mistral_config.get_supported_openai_params("mistral/mistral-large-latest") assert "parallel_tool_calls" in supported_params def test_transform_request_preserves_parallel_tool_calls(self): @@ -691,9 +702,7 @@ class TestMistralEmptyContentHandling: result = MistralConfig._handle_empty_content_response(response_data) - assert ( - result["choices"][0]["message"]["content"] == "Hello, how can I help you?" - ) + assert result["choices"][0]["message"]["content"] == "Hello, how can I help you?" def test_handle_empty_content_response_handles_multiple_choices(self): """Test that only the first choice is processed for empty content.""" @@ -1431,9 +1440,7 @@ def test_mistral_transform_request_keeps_an_empty_tool_message_after_its_tool_ca { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", "content": ""}, ], @@ -1457,9 +1464,7 @@ def test_mistral_transform_request_sends_a_null_tool_reply_as_empty_text(reply): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, {"role": "tool", "tool_call_id": "call_1", **reply}, ], @@ -1514,9 +1519,7 @@ def test_mistral_transform_request_hoists_tool_message_image(): { "role": "assistant", "content": "", - "tool_calls": [ - {"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}} - ], + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "read", "arguments": "{}"}}], }, { "role": "tool", @@ -1665,3 +1668,274 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ ) assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + ("mistral-large-latest", "mistral"), + ("mistral-large-4", "mistral"), + ("a-mistral-model-missing-from-the-cost-map", "mistral"), + ("codestral-latest", "codestral"), + ], +) +def test_prompt_cache_key_reaches_the_mistral_request_body( + local_model_cost_map: None, model: str, custom_llm_provider: str +) -> None: + import litellm + + optional_params: Final = litellm.get_optional_params( + model=model, custom_llm_provider=custom_llm_provider, prompt_cache_key="session-1" + ) + request: Final = MistralConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert request["prompt_cache_key"] == "session-1" + + +AZURE_HOSTED_MISTRAL_API_BASE: Final = "https://my-resource.services.ai.azure.com/openai/v1" + + +def _transform_with_prompt_cache_key( + litellm_params: dict[str, object], prompt_cache_key: object = "session-1" +) -> Mapping[str, object]: + return cast( + Mapping[str, object], + MistralConfig().transform_request( + model="mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + optional_params={"prompt_cache_key": prompt_cache_key, "temperature": 0.2}, + litellm_params=litellm_params, + headers={}, + ), + ) + + +@pytest.mark.parametrize("api_base", ["https://api.mistral.ai/v1", "https://codestral.mistral.ai/v1"]) +def test_prompt_cache_key_reaches_a_mistral_hosted_endpoint(api_base: str) -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": api_base}) + + assert request["prompt_cache_key"] == "session-1" + + +@pytest.mark.parametrize("api_base", [AZURE_HOSTED_MISTRAL_API_BASE, "https://mistral.ai.example.com/v1"]) +def test_prompt_cache_key_for_another_host_is_rejected_without_drop_params(api_base: str) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="prompt_cache_key") as rejection: + _transform_with_prompt_cache_key({"api_base": api_base}) + + upstream_host: Final = urlparse(api_base).hostname or api_base + assert upstream_host not in str(rejection.value) + + +UNSENDABLE_PROMPT_CACHE_KEYS: Final = ("Büro-Sitzung-42", "日本", 123, ["session-1"]) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_rejected_without_drop_params( + prompt_cache_key: object, +) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="ASCII"): + _transform_with_prompt_cache_key({"api_base": "https://api.mistral.ai/v1"}, prompt_cache_key) + + +@pytest.mark.parametrize("prompt_cache_key", UNSENDABLE_PROMPT_CACHE_KEYS) +def test_prompt_cache_key_the_mistral_api_cannot_take_is_dropped_under_drop_params(prompt_cache_key: object) -> None: + request: Final = _transform_with_prompt_cache_key( + {"api_base": "https://api.mistral.ai/v1", "drop_params": True}, prompt_cache_key + ) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_with_ascii_punctuation_reaches_the_mistral_api() -> None: + request: Final = _transform_with_prompt_cache_key({}, "team a/session:42") + + assert request["prompt_cache_key"] == "team a/session:42" + + +def test_prompt_cache_key_allowlisted_for_another_purpose_is_still_rejected_for_another_host() -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError, match="allowed_openai_params"): + _transform_with_prompt_cache_key( + {"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "allowed_openai_params": ["reasoning_effort"]} + ) + + +def _chat_completion_reply() -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "mistral-small-latest", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def test_allowlisted_prompt_cache_key_reaches_a_gateway_in_front_of_mistral(respx_mock: respx.MockRouter) -> None: + import litellm + + gateway_route: Final = respx_mock.post("https://gateway.example.com/mistral/v1/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + litellm.completion( + model="mistral/mistral-small-latest", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/mistral/v1", + api_key="gateway-key", + prompt_cache_key="session-1", + allowed_openai_params=["prompt_cache_key"], + max_retries=0, + ) + + assert json.loads(gateway_route.calls.last.request.content)["prompt_cache_key"] == "session-1" + + +@pytest.mark.asyncio +async def test_prompt_cache_key_is_dropped_from_an_async_codestral_call_to_another_host( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + drop_params=True, + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert "prompt_cache_key" not in json.loads(azure_route.calls.last.request.content) + + +@pytest.mark.asyncio +async def test_prompt_cache_key_rejection_for_an_async_codestral_call_names_codestral( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises(litellm.BadRequestError, match=r"codestral does not support parameters: \['prompt_cache_key'\]"): + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + client=AsyncOpenAI( + api_key="azure-key", base_url=AZURE_HOSTED_MISTRAL_API_BASE, http_client=httpx.AsyncClient() + ), + ) + + assert not azure_route.called + + +@pytest.mark.asyncio +async def test_async_codestral_call_sends_an_uploaded_file_by_its_id(respx_mock: respx.MockRouter) -> None: + import litellm + + codestral_api_base: Final = "https://codestral.mistral.ai/v1" + codestral_route: Final = respx_mock.post(f"{codestral_api_base}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + await litellm.acompletion( + model="codestral/codestral-latest", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Review this file"}, + {"type": "file", "file": {"file_id": "file-12345"}}, + ], + } + ], + api_base=codestral_api_base, + api_key="codestral-key", + max_retries=0, + client=AsyncOpenAI(api_key="codestral-key", base_url=codestral_api_base, http_client=httpx.AsyncClient()), + ) + + sent_content: Final = json.loads(codestral_route.calls.last.request.content)["messages"][0]["content"] + assert sent_content[1] == {"type": "file", "file_id": "file-12345"} + + +def test_prompt_cache_key_rejection_for_a_sync_codestral_call_is_an_unsupported_params_error( + respx_mock: respx.MockRouter, +) -> None: + import litellm + + azure_route: Final = respx_mock.post(f"{AZURE_HOSTED_MISTRAL_API_BASE}/chat/completions").mock( + return_value=_chat_completion_reply() + ) + + with pytest.raises( + litellm.UnsupportedParamsError, match=r"^litellm.UnsupportedParamsError: codestral does not support" + ): + litellm.completion( + model="codestral/codestral-latest", + messages=[{"role": "user", "content": "hi"}], + api_base=AZURE_HOSTED_MISTRAL_API_BASE, + api_key="azure-key", + prompt_cache_key="session-1", + max_retries=0, + ) + + assert not azure_route.called + + +def test_prompt_cache_key_for_another_host_is_dropped_under_drop_params() -> None: + request: Final = _transform_with_prompt_cache_key({"api_base": AZURE_HOSTED_MISTRAL_API_BASE, "drop_params": True}) + + assert "prompt_cache_key" not in request + assert request["temperature"] == 0.2 + + +def test_prompt_cache_key_for_the_azure_endpoint_set_in_the_environment_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISTRAL_AZURE_API_BASE", "https://my-resource.services.ai.azure.com/openai") + + request: Final = _transform_with_prompt_cache_key({"drop_params": True}) + + assert "prompt_cache_key" not in request + + +def test_prompt_cache_key_stays_unsupported_for_vertex_hosted_mistral(local_model_cost_map: None) -> None: + import litellm + + with pytest.raises(litellm.UnsupportedParamsError): + litellm.get_optional_params(model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s") + + dropped: Final = litellm.get_optional_params( + model="mistral-medium-3", custom_llm_provider="vertex_ai", prompt_cache_key="s", drop_params=True + ) + assert "prompt_cache_key" not in dropped From 4c8b90945faf69eedfbdda3d4b7dad58ce74860d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:31:39 -0700 Subject: [PATCH 31/34] fix(realtime)!: send the Gemini Live setup at the first content so session.update settings apply (#39864) * fix(realtime): send Gemini Live setup on the client's first message so session.update modalities apply Gemini and Vertex AI Live accept exactly one setup per session, and the proxy sent its default audio setup at connect, so a client's session.update (GA output_modalities or beta modalities) was dropped and every answer came back as audio. The setup now goes out on the client's first message: a session.update becomes that setup, anything else gets the model default. Bedrock Nova Sonic keeps its connect-time setup through a provider hook. Text setups label deltas response.output_text.delta, and downgrading TEXT to AUDIO on an audio-only model logs a warning * fix(realtime): keep one Gemini Live response modality and skip the "inf" token limit in the setup * fix(realtime): merge every session.update sent before the first content frame into the Gemini Live setup Gemini and Vertex Live accept one setup per connection, so the bridge used to turn only the first session.update into that setup and drop the rest without telling the client. Now every session.update that arrives before the first content frame is held, deep-merged (later values win per field), acknowledged with a session.updated built from the setup it would produce, and the merged session becomes the one setup sent with the first conversation item, audio frame, or tool result. The session events also surface the systemInstruction text as instructions, which the old string check never matched because the setup carries it as a Content object. * fix(realtime): cap the session.update merge depth so the recursive detector accepts it _merge_session_fields recurses once per nested mapping level, so CI's recursive_detector flagged it. Bound the descent at eight levels (the OpenAI session object nests four deep at most), replace anything deeper wholesale with the later session.update's value, and list the helper with the other depth-capped functions the detector allows. * fix(gemini-live): accept a string output modality and drop values Live cannot produce A bare string output_modalities is read as a one-element list and modality values Gemini Live cannot produce are dropped with a warning instead of reaching the backend, so a malformed session.update no longer costs the session. Transcription guardrails now fold their VAD disable into the first held session.update only, so the merged setup carries it exactly once. * fix(gemini realtime): answer an error for session updates Gemini Live cannot apply An output modality the model cannot produce, or a session.update sent after the first content, now gets an invalid_request_error event instead of a silent swap or drop, unless drop_params is set. The synthetic session.created reports the model's default modality, setupComplete no longer adds an extra session.updated, and merging nested session updates is linear in their size * fix(gemini realtime): send the setup before guardrail blocks and idle deadlines, mark push-to-talk turns A pre_call guardrail blocking the first typed message now goes out after the setup, so Gemini and Vertex no longer close the session with 1007. Push-to-talk sessions (turn_detection null or create_response false) wrap each audio turn in empty activityStart and activityEnd messages, which Gemini needs when automatic activity detection is off. A session with no content after 60 seconds sends its setup, since Vertex refuses a setup 300 seconds after connect. Held session.update frames past 10 MB answer session_update_too_large, and deferred mode now errors on a second session.update unless drop_params is set. * fix(realtime): ack unchanged Gemini Live session.update resends and honor null turn_detection An update that leaves the applied setup unchanged now gets session.updated instead of session_update_not_applied, so clients that resend their session with the first response keep working. Deferred setup mode merges held updates like the default mode, turn_detection null turns Gemini's activity detection off, drop_params drops a voice outside the documented Live voices, and the lazy setup deadline moves to 240 seconds * fix(realtime): keep Vertex activity detection off for null turn_detection and set up deferred sessions before text content * fix(realtime): refuse unscreenable Gemini audio, send held setup after buffered audio, reject unknown Vertex voices A realtime_input_transcription guardrail cannot screen Gemini Live audio before the answer starts, so audio input under one is refused with an audio_input_not_screenable error instead of reaching the model. In deferred setup mode a session.update that follows buffered audio is sent as setup at once. Only conversation content triggers the setup, so control events no longer lock in a default one. Vertex rejects an undocumented voice on the session.update, or drops it with drop_params, while Google AI Studio forwards any name. * fix(realtime): hold first content so a following session.update joins the Gemini setup Convert flat realtime tool schemas through Gemini's schema builder, and collapse a text and audio pair to what the model can produce on text-only models too * fix(realtime): check AI Studio Live voices like Vertex * fix(realtime): forward any voice to AI Studio Live models that accept it * fix(realtime): map OpenAI TTS voices before the Live voice check, keep outage spend rows, share the playground session config The Live voice gate now judges the Gemini voice an OpenAI TTS name maps to, so fable and onyx reach Vertex as Umbriel and Orus again. An upstream close before the held setup was sent is logged as a session, not a refusal. The playground sends one server_vad session config for typed and voice turns, and the e2e, integration, and dashboard tests follow lazy setup. * fix(realtime): take the lazy setup clock as a dependency so its tests run on a stepped clock The deadline and grace tests waited on real 50 ms timers, and the merge cost check compared wall-clock runs. The session now reads an injected clock, time.monotonic by default, the two tests step it, and the merge cost check moves to the CodSpeed benchmarks. The Nova Sonic style handler test passes the backend through wrap_backend, which main added after the test was written --- litellm/constants.py | 2 + .../litellm_core_utils/realtime_streaming.py | 330 ++++++++- .../llms/base_llm/realtime/transformation.py | 32 +- litellm/llms/bedrock/realtime/handler.py | 20 +- .../llms/bedrock/realtime/transformation.py | 19 +- litellm/llms/custom_httpx/llm_http_handler.py | 11 +- .../llms/gemini/realtime/transformation.py | 317 +++++++-- .../llms/vertex_ai/realtime/transformation.py | 17 +- litellm/types/llms/gemini.py | 16 +- litellm/utils.py | 2 +- .../test_realtime_session_merge_benchmarks.py | 21 + .../code_coverage_tests/recursive_detector.py | 1 + .../realtime/realtime_client.py | 3 + .../realtime/test_realtime_e2e.py | 4 +- ..._transcription_guardrail_session_update.py | 24 +- .../test_realtime_streaming.py | 670 +++++++++++++++++- .../custom_httpx/test_llm_http_handler.py | 189 +++++ .../test_gemini_realtime_transformation.py | 310 +++++++- .../test_vertex_ai_realtime_transformation.py | 110 +++ .../chat_ui/RealtimePlayground.test.tsx | 18 + .../components/chat_ui/RealtimePlayground.tsx | 65 +- 21 files changed, 1993 insertions(+), 188 deletions(-) create mode 100644 tests/benchmarks/test_realtime_session_merge_benchmarks.py diff --git a/litellm/constants.py b/litellm/constants.py index e7c9c5dcbfb..8e80c07b4ed 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -420,6 +420,8 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) +REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0 +REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS: Final = 1.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 484bacdc72c..4db22e9d5bf 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,5 +1,6 @@ import asyncio import json +import time import traceback from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -15,13 +16,19 @@ from litellm._logging import redact_internal_details_from_client_message, verbos from litellm._uuid import uuid from litellm.constants import ( BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, + REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, + REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend +from litellm.llms.base_llm.realtime.transformation import ( + BaseRealtimeConfig, + RealtimeBackend, + RealtimeSessionUpdateRejection, +) from litellm.types.llms.openai import ( OpenAIRealtimeEvents, OpenAIRealtimeOutputItemDone, @@ -125,6 +132,65 @@ class _LoggingWorker(Protocol): _DECODED_JSON: Final = TypeAdapter(object) +_SESSION_FIELDS: Final = TypeAdapter(dict[str, object]) +_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) +_SESSION_MERGE_MAX_DEPTH: Final = 8 + + +class _SessionUpdateFrame(TypedDict): + type: ReadOnly[str] + session: ReadOnly[Mapping[str, object]] + + +def _merge_session_fields( + base: Mapping[str, object], update: Mapping[str, object], depth: int = 0 +) -> Mapping[str, object]: + def merged_value(key: str) -> object: + if key not in update: + return base[key] + value: Final = update[key] + base_value: Final = base.get(key) + if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping): + return _merge_session_fields( + _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1 + ) + return update[key] + + return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))}) + + +def _session_update_frame(session: Mapping[str, object]) -> str: + frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session} + return json.dumps(frame, default=_SESSION_FIELDS.validate_python) + + +_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection( + code="session_update_too_large", + message=( + "The session.update frames sent before the first conversation content add up to more than 10 MB, " + "so this session.update was not applied" + ), + param="session", +) + + +def _unscreened_audio_error_event(client_event_id: object) -> Mapping[str, object]: + return { + "type": "error", + "event_id": f"event_{uuid.uuid4()}", + "error": { + "type": "invalid_request_error", + "code": "audio_input_not_screenable", + "message": ( + "A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it " + "starts answering it, so audio input was not sent. Send text with conversation.item.create instead" + ), + "param": None, + "event_id": client_event_id if isinstance(client_event_id, str) else None, + }, + } + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" return json.loads(payload) @@ -243,8 +309,17 @@ class RealTimeStreaming: event_normalizer: RealtimeEventNormalizer | None = None, usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS, + lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, + clock: Callable[[], float] = time.monotonic, ): self.websocket: _ClientWebSocket = websocket + self._clock: Final = clock + self._lazy_setup_deadline: Final = clock() + lazy_setup_deadline_seconds + self._lazy_setup_content_grace_seconds: Final = lazy_setup_content_grace_seconds + self._held_client_content: tuple[str, ...] = () + self._held_client_content_bytes: int = 0 + self._held_client_content_deadline: float | None = None self._replay_scope: Final = client_websocket_scope(websocket) self.backend_ws = backend_ws self.logging_obj = logging_obj @@ -300,6 +375,10 @@ class RealTimeStreaming: # Gemini Live rejects a follow-up BidiGenerateContentSetup once any # content (realtimeInput / clientContent / toolResponse) has been sent. self._content_sent_after_setup: bool = False + self._held_session_update: Mapping[str, object] | None = None + self._applied_session_fields: Mapping[str, object] | None = None + self._held_session_update_bytes: int = 0 + self._unscreened_audio_refused: bool = False # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and # their input_audio_transcription.completed usage drives duration-based cost. @@ -337,6 +416,17 @@ class RealTimeStreaming: ] ) _CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"]) + _CLIENT_CONTENT_TYPES = frozenset( + [ + "conversation.item.create", + "response.create", + "input_audio_buffer.append", + "input_audio_buffer.commit", + "input_audio_buffer.end", + ] + ) + _HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create", "input_audio_buffer.append"]) + _DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"]) _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, @@ -377,7 +467,7 @@ class RealTimeStreaming: message_obj: dict[str, Any] = cast(dict[str, Any], message) else: message_obj = cast(dict[str, Any], json.loads(message)) - self._collect_tool_calls_from_response_done(cast(dict, message_obj)) + self._collect_tool_calls_from_response_done(message_obj) if not self._should_store_message(message_obj): return try: @@ -502,7 +592,7 @@ class RealTimeStreaming: self.store_message(flush_event) self._capture_transcription_usage(flush_event) - def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None: + def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None: """Extract function_call items from response.done events for spend logging.""" try: if event_obj.get("type") != "response.done": @@ -607,6 +697,167 @@ class RealTimeStreaming: self._usage_meter.observe_client_event(sent_event) return True + def _builds_setup_lazily(self) -> bool: + if self.provider_config is None: + return False + return self.provider_config.builds_setup_from_session_update() + + def _holds_session_updates_until_first_content(self) -> bool: + return self.session_configuration_request is None and self._builds_setup_lazily() + + async def _rejected_setup_session_update(self, message: str) -> bool: + provider_config: Final = self.provider_config + if provider_config is None or not provider_config.builds_setup_from_session_update(): + return False + client_event: Final = _decode_json_object(message) + session: Final = client_event.get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + if self._leaves_applied_setup_unchanged(update): + await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS) + return True + rejection: Final = provider_config.setup_session_update_rejection( + self.model, update, setup_sent=self.session_configuration_request is not None + ) + if rejection is None: + return False + await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id")))) + return True + + def _held_session_update_message(self) -> str: + return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS) + + async def _hold_session_update(self, message: str) -> None: + client_event: Final = _decode_json_object(message) + if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES: + await self.websocket.send_text( + json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id"))) + ) + return + session: Final = client_event.get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) + self._held_session_update = held + self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8")) + await self._acknowledge_session_setup(held) + + def _setup_preview(self, session: Mapping[str, object]) -> str | None: + if self.provider_config is None: + return None + preview: Final = self.provider_config.transform_realtime_request( + _session_update_frame(session), self.model, None + ) + return next( + (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None + ) + + def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool: + applied: Final = self._applied_session_fields + if self.session_configuration_request is None or applied is None: + return False + applied_setup: Final = self._setup_preview(applied) + merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update)) + return ( + applied_setup is not None + and merged_setup is not None + and _decode_json_object(applied_setup) == _decode_json_object(merged_setup) + ) + + async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None: + if self.provider_config is None: + return + setup: Final = self._setup_preview(session) + if setup is None: + return + acknowledged: Final = self.provider_config.transform_session_created_event( + self.model, str(self.logging_obj.litellm_trace_id), setup + ) + if acknowledged is None: + return + event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"} + event_str: Final = json.dumps(event) + self.store_message(event_str) + await self._send_event_to_client(event, event_str) + + async def _send_held_session_update_as_setup(self) -> None: + held: Final = self._held_session_update + if held is None: + return + message: Final = self._held_session_update_message() + self._held_session_update = None + self._held_session_update_bytes = 0 + if await self._send_to_backend(message): + self._applied_session_fields = held + await self._send_held_client_content() + + def _holds_client_content(self, msg_type: str | None, message: str) -> bool: + held_types: Final = ( + RealTimeStreaming._DEFERRED_SETUP_HELD_CONTENT_TYPES + if self._uses_deferred_backend_setup() + else RealTimeStreaming._HELD_CONTENT_TYPES + ) + return ( + msg_type in held_types + and self._held_session_update is not None + and self._holds_session_updates_until_first_content() + and not self._pending_messages_until_setup + and len(self._held_client_content) < RealTimeStreaming._MAX_BUFFERED_MESSAGES + and self._held_client_content_bytes + len(message.encode("utf-8")) <= RealTimeStreaming._MAX_BUFFERED_BYTES + ) + + def _hold_client_content(self, message: str) -> None: + if self._held_client_content_deadline is None: + self._held_client_content_deadline = self._clock() + self._lazy_setup_content_grace_seconds + self._held_client_content = (*self._held_client_content, message) + self._held_client_content_bytes += len(message.encode("utf-8")) + + async def _send_held_client_content(self) -> None: + held: Final = self._held_client_content + self._held_client_content = () + self._held_client_content_bytes = 0 + self._held_client_content_deadline = None + for message in held: + await self._send_to_backend(message) + + def _setup_deadline(self) -> float: + content_deadline: Final = self._held_client_content_deadline + return ( + self._lazy_setup_deadline if content_deadline is None else min(self._lazy_setup_deadline, content_deadline) + ) + + async def _send_setup_before_content(self, msg_type: str | None) -> None: + if self._holds_session_updates_until_first_content(): + await self._send_held_session_update_as_setup() + await self._send_default_setup_before_first_client_message(msg_type) + + async def _receive_client_message_before_setup_deadline(self) -> str | None: + if not self._holds_session_updates_until_first_content(): + return await self._receive_client_message() + receive: Final = asyncio.ensure_future(self._receive_client_message()) + try: + done, _ = await asyncio.wait((receive,), timeout=self._setup_deadline() - self._clock()) + if not done: + await self._send_setup_before_content(None) + return await receive + finally: + receive.cancel() + + async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None: + if self.session_configuration_request is not None or self.provider_config is None: + return + if msg_type == "session.update" or ( + self._uses_deferred_backend_setup() + and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup) + ): + return + default_setup: Final = self.provider_config.session_configuration_request(self.model) + if not default_setup: + return + setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup) + await self.backend_ws.send(setup) + self._cache_session_configuration_request(setup) + self._applied_session_fields = _NO_SESSION_FIELDS + await self._send_held_client_content() + def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -851,6 +1102,20 @@ class RealTimeStreaming: self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id} await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id})) + async def _send_blocked_turn_to_backend(self, pre_block_backend_message: str | None, item_id: str | None) -> None: + if pre_block_backend_message is None and self._is_transcription_session: + return + await self._send_setup_before_content("conversation.item.create") + # Deliver any caller-supplied backend message FIRST so that + # protocol contracts requiring a specific ordering (e.g. + # Gemini Live's mandatory ``toolResponse`` after a + # ``toolCall``) are honored before the guardrail's own + # clientContent / cancel messages are sent. + if pre_block_backend_message is not None: + await self._send_to_backend(pre_block_backend_message) + if not self._is_transcription_session: + await self._withdraw_blocked_turn(item_id) + async def _withdraw_blocked_turn(self, item_id: str | None) -> None: await self._send_guardrail_cancel() if item_id is not None and self._deletes_conversation_items: @@ -958,6 +1223,21 @@ class RealTimeStreaming: """Transcription-only sessions have no assistant turn to gate and reject realtime session updates.""" return not self._is_transcription_session and self._has_audio_transcription_guardrails() + def _refuses_unscreened_audio(self, msg_type: str | None) -> bool: + return ( + msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES + and self._builds_setup_lazily() + and self._should_disable_vad_auto_response() + ) + + async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None: + if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES: + return + self._unscreened_audio_refused = True + await self.websocket.send_text( + json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id"))) + ) + async def run_realtime_guardrails( self, transcript: str, @@ -1030,15 +1310,7 @@ class RealTimeStreaming: # Use realtime_violation_message if configured; fall back to guardrail error text. error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg - # Deliver any caller-supplied backend message FIRST so that - # protocol contracts requiring a specific ordering (e.g. - # Gemini Live's mandatory ``toolResponse`` after a - # ``toolCall``) are honored before the guardrail's own - # clientContent / cancel messages are sent. - if pre_block_backend_message is not None: - await self._send_to_backend(pre_block_backend_message) - if not self._is_transcription_session: - await self._withdraw_blocked_turn(item_id) + await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id) # Send the policy violation hint (shows as small gray status text in UI). await self.websocket.send_text( json.dumps( @@ -1136,6 +1408,9 @@ class RealTimeStreaming: break finally: self._flushing_pending_messages_until_setup = False + if self.session_created_sent_to_client and self._builds_setup_lazily(): + await self._maybe_send_guardrail_turn_detection_update() + continue if self.session_created_sent_to_client: # A synthetic session.created (with placeholder defaults) was # already forwarded to the client when we connected. The @@ -1295,7 +1570,11 @@ class RealTimeStreaming: return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket") def _backend_refused_session(self, close: BackendClose) -> bool: - return close.code != 1000 and not self._backend_sent_frames + return ( + close.code != 1000 + and not self._backend_sent_frames + and not self._holds_session_updates_until_first_content() + ) async def _log_session_end(self, transport_refusal: Exception | None) -> None: provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal() @@ -1488,7 +1767,7 @@ class RealTimeStreaming: client_event: _ClientEventFrame try: while True: - message = await self._receive_client_message() + message = await self._receive_client_message_before_setup_deadline() if message is None: return ClientLoopExit.CLIENT_DISCONNECTED @@ -1600,6 +1879,7 @@ class RealTimeStreaming: if ( msg_type == "session.update" and self.session_configuration_request is None + and self._held_session_update is None and not self._guardrail_turn_detection_update_sent and self._should_disable_vad_auto_response() and ( @@ -1663,12 +1943,25 @@ class RealTimeStreaming: except (json.JSONDecodeError, AttributeError): pass + if self._refuses_unscreened_audio(msg_type): + await self._refuse_unscreened_audio(msg_type, message) + continue + ## LOGGING # Log after any in-place modifications (GA remap, guardrail # turn_detection injection) so audit logs reflect what we # actually forward to the backend. self.store_input(message=message) + if self._holds_client_content(msg_type, message): + self._hold_client_content(message) + continue + if self._held_client_content and msg_type != "session.update": + await self._send_setup_before_content(msg_type) + + if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup(): + await self._send_setup_before_content(msg_type) + if self._should_buffer_client_message_until_setup(message): self._buffer_pending_message_until_setup(message) continue @@ -1697,6 +1990,15 @@ class RealTimeStreaming: # ``_maybe_send_guardrail_turn_detection_update`` would retry. if msg_type == "session.update" and self.provider_config is None: await self._send_session_gate_once() + if msg_type == "session.update" and await self._rejected_setup_session_update(message): + continue + if msg_type == "session.update" and self._holds_session_updates_until_first_content(): + await self._hold_session_update(message) + if self._pending_messages_until_setup: + await self._send_held_session_update_as_setup() + continue + if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES: + await self._send_setup_before_content(msg_type) try: sent = await self._send_to_backend(message) except RealtimeTranscriptionProtocolError as e: diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index f1539d16b30..ff714bb908b 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -1,10 +1,12 @@ +import dataclasses from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from types import TracebackType -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any, Literal, Protocol import httpx +from litellm._uuid import uuid from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( RealtimeInputAudioTranscriptionUsage, @@ -22,6 +24,26 @@ else: LiteLLMLoggingObj = Any +@dataclasses.dataclass(frozen=True, slots=True) +class RealtimeSessionUpdateRejection: + code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"] + message: str + param: str + + def error_event(self, client_event_id: object) -> dict[str, object]: + return { + "type": "error", + "event_id": f"event_{uuid.uuid4()}", + "error": { + "type": "invalid_request_error", + "code": self.code, + "message": self.message, + "param": self.param, + "event_id": client_event_id if isinstance(client_event_id, str) else None, + }, + } + + class RealtimeBackend(Protocol): async def __aenter__(self) -> object: ... @@ -93,6 +115,14 @@ class BaseRealtimeConfig(ABC): def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session return None + def builds_setup_from_session_update(self) -> bool: + return False + + def setup_session_update_rejection( + self, model: str, session: Mapping[str, object], *, setup_sent: bool + ) -> RealtimeSessionUpdateRejection | None: + return None + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index e9b2f26e977..00622a1e287 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm._logging import redact_string, verbose_proxy_logger -from litellm._uuid import uuid from litellm.constants import ( BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS, BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, @@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes +from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError -from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities +from .transformation import BedrockRealtimeConfig, requested_modalities _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -256,23 +256,11 @@ async def _ack_session_update( async def _reject_session_update( client_ws: RealtimeClientWebSocket, parsed_client_message: Mapping[str, JsonValue], - rejection: BedrockSessionUpdateRejection, + rejection: RealtimeSessionUpdateRejection, ) -> bool: """Send an error event for a session.update Bedrock cannot apply; False means the client is gone.""" - client_event_id: Final = parsed_client_message.get("event_id") - error_event: Final = { - "type": "error", - "event_id": f"event_{uuid.uuid4()}", - "error": { - "type": "invalid_request_error", - "code": rejection.code, - "message": rejection.message, - "param": rejection.param, - "event_id": client_event_id if isinstance(client_event_id, str) else None, - }, - } try: - await client_ws.send_text(json.dumps(error_event)) + await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id")))) except Exception as e: # noqa: BLE001 # any send failure means the client is gone verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True) return False diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 952d8858f78..25498fef46f 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -11,7 +11,7 @@ import math import uuid as uuid_lib from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final, Literal +from typing import Final import httpx from pydantic import ConfigDict, JsonValue, TypeAdapter @@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm from litellm.types.llms.base import LiteLLMBaseModel @@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[ return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed) -@dataclasses.dataclass(frozen=True, slots=True) -class BedrockSessionUpdateRejection: - code: Literal["invalid_type", "invalid_value", "session_update_not_applied"] - message: str - param: str - - @dataclasses.dataclass(frozen=True, slots=True) class BedrockLockedSessionSettings: """The session settings Bedrock Nova Sonic locks in once the prompt starts.""" @@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): max_tokens=max_tokens, ) - def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None: + def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None: """The error for a session.update this session cannot apply, or None when it applies.""" session_config: Final = json_message.get("session") if not isinstance(session_config, Mapping): @@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): malformed: Final = () if self.drop_params else _malformed_settings(session_config) if malformed: name, expected = malformed[0] - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="invalid_type", message=( f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. " @@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config) if unplayable_voice is not None: - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="invalid_value", message=( f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. " @@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not self.prompt_started: return None if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings: - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="session_update_not_applied", message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied", param="session", diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c92315e53b3..61fc23589ac 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6973,14 +6973,19 @@ class BaseLLMHTTPHandler: ), ) - # Auto-send session setup if the provider requires it (e.g. - # Gemini/Vertex AI Live needs a `setup` before any realtime_input). + # Auto-send session setup at connect unless the provider builds it from the + # client's first session.update (Gemini/Vertex AI Live accept exactly one + # setup, so RealTimeStreaming sends theirs on the first client message). # Build the streaming handler first so a transcription guardrail's # auto-response disable can be folded into this one setup: Gemini # rejects a second setup, so a follow-up disable would be dropped # and the guardrail bypassed. _session_config: str | None = None - if provider_config.requires_session_configuration(): + sends_setup_at_connect: Final = ( + provider_config.requires_session_configuration() + and not provider_config.builds_setup_from_session_update() + ) + if sends_setup_at_connect: _session_config = provider_config.session_configuration_request(model) if _session_config: _session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable( diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 6f778ab2761..3134dfa1e4b 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -7,6 +7,7 @@ from collections import OrderedDict from collections.abc import Mapping, Sequence from typing import Final, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly, Required, TypedDict import litellm @@ -15,8 +16,9 @@ from litellm._uuid import uuid from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + OPENAI_TO_GEMINI_TTS_VOICES, VertexGeminiConfig, ) from litellm.responses.litellm_completion_transformation.transformation import ( @@ -76,27 +78,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res # Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped. _KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT} +_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"}) OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset( {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"} ) +# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28) +GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset( + { + "achernar", + "achird", + "algenib", + "algieba", + "alnilam", + "aoede", + "autonoe", + "callirrhoe", + "charon", + "despina", + "enceladus", + "erinome", + "fenrir", + "gacrux", + "iapetus", + "kore", + "laomedeia", + "leda", + "orus", + "puck", + "pulcherrima", + "rasalgethi", + "sadachbia", + "sadaltager", + "schedar", + "sulafat", + "umbriel", + "vindemiatrix", + "zephyr", + "zubenelgenubi", + } +) -def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None: + +def _is_undocumented_live_voice(voice: object) -> bool: + return ( + isinstance(voice, str) + and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES + and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES + ) + + +def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None: """Build the Gemini Live speechConfig for a client-requested voice. - OpenAI stock voice names have no Gemini equivalent and Gemini Live closes - the session on an unknown voice, so they are dropped with a warning and - the model keeps its default voice. Every other name is forwarded verbatim. + OpenAI stock voice names have no Gemini equivalent, so they are always + dropped, and so is any name outside Google's documented Live voices when + ``drop_undocumented`` is set, keeping the model's default voice. Every + other name is forwarded verbatim. """ - if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES: + if not isinstance(voice, str): + return VertexGeminiConfig().map_audio_params({"voice": voice}) + if voice.lower() in OPENAI_STOCK_REALTIME_VOICES: verbose_logger.warning( "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; " "dropping it so the session keeps the model's default voice.", voice, ) return None + if drop_undocumented and _is_undocumented_live_voice(voice): + verbose_logger.warning( + "Gemini Realtime: voice %s is not a documented Gemini Live voice; " + "drop_params is set, so it is dropped and the session keeps the model's default voice.", + voice, + ) + return None return VertexGeminiConfig().map_audio_params({"voice": voice}) @@ -142,16 +199,34 @@ def _base64_decoded_byte_count(data: str) -> int: return max(len(data) * 3 // 4 - padding, 0) +_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...]) + + +def _chat_completion_tool(tool: object) -> dict[str, object]: + fields: Final = as_str_mapping(tool) or {} + if fields.get("type") != "function" or "function" in fields: + return {**fields} + return {"type": "function", "function": {key: value for key, value in fields.items() if key != "type"}} + + +def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None: + parts: Final = system_instruction.get("parts", []) if system_instruction else [] + texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str)) + return "\n".join(texts) if texts else None + + class GeminiRealtimeConfig(BaseRealtimeConfig): _TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping - def __init__(self): + def __init__(self, drop_params: bool = False): super().__init__() + self.drop_params = drop_params self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict() # Gemini Live sometimes emits usageMetadata in a standalone frame between # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None self._unbilled_input_audio_bytes: int = 0 + self._manual_activity_open: bool = False def is_setup_message(self, msg_obj: dict) -> bool: return "setup" in msg_obj @@ -167,6 +242,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``.""" return True + def _rejects_undocumented_voices(self, model: str) -> bool: + """Google AI Studio closes the socket (1007) on an unknown voice only for its native-audio Live models.""" + return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio")) + @staticmethod def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]: detail_mapping: Final = as_str_mapping(details) @@ -272,17 +351,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): except (json.JSONDecodeError, TypeError, AttributeError): return False + def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]: + if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request): + return () + self._manual_activity_open = True + activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}} + return (json.dumps({"realtimeInput": activity_start}),) + + def _end_manual_activity(self) -> tuple[str, ...]: + if not self._manual_activity_open: + return () + self._manual_activity_open = False + activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}} + return (json.dumps({"realtimeInput": activity_end}),) + def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]: """Map OpenAI buffer commit/end to Gemini Live turn-boundary signals.""" if self._manual_turn_detection_enabled(session_configuration_request): - realtime_input_dict: BidiGenerateContentRealtimeInput = { - "activityEnd": True, - } - verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend") - else: - realtime_input_dict = {"audioStreamEnd": True} - verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend") - return [json.dumps({"realtimeInput": realtime_input_dict})] + return list(self._end_manual_activity()) + audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True} + return [json.dumps({"realtimeInput": audio_stream_end})] def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection: """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``. @@ -325,7 +413,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "voice", ] - def map_openai_params(self, optional_params: dict, non_default_params: dict) -> dict: + def map_openai_params(self, optional_params: dict, non_default_params: dict, *, model: str) -> dict: if "generationConfig" not in optional_params: optional_params["generationConfig"] = {} for key, value in non_default_params.items(): @@ -333,12 +421,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}]) elif key == "temperature": optional_params["generationConfig"]["temperature"] = value - elif key == "max_response_output_tokens": + elif key == "max_response_output_tokens" and isinstance(value, int): optional_params["generationConfig"]["maxOutputTokens"] = value elif key == "modalities": - optional_params["generationConfig"]["responseModalities"] = [ - modality.upper() for modality in cast(list[str], value) - ] + optional_params["generationConfig"]["responseModalities"] = list( + GeminiRealtimeConfig._requested_response_modalities(cast(object, value)) + ) elif key == "tools": from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, @@ -347,34 +435,43 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): vertex_gemini_config = VertexGeminiConfig() # Tools should be at the top level of setup, not inside generationConfig optional_params["tools"] = vertex_gemini_config.map_function( - value=value, optional_params=optional_params + value=[_chat_completion_tool(tool) for tool in _JSON_ARRAY.validate_python(value)], + optional_params=optional_params, ) elif key == "input_audio_transcription" and value is not None: optional_params["inputAudioTranscription"] = {} - elif key == "turn_detection" and value is not None: - value_typed = cast(OpenAIRealtimeTurnDetection, value) - if ( - isinstance(value_typed, dict) - and value_typed.get("type") == "semantic_vad" - and "create_response" not in value_typed - ): - # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. - # Only skip when there is no create_response override so that - # a guardrail-injected create_response:false is not dropped. - continue - transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed) - if transformed_audio_activity_config: - optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig( - automaticActivityDetection=transformed_audio_activity_config - ) + elif key == "turn_detection": + realtime_input_config = self._realtime_input_config(cast(object, value)) + if realtime_input_config is not None: + optional_params["realtimeInputConfig"] = realtime_input_config elif key == "voice": - speech_config = _gemini_live_speech_config(value) + speech_config = _gemini_live_speech_config( + value, self.drop_params and self._rejects_undocumented_voices(model) + ) if speech_config: optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: optional_params.pop("generationConfig") return optional_params + def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None: + if turn_detection is None: + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True}) + value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection) + if ( + isinstance(value_typed, dict) + and value_typed.get("type") == "semantic_vad" + and "create_response" not in value_typed + ): + # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. + # Only skip when there is no create_response override so that + # a guardrail-injected create_response:false is not dropped. + return None + transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed) + if not transformed_audio_activity_config: + return None + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config) + @staticmethod def _extract_turn_detection(session: dict) -> dict | None: """Extract turn_detection from a session.update payload. @@ -423,6 +520,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if isinstance(input_cfg, dict): if "input_audio_transcription" not in normalized and "transcription" in input_cfg: normalized["input_audio_transcription"] = input_cfg["transcription"] + if "turn_detection" not in normalized and "turn_detection" in input_cfg: + normalized["turn_detection"] = input_cfg["turn_detection"] output_cfg: Final = audio.get("output") if isinstance(output_cfg, dict) and output_cfg.get("voice"): normalized["voice"] = output_cfg["voice"] @@ -434,7 +533,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return normalized @staticmethod - def _model_cost_entry(model: str) -> dict: + def _model_cost_entry(model: str) -> Mapping[str, object]: entry = litellm.model_cost.get(model) if entry is None: stripped: Final = model.split("/", 1)[-1] @@ -455,18 +554,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO" @staticmethod - def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: - """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for - audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live).""" - normalized: Final = tuple( - modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities + def _requested_response_modalities(value: object) -> tuple[str, ...]: + if isinstance(value, str): + return (value.upper(),) + if not isinstance(value, (list, tuple)): + return () + return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value)) + + @staticmethod + def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]: + unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES) + if unknown: + return unknown + if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)): + return ("TEXT",) + if GeminiRealtimeConfig._is_text_only_live_model(model) and frozenset(requested) == frozenset(("AUDIO",)): + return ("AUDIO",) + return () + + def setup_session_update_rejection( + self, model: str, session: Mapping[str, object], *, setup_sent: bool + ) -> RealtimeSessionUpdateRejection | None: + if self.drop_params: + return None + if setup_sent: + return RealtimeSessionUpdateRejection( + code="session_update_not_applied", + message=( + "Gemini Live accepts one session configuration per connection, and this session already sent " + "its configuration, so this session.update was not applied. Put every setting in session.update " + "before the first conversation content, or set drop_params to drop it instead" + ), + param="session", + ) + return self._modality_rejection(model, session) or self._voice_rejection(model, session) + + @staticmethod + def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None: + param: Final = "output_modalities" if "output_modalities" in session else "modalities" + unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities( + model, GeminiRealtimeConfig._requested_response_modalities(session.get(param)) ) + if not unsupported: + return None + return RealtimeSessionUpdateRejection( + code="invalid_value", + message=( + f"Invalid value for 'session.{param}': {model} cannot produce " + f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead" + ), + param=f"session.{param}", + ) + + def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None: + audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {} + output_voice: Final = audio_output.get("voice") + voice: Final = output_voice or session.get("voice") + if not (self._rejects_undocumented_voices(model) and _is_undocumented_live_voice(voice)): + return None + param: Final = "session.audio.output.voice" if output_voice else "session.voice" + return RealtimeSessionUpdateRejection( + code="invalid_value", + message=( + f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. " + "Set drop_params to drop it instead" + ), + param=param, + ) + + @staticmethod + def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: + """Drop modalities Gemini Live cannot produce at all, swap the ones this + model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for + text-only ones such as transcribe-live), then keep a single modality + because Gemini Live rejects a setup naming two.""" + return GeminiRealtimeConfig._single_response_modality( + GeminiRealtimeConfig._swap_unsupported_response_modalities( + model, GeminiRealtimeConfig._known_response_modalities(model, modalities) + ) + ) + + @staticmethod + def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: + normalized: Final = tuple(str(modality).upper() for modality in modalities) + known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES) + if len(known) < len(normalized): + verbose_logger.warning( + "Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s", + model, + ", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES), + ) + return known or (GeminiRealtimeConfig._default_response_modality(model),) + + @staticmethod + def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]: if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized: + verbose_logger.warning( + "Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO", + model, + ) return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",) if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized: return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",) return normalized + @staticmethod + def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]: + if len(modalities) <= 1: + return modalities + chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0] + verbose_logger.warning( + "Gemini Live: at most one response modality is allowed per session; using %s for the requested %s", + chosen, + ", ".join(modalities), + ) + return (chosen,) + @staticmethod def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]: generation_config: Final = setup.get("generationConfig") @@ -504,7 +707,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # would be silently dropped because ``map_openai_params`` only # recognises the flat OpenAI-beta key names. session_payload = self._normalize_session_payload_for_mapping(session_payload) - new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload) + new_overrides: Final = self.map_openai_params( + optional_params={}, non_default_params=session_payload, model=model + ) if session_configuration_request is None: generation_config: Final = new_overrides.setdefault("generationConfig", {}) @@ -618,14 +823,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): message_str = str(message) raise ValueError(f"Invalid JSON message: {message_str}") - messages: Final[list[str]] = [] msg_type: Final = json_message.get("type") if msg_type == "session.update": return self._handle_session_update(json_message, model, session_configuration_request) if msg_type == "response.create": - return [] # Gemini responds automatically; nothing to forward + return list(self._end_manual_activity()) if msg_type == "conversation.item.create": return self._handle_conversation_item(json_message) @@ -643,8 +847,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict}) verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend") - messages.append(gemini_msg) - return messages + return [*self._start_manual_activity(session_configuration_request), gemini_msg] if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"): return self._handle_input_audio_buffer_commit_or_end(session_configuration_request) @@ -660,22 +863,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): logging_session_id: str, session_configuration_request: str | None = None, ) -> OpenAIRealtimeStreamSessionEvents: - if session_configuration_request: - session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request) - else: - session_configuration_request_dict = {} + session_configuration_request_dict: Final = _parse_setup( + session_configuration_request or self.session_configuration_request(model) + ) _model: Final = session_configuration_request_dict.get("model") or model generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {} gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"]) _modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)] - _system_instruction: Final = session_configuration_request_dict.get("systemInstruction") + instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction")) session: Final = OpenAIRealtimeStreamSession( id=logging_session_id, modalities=_modalities, ) - if _system_instruction is not None and isinstance(_system_instruction, str): - session["instructions"] = _system_instruction + if instructions is not None: + session["instructions"] = instructions if _model is not None and isinstance(_model, str): # Strip Vertex/AI Studio path prefixes to expose the bare model name. if "/models/" in _model: @@ -1662,6 +1864,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # Deferred setup opt-in: litellm.gemini_live_defer_setup = True return not litellm.gemini_live_defer_setup + def builds_setup_from_session_update(self) -> bool: + return True + def session_configuration_request(self, model: str) -> str: """ diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 13f547b3b92..7635cd32a06 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): (they require async I/O) and injected at construction time. """ - def __init__(self, access_token: str, project: str, location: str) -> None: - super().__init__() + def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None: + super().__init__(drop_params=drop_params) self._access_token = access_token self._project = project self._location = location @@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): def _rejects_vad_tuning_with_detection_off(self) -> bool: return False + def _rejects_undocumented_voices(self, model: str) -> bool: + return True + # ------------------------------------------------------------------ # URL # ------------------------------------------------------------------ @@ -163,7 +166,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # settings would be silently dropped because ``map_openai_params`` only # recognises the flat OpenAI-beta key names. session_params = self._normalize_session_payload_for_mapping(session_params) - setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params) + setup_config: Final = self.map_openai_params(optional_params={}, non_default_params=session_params, model=model) # Use full Vertex AI model path setup_config["model"] = self._vertex_model_path(model) @@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets # ``disabled=True`` whenever ``create_response`` is absent or # ``False``. Force ``disabled=False`` only when the client did - # not explicitly request ``create_response: False`` — that path + # not explicitly request ``create_response: False`` or + # ``turn_detection: null`` — the first of those paths # is how transcription guardrails suppress automatic responses, # and overriding it here would silently bypass the guardrail. # Vertex Live has no "VAD on, no auto-response" mode, so callers # that need that behaviour must accept that VAD is off. client_turn_detection: Final = session_params.get("turn_detection") client_disabled_auto_response: Final = ( - isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False - ) + "turn_detection" in session_params and client_turn_detection is None + ) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False) realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {}) automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {}) if not client_disabled_auto_response: @@ -261,4 +265,5 @@ def vertex_realtime_config( access_token=access_token, project=project, location=VertexBase.get_vertex_region(vertex_region=location, model=model), + drop_params=drop_params, ) diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index 2ad81e64e05..67d90eefab0 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -81,6 +81,14 @@ class BidiGenerateContentServerMessage(TypedDict, total=False): """Output only. The setup complete message.""" +class BidiGenerateContentActivityStart(TypedDict): + pass + + +class BidiGenerateContentActivityEnd(TypedDict): + pass + + class BidiGenerateContentRealtimeInput(TypedDict, total=False): text: str """The text to be sent to the model.""" @@ -94,11 +102,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False): audioStreamEnd: bool """Output only. If true, indicates that the audio stream has ended.""" - activityStart: bool - """Output only. If true, indicates that the activity has started.""" + activityStart: BidiGenerateContentActivityStart + """Marks the start of user activity when automatic activity detection is disabled.""" - activityEnd: bool - """Output only. If true, indicates that the activity has ended.""" + activityEnd: BidiGenerateContentActivityEnd + """Marks the end of user activity when automatic activity detection is disabled.""" StartOfSpeechSensitivityEnum = Literal[ diff --git a/litellm/utils.py b/litellm/utils.py index e85e089289a..6f56cc65f45 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10911,7 +10911,7 @@ class ProviderConfigManager: if LlmProviders.GEMINI == provider: from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig - return GeminiRealtimeConfig() + return GeminiRealtimeConfig(drop_params=drop_params) if LlmProviders.META == provider: from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig diff --git a/tests/benchmarks/test_realtime_session_merge_benchmarks.py b/tests/benchmarks/test_realtime_session_merge_benchmarks.py new file mode 100644 index 00000000000..29ee1b638ad --- /dev/null +++ b/tests/benchmarks/test_realtime_session_merge_benchmarks.py @@ -0,0 +1,21 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.realtime_streaming import _merge_session_fields + +NESTED_LEVELS: Final = 8 +LEAVES: Final = 2000 + + +def _nested_session(depth: int) -> dict[str, object]: + return {"child": _nested_session(depth - 1)} if depth else {f"k{index}": index for index in range(LEAVES)} + + +@pytest.mark.benchmark +def test_merging_a_session_update_that_repeats_every_nested_level() -> None: + session: Final = _nested_session(NESTED_LEVELS) + + merged: Final = _merge_session_fields(session, session) + + assert merged == session diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index c943ff2cbd0..9b61c1d39ae 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -85,6 +85,7 @@ IGNORE_FUNCTIONS = [ "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. + "_merge_session_fields", # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update. ] diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 4d8336775ef..6bd5d61d721 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -47,6 +47,7 @@ class RealtimeProvider: id: str alias: str litellm_params: LiteLLMParamsBody + modalities: tuple[str, ...] = ("text",) PROVIDERS = ( @@ -75,6 +76,7 @@ PROVIDERS = ( model="gemini/gemini-3.1-flash-live-preview", api_key="os.environ/GEMINI_API_KEY", ), + modalities=("audio",), ), RealtimeProvider( "vertex_ai", @@ -84,6 +86,7 @@ PROVIDERS = ( vertex_location="us-central1", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", ), + modalities=("audio",), ), RealtimeProvider( "bedrock", diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index f5592abb34d..3b7ba3a2430 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -124,7 +124,8 @@ def test_text_conversation( session.send( SessionUpdate( session=SessionConfig( - instructions="You are a terse assistant. Reply in one short sentence." + modalities=list(provider.modalities), + instructions="You are a terse assistant. Reply in one short sentence.", ) ) ) @@ -159,6 +160,7 @@ def test_tool_call_round_trip( session.send( SessionUpdate( session=SessionConfig( + modalities=list(provider.modalities), tools=[WEATHER_TOOL], tool_choice="auto", instructions=( diff --git a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py index ad58a400cec..2712d6e30cd 100644 --- a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py +++ b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py @@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe def _live_scenario() -> RealtimeResponse: - return RealtimeResponse(content_type="application/x-realtime", events=()) + return RealtimeResponse( + content_type="application/x-realtime", + events=( + {"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}}, + {"serverContent": {"turnComplete": True}}, + ), + ) def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]: @@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, - (_step(_update_frame(update), until="session.updated"),), + ( + _step(_update_frame(update), until="session.updated"), + _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE), + ), ) observed: Final = _observed(upstream_url, handle.scenario_id) - assert session.types == ("session.created", "session.updated"), session + assert session.types[:2] == ("session.created", "session.updated"), session + assert session.types[-1] == RESPONSE_DONE, session assert _setup_detection(observed) == {"disabled": True}, _sent(observed) @@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, - (_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),), + ( + _step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"), + _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE), + ), ) observed: Final = _observed(upstream_url, handle.scenario_id) - assert session.types == ("session.created", "session.updated"), session + assert session.types[:2] == ("session.created", "session.updated"), session + assert session.types[-1] == RESPONSE_DONE, session assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index a8543ebcfc2..ba76138a268 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1,7 +1,7 @@ import asyncio import base64 import json -from collections.abc import Coroutine, Mapping, Sequence +from collections.abc import Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -14,6 +14,7 @@ from websockets.frames import Close import litellm from litellm.constants import ( BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, + REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, @@ -21,6 +22,7 @@ from litellm.constants import ( from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import ( RealTimeStreaming, + _merge_session_fields, client_sent_openai_beta_realtime_header, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1255,11 +1257,15 @@ async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup side_effect=[ json.dumps({"type": "session.update", "session": {"tools": []}}), json.dumps({"type": "session.update", "session": {"tools": []}}), + _TEXT_ITEM_CREATE, + _RESPONSE_CREATE, Exception("client done"), ] ) + websocket.send_text = AsyncMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) def _transform(message: str, model: str, session_configuration_request=None): if session_configuration_request is None: @@ -2285,6 +2291,7 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.builds_setup_from_session_update = MagicMock(return_value=False) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2351,6 +2358,7 @@ async def test_guardrail_turn_detection_injection_tolerates_non_dict_value( logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.builds_setup_from_session_update = MagicMock(return_value=False) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2413,6 +2421,7 @@ async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_act logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2465,6 +2474,7 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) follow_up_setup = json.dumps( { "setup": { @@ -2498,6 +2508,636 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): assert streaming.session_configuration_request == follow_up_setup +def _gemini_default_mode_streaming( + monkeypatch: pytest.MonkeyPatch, + client_messages: Sequence[str], + model: str = "gemini-live-2.5-flash-native-audio", + drop_params: bool = False, + defer_setup: bool = False, +) -> tuple[RealTimeStreaming, MagicMock]: + monkeypatch.setattr(litellm, "gemini_live_defer_setup", defer_setup, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws = MagicMock() + client_ws.receive_text = AsyncMock(side_effect=[*client_messages, ConnectionClosed(None, None)]) + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(drop_params=drop_params), + model=model, + ) + return streaming, backend_ws + + +def _events_sent_to_client(streaming: RealTimeStreaming) -> list[dict]: + return [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list] + + +_TEXT_ITEM_CREATE = json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + } +) + +_RESPONSE_CREATE = json.dumps({"type": "response.create"}) + + +@pytest.mark.asyncio +async def test_default_mode_sends_default_setup_before_first_non_session_client_message(monkeypatch): + """Gemini Live setup is no longer pre-sent at connect, so a client that never + sends session.update still gets the provider default setup ahead of its + first content frame.""" + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE]) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + assert streaming.session_configuration_request == backend_ws.send.await_args_list[0].args[0] + + +@pytest.mark.asyncio +async def test_default_mode_session_update_first_builds_the_only_setup(monkeypatch): + session_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + + +@pytest.mark.asyncio +async def test_default_mode_merges_every_session_update_sent_before_the_first_content_frame(monkeypatch): + """Clients configure a session over several session.update messages. Gemini + Live takes one setup, so every update sent before the first content frame is + merged into that setup instead of only the first one applying.""" + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, modalities_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_merges_nested_audio_settings_across_session_updates(monkeypatch): + vad_update = json.dumps( + {"type": "session.update", "session": {"audio": {"input": {"turn_detection": {"silence_duration_ms": 1500}}}}} + ) + voice_update = json.dumps({"type": "session.update", "session": {"audio": {"output": {"voice": "Kore"}}}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [vad_update, voice_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + setup = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["realtimeInputConfig"]["automaticActivityDetection"]["silenceDurationMs"] == 1500 + assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + +def test_merge_session_fields_replaces_nested_mappings_wholesale_past_the_depth_cap(): + def nested(levels: int, key: str) -> dict: + return {key: levels, **({"child": nested(levels - 1, key)} if levels else {})} + + def level(node, depth: int): + return node if depth == 0 else level(node["child"], depth - 1) + + merged = _merge_session_fields(nested(10, "keep"), nested(10, "new")) + + assert all({"keep", "new"} <= set(level(merged, depth)) for depth in range(9)) + assert "keep" not in level(merged, 9) + assert level(merged, 9)["new"] == 1 + + +@pytest.mark.asyncio +async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch): + """A client that waits for session.updated before sending content must not + hang while its updates are held, so each one is acknowledged right away + with the modalities the merged setup will carry.""" + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, modalities_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + acks = [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list] + assert [ack["type"] for ack in acks] == ["session.updated", "session.updated"] + assert [ack["session"].get("output_modalities") or ack["session"].get("modalities") for ack in acks] == [ + ["audio"], + ["text"], + ] + assert acks[1]["session"]["instructions"] == "Be brief." + backend_ws.send.assert_not_awaited() + + +_LATE_INSTRUCTIONS_UPDATE = json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Be brief."}} +) + + +@pytest.mark.asyncio +async def test_default_mode_answers_a_session_update_after_the_first_content_with_an_error(monkeypatch): + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE]) + + await streaming.client_ack_messages() + + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + + +@pytest.mark.asyncio +async def test_default_mode_drops_a_late_session_update_quietly_under_drop_params(monkeypatch): + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE], drop_params=True + ) + + await streaming.client_ack_messages() + + assert _events_sent_to_client(streaming) == [] + assert backend_ws.send.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "requested_output"), + [ + ("gemini-live-2.5-flash-native-audio", {"modalities": ["text"]}), + ("gemini-live-2.5-flash", {"output_modalities": ["image"]}), + ], +) +async def test_default_mode_rejects_an_output_modality_the_model_cannot_produce(monkeypatch, model, requested_output): + update: Final = json.dumps( + {"type": "session.update", "session": {**requested_output, "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [update, _TEXT_ITEM_CREATE], model=model) + + await streaming.client_ack_messages() + + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event["error"]["code"]) for event in events] == [("error", "invalid_value")] + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert "systemInstruction" not in setup + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("modalities", "drop_params"), + [(["text", "audio"], False), (["text"], True)], + ids=["openai-default-pair", "text-dropped-under-drop-params"], +) +async def test_default_mode_applies_the_update_with_audio_output_on_an_audio_only_model( + monkeypatch, modalities, drop_params +): + update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": modalities, "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], drop_params=drop_params + ) + + await streaming.client_ack_messages() + + assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated"] + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] + assert setup["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_sends_no_extra_session_updated_when_gemini_confirms_the_setup(monkeypatch): + """Each held session.update was already acknowledged, so Gemini's setupComplete + must not reach the client as one more session.updated nobody asked for.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + model: Final = "gemini-live-2.5-flash-native-audio" + config: Final = GeminiRealtimeConfig() + client_ws: Final = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=[json.dumps({"setupComplete": {}}).encode(), ConnectionClosed(None, None)]) + logging_obj: Final = _mock_logging() + logging_obj.litellm_trace_id = "trace_lazy" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, provider_config=config, model=model) + streaming.session_created_sent_to_client = True + streaming.session_configuration_request = config.session_configuration_request(model) + + await streaming.backend_to_client_send_messages() + + assert [json.loads(call.args[0])["type"] for call in client_ws.send_text.await_args_list] == [] + + +@pytest.mark.asyncio +async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_merged_setup(monkeypatch): + """A transcription guardrail gates every turn, so its auto-response disable + has to land in the one setup built from the held session.updates, injected + once rather than into every update that gets merged.""" + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + debug = MagicMock() + monkeypatch.setattr("litellm.litellm_core_utils.realtime_streaming.verbose_logger.debug", debug) + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + vad_update = json.dumps( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "silence_duration_ms": 900}}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, vad_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["realtimeInputConfig"]["automaticActivityDetection"] == {"disabled": True} + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert streaming._guardrail_turn_detection_update_sent is True + injected = [call for call in debug.call_args_list if str(call.args[0]).startswith("Injected turn_detection")] + assert len(injected) == 1 + + +def _blocking_text_guardrail() -> CustomGuardrail: + class _BlockingTextGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + if any("zebrafruit" in text for text in inputs.get("texts", [])): + raise ValueError("Content blocked") + return inputs + + return _BlockingTextGuardrail( + guardrail_name="test_text_guard", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + + +@pytest.mark.asyncio +async def test_default_mode_sends_the_setup_before_a_guardrail_blocks_the_first_message(monkeypatch): + """Gemini Live closes the socket when content arrives before the setup, so a + guardrail that blocks the first typed message must not reach the backend first.""" + monkeypatch.setattr(litellm, "callbacks", [_blocking_text_guardrail()]) + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + blocked_item = json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "zebrafruit"}]}, + } + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, blocked_item], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert "zebrafruit" not in json.dumps(frames[1]) + errors: Final = [event["error"]["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert errors == ["guardrail_violation"] + + +class _SteppedClock: + def __init__(self, epoch: float) -> None: + self.now = epoch + + def __call__(self) -> float: + return self.now + + +def _client_ws_receiving(*steps: Callable[[], Coroutine[object, object, str]]) -> MagicMock: + remaining: Final = iter(steps) + + async def receive_text() -> str: + return await next(remaining)() + + client_ws: Final = MagicMock() + client_ws.receive_text = receive_text + client_ws.send_text = AsyncMock() + return client_ws + + +@pytest.mark.asyncio +@pytest.mark.parametrize("epoch", [0.0, 1e9]) +async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_the_deadline(monkeypatch, epoch): + """Vertex AI refuses a setup sent minutes after connecting, so an idle session + sends what it holds at the deadline and answers any later session.update as late.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + clock: Final = _SteppedClock(epoch) + setup_sent: Final = asyncio.Event() + + async def instructions_update_then_the_deadline_passes() -> str: + clock.now += REALTIME_LAZY_SETUP_DEADLINE_SECONDS + return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + + async def late_update_once_the_setup_is_out() -> str: + await setup_sent.wait() + return json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ) + + async def disconnect() -> str: + raise ConnectionClosed(None, None) + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock(side_effect=lambda frame: setup_sent.set()) + streaming: Final = RealTimeStreaming( + _client_ws_receiving(instructions_update_then_the_deadline_passes, late_update_once_the_setup_is_out, disconnect), + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(), + model="gemini-live-2.5-flash", + clock=clock, + ) + + await asyncio.wait_for(streaming.client_ack_messages(), timeout=5) + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event.get("error", {}).get("code")) for event in events] == [ + ("session.updated", None), + ("error", "session_update_not_applied"), + ] + + +@pytest.mark.asyncio +async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monkeypatch): + def instructions_update(text: str) -> str: + return json.dumps({"type": "session.update", "event_id": f"evt_{text[0]}", "session": {"instructions": text}}) + + half_the_cap: Final = RealTimeStreaming._MAX_BUFFERED_BYTES // 2 + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [instructions_update("a" * half_the_cap), instructions_update("b" * half_the_cap), _TEXT_ITEM_CREATE, _RESPONSE_CREATE], + model="gemini-live-2.5-flash", + ) + + await streaming.client_ack_messages() + + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event.get("error", {}).get("event_id")) for event in events] == [ + ("session.updated", None), + ("error", "evt_b"), + ] + assert events[1]["error"]["code"] == "session_update_too_large" + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["systemInstruction"]["parts"][0]["text"].startswith("a") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("defer_setup", [False, True]) +async def test_a_session_update_between_the_first_items_and_response_create_joins_the_setup(monkeypatch, defer_setup): + modalities_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + instructions_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [modalities_update, _TEXT_ITEM_CREATE, instructions_update, _RESPONSE_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated", "session.updated"] + + +@pytest.mark.asyncio +async def test_default_mode_applies_a_session_update_that_arrives_while_audio_streams_in(monkeypatch): + modalities_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + instructions_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit"}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [modalities_update, append, instructions_update, commit], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames][:2] == ["setup", "realtimeInput"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert frames[1]["realtimeInput"]["audio"]["data"] == "AAAA" + assert "error" not in [event["type"] for event in _events_sent_to_client(streaming)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("epoch", [0.0, 1e9]) +async def test_held_items_go_out_with_the_setup_when_no_response_create_follows(monkeypatch, epoch): + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + item_sent: Final = asyncio.Event() + + async def instructions_update() -> str: + return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + + async def text_item() -> str: + return _TEXT_ITEM_CREATE + + async def disconnect_once_the_item_is_out() -> str: + await item_sent.wait() + raise ConnectionClosed(None, None) + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock( + side_effect=lambda frame: item_sent.set() if "clientContent" in json.loads(frame) else None + ) + streaming: Final = RealTimeStreaming( + _client_ws_receiving(instructions_update, text_item, disconnect_once_the_item_is_out), + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(), + model="gemini-live-2.5-flash", + lazy_setup_content_grace_seconds=0, + clock=_SteppedClock(epoch), + ) + + await asyncio.wait_for(streaming.client_ack_messages(), timeout=5) + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch): + """Pipecat resends its unchanged session.update right after its first + conversation items and treats any error event as fatal.""" + session_update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE, session_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + events: Final = _events_sent_to_client(streaming) + assert [event["type"] for event in events] == ["session.updated", "session.updated"] + assert events[0]["session"] == events[1]["session"] + + +@pytest.mark.asyncio +async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch): + """Deferred setup merges every session.update sent before the first content, + the same as the default mode, and answers a changed one after it as late.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = MagicMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}), + json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}), + _TEXT_ITEM_CREATE, + _RESPONSE_CREATE, + json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ), + ConnectionClosed(None, None), + ] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + + +@pytest.mark.asyncio +async def test_deferred_setup_sends_the_default_setup_before_text_content_that_comes_first(monkeypatch): + """Gemini Live closes the socket on content sent before any setup, so text that arrives + before a session.update in deferred mode gets the model's default setup first.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = MagicMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + _TEXT_ITEM_CREATE, + json.dumps({"type": "session.update", "event_id": "evt_late", "session": {"modalities": ["text"]}}), + ConnectionClosed(None, None), + ] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + + +@pytest.mark.asyncio +async def test_deferred_setup_sends_a_session_update_that_follows_buffered_audio_at_once(monkeypatch): + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit"}) + session_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be a pirate."}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [append, commit, session_update], model="gemini-live-2.5-flash", defer_setup=True + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + assert "Be a pirate." in json.dumps(frames[0]["setup"]["systemInstruction"]) + + +@pytest.mark.parametrize("defer_setup", [False, True]) +@pytest.mark.asyncio +async def test_a_transcription_guardrail_refuses_gemini_audio_it_cannot_screen(monkeypatch, defer_setup): + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + session_update: Final = json.dumps( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "create_response": False}}} + ) + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit", "event_id": "commit_1"}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [session_update, append, append, commit, json.dumps({"type": "response.create"}), _TEXT_ITEM_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + refusals: Final = [ + event["error"]["event_id"] + for event in _events_sent_to_client(streaming) + if event["type"] == "error" and event["error"]["code"] == "audio_input_not_screenable" + ] + assert refusals == [None, "commit_1"] + + +@pytest.mark.parametrize("defer_setup", [False, True]) +@pytest.mark.parametrize("control_type", ["response.cancel", "conversation.item.delete", "input_audio_buffer.clear"]) +@pytest.mark.asyncio +async def test_a_control_event_before_session_update_leaves_the_setup_open(monkeypatch, defer_setup, control_type): + control_event: Final = json.dumps({"type": control_type, "item_id": "item_x"}) + session_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [control_event, session_update, _TEXT_ITEM_CREATE, _RESPONSE_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert [event["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] == [] + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" @@ -2533,14 +3173,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp @pytest.mark.asyncio -async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch): +async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch): monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig client_ws = MagicMock() + client_ws.send_text = AsyncMock() audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="}) session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}}) - client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)]) + later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="}) + client_ws.receive_text = AsyncMock( + side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)] + ) backend_ws = MagicMock() backend_ws.send = AsyncMock() logging_obj: Final = _mock_logging() @@ -2560,7 +3204,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0]) assert "setup" in sent_payload assert "realtimeInput" not in sent_payload - assert streaming._pending_messages_until_setup == [audio_msg] + assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg] @pytest.mark.asyncio @@ -3523,6 +4167,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f assert session.logging.logged_sessions == () +class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig): + def builds_setup_from_session_update(self) -> bool: + return True + + +@pytest.mark.asyncio +async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal(): + upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig() + ) + + await session.run() + + assert session.logging.logged_failures == () + assert len(session.logging.logged_sessions) == 1 + + @pytest.mark.asyncio async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success(): client_ws: Final = _client_ws_that_never_sends() diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 794ac2d26d5..23d7feb2557 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -14,6 +14,7 @@ import httpcore import httpx import pytest import respx +import websockets.exceptions from botocore.credentials import RefreshableCredentials from h2.events import ConnectionTerminated @@ -2787,6 +2788,194 @@ async def test_async_realtime_error_event_send_failure_still_closes(): assert client_ws.events == [("close", (1011, "Internal server error: vertex token refresh exploded"))] +_GEMINI_LIVE_SETUP_COMPLETE = json.dumps({"setupComplete": {}}) +_GEMINI_LIVE_TEXT_TURN = json.dumps({"serverContent": {"modelTurn": {"parts": [{"text": "Blue."}]}}}) +_GEMINI_LIVE_TURN_COMPLETE = json.dumps({"serverContent": {"turnComplete": True}}) + + +class _ScriptedRealtimeBackend: + """Backend socket that answers the first setup frame with the scripted replies + and closes once the client is done, so both bridge loops run to completion.""" + + def __init__(self, replies_after_setup): + self.sent = [] + self.closed = asyncio.Event() + self._replies_after_setup = list(replies_after_setup) + self._inbox = asyncio.Queue() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def send(self, payload): + self.sent.append(payload) + if "setup" in json.loads(payload): + for reply in self._replies_after_setup: + self._inbox.put_nowait(reply) + + def close_after_replies(self): + self._inbox.put_nowait(None) + + async def recv(self, decode=True): + reply = await self._inbox.get() + if reply is None: + self.closed.set() + raise websockets.exceptions.ConnectionClosedOK(None, None) + return reply + + +class _ScriptedRealtimeClient: + def __init__(self, messages, backend): + self.scope = {"headers": []} + self.exceptions = websockets.exceptions + self.sent = [] + self._messages = list(messages) + self._backend = backend + + async def send_text(self, payload): + self.sent.append(payload) + + async def receive_text(self): + if self._messages: + return self._messages.pop(0) + self._backend.close_after_replies() + await self._backend.closed.wait() + raise websockets.exceptions.ConnectionClosedOK(None, None) + + async def close(self, code=None, reason=None): + return None + + +def _realtime_logging_obj(): + logging_obj = Mock() + logging_obj.litellm_trace_id = "trace-lit-6974" + logging_obj.async_success_handler = AsyncMock() + logging_obj.dispatch_failure_handlers = AsyncMock() + return logging_obj + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, session, backend_modality, client_modality, backend_turn, drop_params", + [ + ( + "gemini-live-2.5-flash", + {"type": "realtime", "output_modalities": ["text"]}, + "TEXT", + "text", + [_GEMINI_LIVE_TEXT_TURN], + False, + ), + ("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN], False), + ( + "gemini-live-2.5-flash-native-audio", + {"type": "realtime", "output_modalities": ["text"]}, + "AUDIO", + "audio", + [], + True, + ), + ], +) +async def test_async_realtime_vertex_first_session_update_builds_the_only_setup( + model, session, backend_modality, client_modality, backend_turn, drop_params: bool, monkeypatch +): + """Regression for text-only Vertex Live sessions answered in audio: the + connect-time default setup (AUDIO) used to go out before the client's + session.update, and Gemini accepts only one setup per connection, so the + requested modality was dropped and billed at the audio rate. The setup now + goes out on the client's first message, built from session.update when that + comes first. Under drop_params an audio-only model still gets AUDIO, and the + client learns the downgrade from session.updated.""" + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + backend = _ScriptedRealtimeBackend([_GEMINI_LIVE_SETUP_COMPLETE, *backend_turn, _GEMINI_LIVE_TURN_COMPLETE]) + client_ws = _ScriptedRealtimeClient( + [ + json.dumps({"type": "session.update", "session": session}), + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What color is the sky? One word."}], + }, + } + ), + json.dumps({"type": "response.create"}), + ], + backend, + ) + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)): + await asyncio.wait_for( + handler.async_realtime( + model=model, + websocket=client_ws, + logging_obj=_realtime_logging_obj(), + provider_config=VertexAIRealtimeConfig( + access_token="fake-token", project="fake-project", location="us-central1", drop_params=drop_params + ), + headers={}, + ), + timeout=10, + ) + + backend_frames = [json.loads(frame) for frame in backend.sent] + setups = [frame["setup"] for frame in backend_frames if "setup" in frame] + assert len(setups) == 1 + assert "setup" in backend_frames[0] + assert setups[0]["generationConfig"]["responseModalities"] == [backend_modality] + assert any("clientContent" in frame for frame in backend_frames[1:]) + + client_events = [json.loads(frame) for frame in client_ws.sent] + event_types = [event["type"] for event in client_events] + assert event_types[:2] == ["session.created", "session.updated"] + assert client_events[1]["session"]["modalities"] == [client_modality] + if backend_turn: + assert "response.output_text.delta" in event_types + assert "response.output_audio_transcript.delta" not in event_types + + +@pytest.mark.asyncio +async def test_async_realtime_sends_setup_at_connect_when_provider_cannot_build_it_from_session_update(): + """Bedrock Nova Sonic style providers keep the connect-time setup and get no + synthetic session.created.""" + setup = json.dumps({"event": {"sessionStart": {}}}) + backend = _ScriptedRealtimeBackend([]) + client_ws = _ScriptedRealtimeClient([], backend) + provider_config = Mock() + provider_config.get_complete_url.return_value = "wss://backend.example/live" + provider_config.validate_environment.return_value = {} + provider_config.requires_session_configuration.return_value = True + provider_config.builds_setup_from_session_update.return_value = False + provider_config.session_configuration_request.return_value = setup + provider_config.unbilled_usage_on_session_close.return_value = None + provider_config.open_backend = AsyncMock(return_value=None) + provider_config.wrap_backend.side_effect = lambda opened: opened + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)): + await asyncio.wait_for( + handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=client_ws, + logging_obj=_realtime_logging_obj(), + provider_config=provider_config, + headers={}, + ), + timeout=10, + ) + + assert backend.sent == [setup] + assert client_ws.sent == [] + + class _JSONBodyAudioTranscriptionConfig(BaseAudioTranscriptionConfig): def get_supported_openai_params(self, model): return [] diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 017f279d6c4..460a50ed61a 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1,3 +1,4 @@ +import base64 import json from collections.abc import Mapping from typing import Final, cast @@ -7,7 +8,7 @@ import pytest import litellm -from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig +from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -439,6 +440,51 @@ def test_gemini_realtime_session_update_with_tools(): assert "parameters" in function_decl +def _schema_nodes(schema: object): + if isinstance(schema, dict): + yield schema + for value in schema.values(): + yield from _schema_nodes(value) + elif isinstance(schema, list): + for value in schema: + yield from _schema_nodes(value) + + +def test_gemini_session_update_converts_flat_realtime_tool_schemas_to_gemini_schemas(): + session_update: Final = { + "type": "session.update", + "session": { + "tools": [ + { + "type": "function", + "name": "lookup_weather", + "description": "Look up the weather in a city.", + "parameters": { + "type": "object", + "$defs": { + "City": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]} + }, + "properties": {"city": {"$ref": "#/$defs/City"}, "note": {"type": ["string", "null"]}}, + "required": ["city"], + }, + } + ] + }, + } + + messages: Final = GeminiRealtimeConfig().transform_realtime_request( + json.dumps(session_update), "gemini-2.5-flash", session_configuration_request=None + ) + + declaration: Final = json.loads(messages[0])["setup"]["tools"][0]["function_declarations"][0] + assert declaration["name"] == "lookup_weather" + parameters: Final = declaration["parameters"] + assert parameters["properties"]["city"]["properties"]["name"]["type"] == "string" + nodes: Final = tuple(_schema_nodes(parameters)) + assert not any("$defs" in node or "$ref" in node for node in nodes) + assert all(isinstance(node["type"], str) for node in nodes if "type" in node) + + def test_gemini_session_update_defaults_to_audio_modality(): config = GeminiRealtimeConfig() @@ -613,6 +659,35 @@ def test_gemini_audio_only_live_models_drop_text_from_text_audio_combo(patch_gem assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] +@pytest.mark.parametrize("output_modalities", ["text", ["text"]]) +def test_gemini_session_update_accepts_a_single_string_output_modality(output_modalities): + """Regression: a bare string was iterated per character, so output_modalities + "text" reached Vertex as responseModalities ["T"], which it rejects with 1007.""" + config = GeminiRealtimeConfig() + session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}} + + messages = config.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + + +@pytest.mark.parametrize( + ("output_modalities", "expected"), + [(["video", "text"], ["TEXT"]), (["video"], ["AUDIO"]), (None, ["AUDIO"])], +) +def test_gemini_session_update_drops_modalities_live_cannot_produce(output_modalities, expected): + config = GeminiRealtimeConfig() + session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}} + + messages = config.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == expected + + def test_gemini_non_live_model_preserves_text_modality(): config = GeminiRealtimeConfig() session_update = { @@ -1396,22 +1471,71 @@ def test_gemini_input_audio_buffer_clear_is_local_noop(): assert messages == [] -def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad(): +_MANUAL_VAD_SETUP = json.dumps({"setup": {"realtimeInputConfig": {"automaticActivityDetection": {"disabled": True}}}}) +_APPEND = json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 80).decode()}) + + +def _realtime_input_keys(config: GeminiRealtimeConfig, client_event: str, setup: str | None) -> list[str]: + frames = config.transform_realtime_request(client_event, "gemini-live-2.5-flash-native-audio", setup) + return [key for frame in frames for key in json.loads(frame)["realtimeInput"]] + + +@pytest.mark.parametrize("turn_end", ["input_audio_buffer.commit", "response.create"]) +def test_gemini_manual_vad_wraps_each_audio_turn_in_activity_start_and_end(turn_end): + """With activity detection off, Gemini Live answers only after activityEnd closes an + activityStart, and it closes the socket on any other shape of those markers.""" config = GeminiRealtimeConfig() - setup = { - "setup": { - "realtimeInputConfig": { - "automaticActivityDetection": {"disabled": True}, - } - } - } - messages = config.transform_realtime_request( - json.dumps({"type": "input_audio_buffer.commit"}), - "gemini-live-2.5-flash-native-audio", - session_configuration_request=json.dumps(setup), + end = json.dumps({"type": turn_end}) + + turns = [ + [_realtime_input_keys(config, event, _MANUAL_VAD_SETUP) for event in (_APPEND, _APPEND, end, end)] + for _ in range(2) + ] + + assert turns == [[["activityStart", "audio"], ["audio"], ["activityEnd"], []]] * 2 + + +def test_gemini_manual_vad_activity_markers_are_empty_messages(): + config = GeminiRealtimeConfig() + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + first_append = config.transform_realtime_request(_APPEND, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP) + turn_end = config.transform_realtime_request(commit, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP) + + assert json.loads(first_append[0]) == {"realtimeInput": {"activityStart": {}}} + assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}] + + +@pytest.mark.parametrize( + ("session", "expected_keys"), + [ + ({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]), + ], +) +def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys): + """OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself.""" + config = GeminiRealtimeConfig() + setup = config.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None ) - assert len(messages) == 1 - assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}} + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)] + + assert keys == expected_keys + + +def test_gemini_automatic_vad_sends_no_activity_markers(): + config = GeminiRealtimeConfig() + commit = json.dumps({"type": "input_audio_buffer.commit"}) + response_create = json.dumps({"type": "response.create"}) + + keys = [_realtime_input_keys(config, event, None) for event in (_APPEND, commit)] + + assert keys == [["audio"], ["audioStreamEnd"]] + assert config.transform_realtime_request(response_create, "gemini-live-2.5-flash", None) == [] def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped(): @@ -1883,13 +2007,75 @@ def test_map_openai_params_drops_stock_voice_case_insensitively(): """ cfg = GeminiRealtimeConfig() - dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"}) + dropped = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Alloy"}, model="gemini-3.8-live") assert "speechConfig" not in dropped.get("generationConfig", {}) - passthrough = cfg.map_openai_params(optional_params={}, non_default_params={"voice": "Kore"}) + passthrough = cfg.map_openai_params( + optional_params={}, non_default_params={"voice": "Kore"}, model="gemini-3.8-live" + ) assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" +_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize() + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_google_ai_studio_forwards_a_documented_voice(drop_params): + config: Final = GeminiRealtimeConfig(drop_params=drop_params) + mapped: Final = config.map_openai_params( + optional_params={}, non_default_params={"voice": _DOCUMENTED_VOICE}, model="gemini-3.8-live" + ) + + speech_config: Final = mapped["generationConfig"]["speechConfig"] + assert speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == _DOCUMENTED_VOICE + assert config.setup_session_update_rejection("gemini-3.8-live", {"voice": _DOCUMENTED_VOICE}, setup_sent=False) is None + + +@pytest.mark.parametrize( + ("session", "param"), + [ + ({"voice": "Nova"}, "session.voice"), + ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"), + ], +) +def test_google_ai_studio_rejects_an_undocumented_voice_on_the_session_update(session, param): + rejection: Final = GeminiRealtimeConfig().setup_session_update_rejection( + "gemini-2.5-flash-native-audio-preview-12-2025", session, setup_sent=False + ) + + assert rejection is not None + assert (rejection.code, rejection.param) == ("invalid_value", param) + + +def _setup_speech_config(config: GeminiRealtimeConfig, model: str, session: dict[str, object]) -> object: + messages: Final = config.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), model, session_configuration_request=None + ) + return json.loads(messages[0])["setup"].get("generationConfig", {}).get("speechConfig") + + +@pytest.mark.parametrize( + "session", + [{"voice": "Nova"}, {"type": "realtime", "audio": {"output": {"voice": "Nova"}}}], +) +def test_google_ai_studio_drops_an_undocumented_voice_with_drop_params(session): + config: Final = GeminiRealtimeConfig(drop_params=True) + model: Final = "gemini-2.5-flash-native-audio-preview-12-2025" + + assert config.setup_session_update_rejection(model, session, setup_sent=False) is None + assert _setup_speech_config(config, model, session) is None + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("model", ["gemini-3.8-live", "gemini-3.1-flash-live-preview"]) +def test_google_ai_studio_forwards_any_voice_to_models_that_accept_it(model, drop_params): + config: Final = GeminiRealtimeConfig(drop_params=drop_params) + + assert config.setup_session_update_rejection(model, {"voice": "Nova"}, setup_sent=False) is None + speech_config: Final = _setup_speech_config(config, model, {"voice": "Nova"}) + assert speech_config == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Nova"}}} + + @pytest.fixture(autouse=False) def patch_gemini_transcribe_live_cost_map_entry(monkeypatch): """Inject the gemini-3.5-transcribe-live registry entry locally. @@ -1954,6 +2140,55 @@ def test_gemini_transcribe_live_coerces_audio_modality_to_text(modalities, patch assert setup["generationConfig"]["responseModalities"] == ["TEXT"] +@pytest.mark.parametrize("model", ["gemini-3.5-transcribe-live", "gemini/gemini-3.5-transcribe-live"]) +def test_gemini_transcribe_live_synthetic_session_created_reports_text_output( + model, patch_gemini_transcribe_live_cost_map_entry +): + session = GeminiRealtimeConfig().transform_session_created_event(model, "sess_transcribe")["session"] + + assert session["modalities"] == ["text"] + + +@pytest.mark.parametrize( + ("model", "modalities", "producible"), + [ + ("gemini-3.5-transcribe-live", ["audio"], False), + ("gemini-3.5-transcribe-live", ["audio", "text"], True), + ("gemini-live-2.5-flash-native-audio", ["text"], False), + ("gemini-live-2.5-flash-native-audio", ["text", "audio"], True), + ], +) +@pytest.mark.parametrize("drop_params", [False, True]) +def test_gemini_live_rejects_only_output_modalities_the_model_cannot_produce_at_all( + model, + modalities, + producible, + drop_params, + patch_gemini_transcribe_live_cost_map_entry, + patch_gemini_audio_cost_map_entries, +): + rejection = GeminiRealtimeConfig(drop_params=drop_params).setup_session_update_rejection( + model, {"output_modalities": modalities}, setup_sent=False + ) + + assert (rejection is None) is (drop_params or producible) + assert rejection is None or rejection.param == "session.output_modalities" + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_gemini_provider_realtime_config_carries_drop_params(drop_params): + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + model = "gemini-live-2.5-flash-native-audio" + config = ProviderConfigManager.get_provider_realtime_config( + model=model, provider=LlmProviders.GEMINI, drop_params=drop_params + ) + + rejection = config.setup_session_update_rejection(model, {"modalities": ["text"]}, setup_sent=False) + assert (rejection is None) is drop_params + + def test_gemini_chat_model_with_text_output_modalities_keeps_audio_eager_setup( patch_gemini_transcribe_live_cost_map_entry, ): @@ -2169,6 +2404,47 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None +def test_gemini_realtime_two_output_modalities_collapse_to_audio(): + """Gemini Live closes the socket with 1007 when the setup names two response + modalities, so a GA session.update asking for ["text", "audio"] must reach the + backend as AUDIO only, the same choice the beta shim makes.""" + config = GeminiRealtimeConfig() + session_update = { + "type": "session.update", + "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + + +def test_gemini_realtime_inf_max_response_output_tokens_is_not_forwarded(): + """OpenAI's default max_response_output_tokens is the string "inf", which Gemini + rejects with 1007 (TYPE_INT32), so only integer limits map to maxOutputTokens.""" + config = GeminiRealtimeConfig() + + def generation_config_for(limit): + session_update = { + "type": "session.update", + "session": {"modalities": ["text"], "max_response_output_tokens": limit}, + } + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + return json.loads(messages[0])["setup"]["generationConfig"] + + assert "maxOutputTokens" not in generation_config_for("inf") + assert generation_config_for(200)["maxOutputTokens"] == 200 + + def _grounded_live_frame(grounding_metadata: Mapping[str, object] | None) -> Mapping[str, object]: """One Live server frame. Grounding metadata and usageMetadata arrive together, as Vertex sends them.""" from typing import Final diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 0bfff870d15..7f0106616ce 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -18,6 +18,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket import litellm from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig # --------------------------------------------------------------------------- @@ -179,6 +180,25 @@ def test_vertex_audio_only_live_model_coerces_text_modality_to_audio( assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] +@pytest.mark.parametrize("drop_params", [False, True]) +def test_vertex_realtime_config_carries_drop_params(drop_params, patch_native_audio_cost_map_entry): + from litellm.llms.vertex_ai.realtime.transformation import vertex_realtime_config + + cfg: Final = vertex_realtime_config( + _NATIVE_AUDIO_MODEL, + access_token="tok", + resolve_access_token=AsyncMock(return_value="tok"), + project="my-proj", + location="us-central1", + drop_params=drop_params, + ) + + rejection: Final = cfg.setup_session_update_rejection( + _NATIVE_AUDIO_MODEL, {"modalities": ["text"]}, setup_sent=False + ) + assert (rejection is None) is drop_params + + def test_vertex_session_update_normalizes_ga_remapped_fields( patch_native_audio_cost_map_entry, ): @@ -253,6 +273,26 @@ def test_vertex_keeps_turn_detection_tuning_when_auto_response_is_off(): } + +@pytest.mark.parametrize( + ("session", "disabled"), + [ + ({"turn_detection": None}, True), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, True), + ({"type": "realtime", "audio": {"input": {}}}, False), + ], +) +def test_vertex_turns_activity_detection_off_for_null_turn_detection(session, disabled): + """OpenAI's turn_detection null is push-to-talk, so Vertex must wait for the client's activity markers.""" + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + messages: Final = cfg.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + setup: Final = json.loads(messages[0])["setup"] + assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is disabled + # --------------------------------------------------------------------------- # Round-trip test: text-in / text-out via RealTimeStreaming # --------------------------------------------------------------------------- @@ -607,3 +647,73 @@ def test_vertex_native_audio_unmapped_voice_passes_through(patch_native_audio_co generation_config = json.loads(messages[0])["setup"]["generationConfig"] assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + +def test_vertex_session_update_two_output_modalities_collapse_to_audio(): + """Vertex Live rejects a setup naming two response modalities, so a GA + ["text", "audio"] request must be sent as AUDIO only.""" + cfg = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="global") + session_update = { + "type": "session.update", + "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."}, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + + +@pytest.mark.parametrize( + ("session", "param"), + [ + ({"voice": "Nova"}, "session.voice"), + ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"), + ], +) +def test_vertex_rejects_an_undocumented_voice_on_the_session_update(session, param): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + + rejection: Final = cfg.setup_session_update_rejection("gemini-live-2.5-flash", session, setup_sent=False) + + assert rejection is not None + assert (rejection.code, rejection.param) == ("invalid_value", param) + + +@pytest.mark.parametrize("voice", ["Kore", "alloy"]) +def test_vertex_accepts_documented_and_openai_stock_voices_on_the_session_update(voice): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", {"voice": voice}, setup_sent=False) is None + + +def test_vertex_drops_an_undocumented_voice_with_drop_params(): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1", drop_params=True) + session_update: Final = {"type": "session.update", "session": {"voice": "Nova", "instructions": "Be brief."}} + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None + messages: Final = cfg.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {}) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("voice", ["fable", "onyx"]) +def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params): + cfg: Final = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params + ) + session_update: Final = {"type": "session.update", "session": {"voice": voice}} + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None + messages: Final = cfg.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}} + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx index 117796d658e..56135c779e9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx @@ -173,6 +173,24 @@ describe("RealtimePlayground", () => { expect(update.session.type).toBe("realtime"); }); + it("starts recording on the configuration the session was created with", async () => { + const user = userEvent.setup(); + render(); + + await connect(user); + await act(async () => { + latestSocket().emit({ type: "session.created" }); + }); + await user.click(screen.getByTitle("Start recording")); + + const updates = latestSocket() + .sent.map((raw) => JSON.parse(raw)) + .filter((payload) => payload.type === "session.update"); + expect(updates).toHaveLength(2); + expect(updates[1]).toEqual(updates[0]); + expect(updates[0].session.turn_detection).toEqual({ type: "server_vad" }); + }); + it("sends what was typed and then asks for a response", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx index 01151203e65..408cbd58593 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx @@ -22,6 +22,20 @@ interface RealtimePlaygroundProps { selectedGuardrails?: string[]; } +const sessionUpdate = (voice: string) => + JSON.stringify({ + type: "session.update", + session: { + type: "realtime", + modalities: ["text", "audio"], + voice, + input_audio_format: "pcm16", + output_audio_format: "pcm16", + input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, + turn_detection: { type: "server_vad" }, + }, + }); + const RealtimePlayground: React.FC = ({ accessToken, selectedModel, @@ -125,21 +139,7 @@ const RealtimePlayground: React.FC = ({ const type = data.type; if (type === "session.created") { - // GA: session.type is required ("realtime" | "transcription") - ws.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: null, - }, - }), - ); + ws.send(sessionUpdate(selectedVoice)); } else if (type === "session.updated") { // session configured } else if ( @@ -231,22 +231,7 @@ const RealtimePlayground: React.FC = ({ const startRecording = useCallback(async () => { if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; - // Switch to server VAD mode for voice input - // GA: session.type is required - wsRef.current.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: { type: "server_vad" }, - }, - }), - ); + wsRef.current.send(sessionUpdate(selectedVoice)); try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -301,7 +286,7 @@ const RealtimePlayground: React.FC = ({ } catch (err: any) { addMessage("status", `Microphone error: ${err.message}`); } - }, [addMessage]); + }, [addMessage, selectedVoice]); const stopRecording = useCallback(() => { processorRef.current?.disconnect(); @@ -317,21 +302,7 @@ const RealtimePlayground: React.FC = ({ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; if (configureSessionRef.current) return; configureSessionRef.current = true; - // GA: session.type is required - wsRef.current.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: null, - }, - }), - ); + wsRef.current.send(sessionUpdate(selectedVoice)); }, [selectedVoice]); const sendTextMessage = useCallback(() => { From 373f1752d30c70f8d2c87022e677f4c0ec58e14e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 02:21:39 -0700 Subject: [PATCH 32/34] fix(anthropic): send PDF image_url content to Claude as document blocks (#46984) * fix(anthropic): convert PDF image_url content to documents Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(prompt_templates): strip content-type parameters from fetched media Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(prompt_templates): format MIME normalization Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 6 +- .../prompt_templates/factory.py | 29 +++----- ...llm_core_utils_prompt_templates_factory.py | 74 +++++++++++++++++++ .../litellm_core_utils/test_image_handling.py | 31 ++++++++ 4 files changed, 119 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b3f89b33d1d..862e61cb631 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2023,12 +2023,14 @@ def infer_content_type_from_url_and_content( """ from litellm.litellm_core_utils.token_counter import get_image_type + media_type: Final = current_content_type.split(";", 1)[0].strip().lower() if current_content_type else None + # If we have a valid content type that's not generic, use it - if current_content_type and current_content_type not in [ + if media_type and media_type not in [ "binary/octet-stream", "application/octet-stream", ]: - return current_content_type + return media_type # Extension to MIME type mapping # Supports images, documents, and other common file types diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 834d26b4fe8..de896cde4dc 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -912,9 +912,9 @@ def create_anthropic_image_param( image_url_input: str | dict, format: str | None = None, is_bedrock_invoke: bool = False, -) -> AnthropicMessagesImageParam: +) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: """ - Create an AnthropicMessagesImageParam from an image URL input. + Create an Anthropic content block from an image URL input. Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding. """ @@ -933,14 +933,7 @@ def create_anthropic_image_param( if is_bedrock_invoke or image_url.startswith("http://"): base64_url: Final = convert_url_to_base64(url=image_url) image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) - return AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSource( - type="base64", - media_type=image_chunk["media_type"], - data=image_chunk["data"], - ), - ) + return _anthropic_content_element_factory(image_chunk) else: # HTTPS URL - pass directly for regular Anthropic return AnthropicMessagesImageParam( @@ -953,14 +946,7 @@ def create_anthropic_image_param( else: # Convert to base64 for data URIs or other formats image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format) - return AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSource( - type="base64", - media_type=image_chunk["media_type"], - data=image_chunk["data"], - ), - ) + return _anthropic_content_element_factory(image_chunk) # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex @@ -1910,7 +1896,12 @@ def convert_to_anthropic_tool_result( anthropic_content_element=_anthropic_image_param, original_content_element=content, ) - anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param)) + anthropic_content_list.append( + cast( + AnthropicMessagesImageParam | AnthropicMessagesDocumentParam, + _anthropic_image_param, + ) + ) elif content["type"] == "tool_reference": anthropic_content_list.append(ToolReference(type="tool_reference", tool_name=content["tool_name"])) elif content["type"] == "file": diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 53be3608979..778c9d68be5 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1246,6 +1246,80 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block(): assert thinking_blocks[0]["thinking"] == "genuine anthropic reasoning" +@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai")) +def test_anthropic_messages_pt_converts_pdf_image_url_data_uri_to_document(llm_provider: str) -> None: + pdf_data = "JVBERi0xLjQK" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": {"url": f"data:application/pdf;base64,{pdf_data}"}}, + ], + } + ], + model="claude-opus-5-5", + llm_provider=llm_provider, + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + } + + +@pytest.mark.parametrize("llm_provider", ("anthropic", "vertex_ai")) +def test_anthropic_messages_pt_uses_pdf_format_for_generic_mime_data_uri(llm_provider: str) -> None: + pdf_data = "JVBERi0xLjQK" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + { + "type": "image_url", + "image_url": { + "url": f"data:application/octet-stream;base64,{pdf_data}", + "format": "application/pdf", + }, + }, + ], + } + ], + model="claude-opus-5-5", + llm_provider=llm_provider, + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_data}, + } + + +def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None: + image_data = "iVBORw0KGgo=" + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": f"data:image/png;base64,{image_data}"}, + ], + } + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": image_data}, + } + + EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize( "empty_user_message", [{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}], diff --git a/tests/unit/litellm_core_utils/test_image_handling.py b/tests/unit/litellm_core_utils/test_image_handling.py index 9d9aef0c22a..d1bebde9917 100644 --- a/tests/unit/litellm_core_utils/test_image_handling.py +++ b/tests/unit/litellm_core_utils/test_image_handling.py @@ -288,6 +288,37 @@ def test_convert_url_to_base64_keeps_a_real_content_type(monkeypatch): assert result.startswith("data:image/jpeg;base64,") +@pytest.mark.parametrize( + ("url", "content_type", "expected"), + [ + ( + "https://docs.example/document.pdf", + "application/pdf; qs=0.001", + "data:application/pdf;base64,cGRm", + ), + ( + "https://img.example/image.png", + "image/png; charset=binary", + "data:image/png;base64,cGRm", + ), + ( + "https://docs.example/document.pdf", + "application/octet-stream; charset=binary", + "data:application/pdf;base64,cGRm", + ), + ], +) +def test_convert_url_to_base64_uses_bare_content_type( + monkeypatch: pytest.MonkeyPatch, + url: str, + content_type: str, + expected: str, +) -> None: + monkeypatch.setattr(litellm, "module_level_client", _stub_image_client(b"pdf", content_type)) + + assert convert_url_to_base64(url) == expected + + def test_convert_url_to_base64_raises_when_no_content_type_is_determinable(monkeypatch): monkeypatch.setattr( litellm, From d1758cda91a663b4e00b35af0921389bc6e2c95e Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Fri, 30 Oct 2026 02:40:01 -0700 Subject: [PATCH 33/34] fix(router)!: keep a deployment's own model_id over the request body's (#49045) * fix(router): keep a deployment's own model_id over the request body's A request body model_id overrode the deployment's pinned model_id, so a client could move SageMaker chat and embedding calls to another inference component. The router now drops the request's model_id when the deployment sets one. * fix(router): let deployments opt back in to a client model_id and stop client component headers on SageMaker chat Drop the request's model_id after the router defaults are applied, so a default_litellm_params model_id cannot win over the deployment either, and apply the same precedence on the sync text_completion path. A deployment that lists model_id in configurable_clientside_auth_params keeps the client's value. SageMaker chat drops any case variant of a client's X-Amzn-SageMaker-Inference-Component header when the deployment sets a component * fix(router): keep a SageMaker deployment's component when it pins it through extra_headers * fix(router): read the SageMaker component header as a pin only on SageMaker deployments * fix(sagemaker): send the inference component only as a header, never as a body model_id SageMaker picks the component from X-Amzn-SageMaker-Inference-Component, so the chat container body and legacy parameters no longer carry model_id. Nova endpoints accept a fixed body field list, so a pinned Nova deployment could reject the extra field. * fix(health): probe header-pinned SageMaker deployments at their component Health checks bypass the router, so a legacy sagemaker/ deployment pinned only through its inference component header sent no component on /health while routed requests now send it. Also update the two SageMaker integration tests that still asserted the old precedence and the dropped body field. * fix(health): apply the router's model_id pin rule to health probes Health probes now share the router's pin function, so a deployment that lets clients set model_id gets no header pin and its own model_id wins without reading the header. Header keys that are not strings are skipped when reading the inference component header instead of failing the call. * fix(sagemaker): keep header keys that aren't strings and report a string opt-in in health checks The chat transformation now keeps extra_headers keys that aren't strings, so such a deployment fails the way it did before instead of answering 500, and the model_id pin checks configurable_clientside_auth_params with isinstance, so a string there reaches the health probe's own error capture again. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/sagemaker/chat/transformation.py | 14 +- litellm/llms/sagemaker/common_utils.py | 33 ++- .../sagemaker/completion/transformation.py | 6 +- litellm/proxy/health_check.py | 8 +- litellm/router.py | 7 + litellm/router_utils/common_utils.py | 11 + .../providers/test_sagemaker_chat_wire.py | 2 +- .../test_sagemaker_embedding_wire.py | 4 +- .../test_sagemaker_chat_transformation.py | 20 ++ .../test_sagemaker_completion_handler.py | 21 ++ .../proxy/test_health_check_max_tokens.py | 82 +++++++ tests/unit/test_router/test_router.py | 219 ++++++++++++++++++ 12 files changed, 414 insertions(+), 13 deletions(-) diff --git a/litellm/llms/sagemaker/chat/transformation.py b/litellm/llms/sagemaker/chat/transformation.py index 4a59d5f1802..76eafcd6bb6 100644 --- a/litellm/llms/sagemaker/chat/transformation.py +++ b/litellm/llms/sagemaker/chat/transformation.py @@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import LlmProviders from ...openai.chat.gpt_transformation import OpenAIGPTConfig -from ..common_utils import AWSEventStreamDecoder, SagemakerError +from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -59,7 +59,12 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): inference_component_name: Final = optional_params.get("model_id") if not isinstance(inference_component_name, str): return headers - return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name} + other_headers: Final = { + name: value + for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items() + if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower() + } + return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name} def transform_request( self, @@ -76,9 +81,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): litellm_params=litellm_params, headers=headers, ) - container_request: Final = { - key: value for key, value in request.items() if key not in self.aws_authentication_params - } + header_only_params: Final = frozenset((*self.aws_authentication_params, "model_id")) + container_request: Final = {key: value for key, value in request.items() if key not in header_only_params} served_model_name: Final = litellm_params.get("hf_model_name") if not isinstance(served_model_name, str): return container_request diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index eaa805e7769..bb7719a19a1 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -14,9 +14,10 @@ from typing_extensions import ReadOnly, Required, TypedDict from litellm import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import GenericStreamingChunk as GChunk -from litellm.types.utils import StreamingChatCompletionChunk +from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True)) @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) @@ -167,6 +168,36 @@ def get_sagemaker_response_stream_shape(): return _load_sagemaker_response_stream_shape() +INFERENCE_COMPONENT_HEADER: Final = "X-Amzn-SageMaker-Inference-Component" + + +_SAGEMAKER_PROVIDERS: Final = frozenset( + provider.value for provider in (LlmProviders.SAGEMAKER, LlmProviders.SAGEMAKER_CHAT, LlmProviders.SAGEMAKER_NOVA) +) + + +def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None: + custom_llm_provider: Final = litellm_params.get("custom_llm_provider") + if isinstance(custom_llm_provider, str) and custom_llm_provider: + return custom_llm_provider + model: Final = litellm_params.get("model") + return model.split("/", 1)[0] if isinstance(model, str) else None + + +def deployment_inference_component(litellm_params: Mapping[str, object]) -> str | None: + headers: Final = litellm_params.get("extra_headers") + if _deployment_provider(litellm_params) not in _SAGEMAKER_PROVIDERS or not isinstance(headers, Mapping): + return None + return next( + ( + value + for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items() + if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str) + ), + None, + ) + + def with_inference_component_hint(error_message: str) -> str: if "Inference Component Name header is required" in error_message: return f"{error_message}\n pass in via `litellm.embedding(..., model_id={{InferenceComponentName}})`" diff --git a/litellm/llms/sagemaker/completion/transformation.py b/litellm/llms/sagemaker/completion/transformation.py index a69846c816f..5f6bef4246e 100644 --- a/litellm/llms/sagemaker/completion/transformation.py +++ b/litellm/llms/sagemaker/completion/transformation.py @@ -23,7 +23,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, Usage from litellm.utils import token_counter -from ..common_utils import SagemakerError +from ..common_utils import STR_KEYED_MAPPING, SagemakerError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -153,7 +153,9 @@ class SagemakerConfig(BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - inference_params: Final = optional_params.copy() + inference_params: Final = { + key: value for key, value in STR_KEYED_MAPPING.validate_python(optional_params).items() if key != "model_id" + } stream: Final = inference_params.pop("stream", False) data: Final[dict] = {"parameters": inference_params} if stream is True: diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 73c2fc35492..20d920f1104 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -32,7 +32,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import ( classify_strategy_router_model, strategy_router_dependencies, ) -from litellm.router_utils.common_utils import is_model_capability_field +from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params # Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy @@ -855,6 +855,8 @@ def update_litellm_params_for_health_check( `deployment_model` (default: the incoming `litellm_params` model), so a `health_check_model` or a `model` in `health_check_params` is judged on its own. Keys already set through `health_check_params` are kept, and `modalities` set there without "audio" turns the audio fields off + - sends a SageMaker deployment's `X-Amzn-SageMaker-Inference-Component` extra header as `model_id` when the + deployment sets none, so legacy `sagemaker/` probes reach that component the way routed requests do """ incoming_model: Final = _PROBE_PARAMS_ADAPTER.validate_python(litellm_params).get("model") base_model_owner: Final = deployment_model or (incoming_model if isinstance(incoming_model, str) else None) @@ -935,9 +937,11 @@ def update_litellm_params_for_health_check( audio_output_params: Final = _health_check_audio_output_params( _MODEL_INFO_ADAPTER.validate_python(model_info), probe_params, mode, base_model_owner ) + model_id_pin: Final = deployment_model_id_pin(probe_params) + model_id_params: Final = {} if model_id_pin is None else {"model_id": model_id_pin} return { key: value - for key, value in {**audio_output_params, **probe_params}.items() + for key, value in {**audio_output_params, **model_id_params, **probe_params}.items() if not is_model_capability_field(key) } diff --git a/litellm/router.py b/litellm/router.py index 20af2e5649e..af2181cedb5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -255,6 +255,7 @@ from litellm.router_utils.clientside_credential_handler import ( from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, as_retry_skipped_deployment_ids, + deployment_model_id_pin, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, @@ -5316,6 +5317,9 @@ class Router(DeprecatedRouterMembers): ): kwargs[STREAM_USAGE_INJECTED_FIELD] = True self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name) + pinned_model_id: Final = deployment_model_id_pin(deployment_context) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this method's contract is filling the caller's kwargs hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs) if _is_fallback_hop(hop_kwargs): EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( @@ -6242,6 +6246,9 @@ class Router(DeprecatedRouterMembers): kwargs[k] = v elif k == "metadata": kwargs[k].update(v) + pinned_model_id: Final = deployment_model_id_pin(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if pinned_model_id is not None: + kwargs["model_id"] = pinned_model_id # rebind-ok: this call's own kwargs, filled like the defaults # call via litellm.completion() return litellm.text_completion( diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 2451422b5e6..a06860e7786 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -16,6 +16,7 @@ from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure +from litellm.llms.sagemaker.common_utils import deployment_inference_component from litellm.types.router import ( CredentialLiteLLMParams, Deployment, @@ -522,6 +523,16 @@ PROVIDER_SPECIFIC_CAPABILITY_FIELDS: Final = frozenset(ProviderSpecificModelInfo _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +def deployment_model_id_pin(deployment_litellm_params: Mapping[str, object]) -> object | None: + clientside_params: Final = deployment_litellm_params.get("configurable_clientside_auth_params") + if isinstance(clientside_params, list | tuple) and "model_id" in clientside_params: + return None + own_model_id: Final = deployment_litellm_params.get("model_id") + if own_model_id is not None: + return own_model_id + return deployment_inference_component(deployment_litellm_params) + + def is_model_capability_field(name: str) -> bool: return name in PROVIDER_SPECIFIC_CAPABILITY_FIELDS or ( name.startswith("supports_") and name in GetModelCostMap.loaded_model_cost_map_fields() diff --git a/tests/integration/providers/test_sagemaker_chat_wire.py b/tests/integration/providers/test_sagemaker_chat_wire.py index fdf0d94f7c7..784e43f85e2 100644 --- a/tests/integration/providers/test_sagemaker_chat_wire.py +++ b/tests/integration/providers/test_sagemaker_chat_wire.py @@ -243,7 +243,7 @@ def test_sagemaker_chat_signs_the_inference_component_header_and_sends_hf_model_ ) assert "x-amzn-sagemaker-inference-component" in signed_headers.removeprefix("SignedHeaders=").split(";") assert JSON_OBJECT.validate_json(request.body) == { - **container_body(wire.url, PROMPT, stream=False, model_id=_INFERENCE_COMPONENT), + **container_body(wire.url, PROMPT, stream=False), "model": SERVED_MODEL, }, request.body diff --git a/tests/integration/providers/test_sagemaker_embedding_wire.py b/tests/integration/providers/test_sagemaker_embedding_wire.py index 74f3c8c8a00..332f473aeaf 100644 --- a/tests/integration/providers/test_sagemaker_embedding_wire.py +++ b/tests/integration/providers/test_sagemaker_embedding_wire.py @@ -668,12 +668,12 @@ def test_an_extra_body_that_is_not_a_populated_object_adds_nothing_to_the_body( assert _body(request) == {"input": [text]}, request.body -def test_a_caller_sent_model_id_overrides_the_deployments_inference_component(proxy: Gateway, wire: Wire) -> None: +def test_a_caller_sent_model_id_cannot_override_the_deployments_inference_component(proxy: Gateway, wire: Wire) -> None: text: Final = _text() response: Final = _embed(proxy, "sm-openai", [text], model_id="caller-chosen-component") assert response.status_code == 200, response.text request: Final = _only_request(wire) - assert request.headers[COMPONENT_HEADER] == "caller-chosen-component", dict(request.headers) + assert request.headers[COMPONENT_HEADER] == COMPONENT, dict(request.headers) assert _body(request) == {"input": [text], "model": SERVED_MODEL}, request.body diff --git a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py index f8caa8d5fed..356528d8909 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py @@ -343,6 +343,26 @@ def test_body_carries_no_aws_credentials( assert client.request_headers["X-Amz-Security-Token"] == "test-session-token" +@pytest.mark.parametrize("model", ["sagemaker_chat/my-endpoint", "sagemaker_nova/my-endpoint"]) +def test_inference_component_travels_only_in_the_header(monkeypatch: pytest.MonkeyPatch, model: str): + client = _invoke_sagemaker_chat(monkeypatch, model=model, model_id="my-inference-component") + + assert client.request_headers[_INFERENCE_COMPONENT_HEADER] == "my-inference-component" + assert "model_id" not in client.request_body + + +def test_inference_component_header_keeps_header_keys_that_are_not_strings(): + headers = SagemakerChatConfig().validate_environment( + headers={7: "other", "x-amzn-sagemaker-inference-component": "client-component"}, + model="my-endpoint", + messages=[], + optional_params={"model_id": "my-inference-component"}, + litellm_params={}, + ) + + assert headers == {7: "other", _INFERENCE_COMPONENT_HEADER: "my-inference-component"} + + def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch): """Without `hf_model_name` the body must keep the model it has today.""" client = _invoke_sagemaker_chat(monkeypatch) diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 0980c9f5c66..5e40c2ebb60 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -372,6 +372,27 @@ def test_completion_sagemaker_non_stream( assert response.choices[0].message.content == "SageMaker reply" +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.usefixtures("fake_provider_credentials") +def test_completion_sagemaker_sends_the_inference_component_only_as_a_header(respx_mock: respx.MockRouter) -> None: + url: Final = "https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/my-endpoint/invocations" + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"generated_text": "SageMaker reply"}) + ) + litellm.completion( + model="sagemaker/my-endpoint", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + model_id="my-inference-component", + aws_region_name="us-west-2", + num_retries=0, + ) + + request: Final = route.calls[0].request + assert request.headers["X-Amzn-SageMaker-Inference-Component"] == "my-inference-component" + assert json.loads(request.content) == {"inputs": "hi", "parameters": {"temperature": 0.2}} + + @pytest.mark.usefixtures("httpx_transport") @pytest.mark.usefixtures("fake_provider_credentials") def test_completion_sagemaker_non_stream_with_aws_params(respx_mock: respx.MockRouter) -> None: diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index df0429316ed..26d9fe2fb3d 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1,5 +1,6 @@ import asyncio import copy +import io import json import logging from typing import Final @@ -1564,3 +1565,84 @@ async def test_a_failing_health_probe_leaves_the_deployment_serving_requests(mon assert "error" in probe assert reply.choices[0].message.content == "ok" + + +@pytest.mark.parametrize( + ("component_params", "probed_component"), + [ + ({"extra_headers": {"X-Amzn-SageMaker-Inference-Component": "header-component"}}, "header-component"), + ( + {"model_id": "own-component", "extra_headers": {"X-Amzn-SageMaker-Inference-Component": "header-component"}}, + "own-component", + ), + ( + {"model_id": "own-component", "extra_headers": {1: "other", "X-Amzn-SageMaker-Inference-Component": "x"}}, + "own-component", + ), + ( + {"extra_headers": {1: "other", "X-Amzn-SageMaker-Inference-Component": "header-component"}}, + "header-component", + ), + ( + { + "configurable_clientside_auth_params": ["model_id"], + "extra_headers": {"X-Amzn-SageMaker-Inference-Component": "header-component"}, + }, + None, + ), + ], + ids=[ + "header_pinned", + "own_model_id_beats_the_header", + "own_model_id_beside_a_non_string_header_key", + "header_pinned_beside_a_non_string_header_key", + "client_configurable_model_id_takes_no_header_pin", + ], +) +@pytest.mark.asyncio +async def test_run_model_health_check_probes_a_legacy_sagemaker_embedding_at_the_deployments_component( + component_params: dict[str, object], probed_component: str | None +) -> None: + embedding_reply: Final = {"object": "list", "data": [{"object": "embedding", "index": 0, "embedding": [0.1]}]} + invoke_endpoint: Final = MagicMock(return_value={"Body": io.BytesIO(json.dumps(embedding_reply).encode())}) + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + deployment: Final = { + "litellm_params": { + "model": "sagemaker/openai/embed-endpoint", + "aws_region_name": "us-east-1", + "aws_access_key_id": "test-key", + "aws_secret_access_key": "test-secret", + **component_params, + }, + "model_info": {"mode": "embedding"}, + } + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + result: Final = await hc_module._run_model_health_check(deployment) + + assert "error" not in result, result + assert invoke_endpoint.call_count == 1 + assert invoke_endpoint.call_args.kwargs.get("InferenceComponentName") == probed_component + + +@pytest.mark.asyncio +async def test_run_model_health_check_reports_why_a_string_configurable_clientside_auth_params_fails() -> None: + invoke_endpoint: Final = MagicMock() + deployment: Final = { + "litellm_params": { + "model": "sagemaker/openai/embed-endpoint", + "model_id": "own-component", + "aws_region_name": "us-east-1", + "aws_access_key_id": "test-key", + "aws_secret_access_key": "test-secret", + }, + "model_info": {"mode": "embedding", "health_check_params": {"configurable_clientside_auth_params": "model_id"}}, + } + + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + result: Final = await hc_module._run_model_health_check(deployment) + + assert "configurable_clientside_auth_params" in result["error"] + invoke_endpoint.assert_not_called() diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a0a7dd78193..74959155380 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -4,6 +4,7 @@ import copy import functools import gc import itertools +import io import json import linecache import logging @@ -31523,6 +31524,224 @@ def test_router_keeps_supported_fallback_shapes( assert router.fallbacks == expected_fallbacks +_SAGEMAKER_ROUTER_CREDENTIALS: Final = MappingProxyType( + {"aws_region_name": "us-east-1", "aws_access_key_id": "test-key", "aws_secret_access_key": "test-secret"} +) + +_PINNED_COMPONENT: Final = MappingProxyType({"model_id": "deployment-component"}) +_HEADER_PINNED_COMPONENT: Final = MappingProxyType( + {"extra_headers": {"x-amzn-sagemaker-inference-component": "deployment-component"}} +) + +_INFERENCE_COMPONENT_PRECEDENCE: Final = pytest.mark.parametrize( + ("deployment_params", "sent_component"), + [ + (_PINNED_COMPONENT, "deployment-component"), + (_HEADER_PINNED_COMPONENT, "deployment-component"), + (MappingProxyType({}), "client-component"), + ( + MappingProxyType({**_PINNED_COMPONENT, "configurable_clientside_auth_params": ["model_id"]}), + "client-component", + ), + ], + ids=[ + "deployment_pins_the_component", + "deployment_pins_the_component_with_a_header", + "deployment_leaves_the_component_to_the_client", + "deployment_lets_the_client_override_its_component", + ], +) + + +def _sagemaker_router( + model: str, + deployment_params: Mapping[str, object], + mode: str, + router_defaults: Mapping[str, str] = MappingProxyType({}), + model_name: str = "sagemaker-model", +) -> Router: + return Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": model, **_SAGEMAKER_ROUTER_CREDENTIALS, **deployment_params}, + "model_info": {"mode": mode}, + } + ], + default_litellm_params=dict(router_defaults), + ) + + +_CHAT_COMPLETION_REPLY: Final = MappingProxyType( + { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "served-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } +) + + +def _sagemaker_chat_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post( + "https://runtime.sagemaker.us-east-1.amazonaws.com/endpoints/chat-endpoint/invocations" + ).respond(json=dict(_CHAT_COMPLETION_REPLY)) + + +@_INFERENCE_COMPONENT_PRECEDENCE +@pytest.mark.asyncio +async def test_a_client_model_id_cannot_move_a_sagemaker_chat_call_off_the_deployments_component( + respx_mock: respx.MockRouter, + _router_httpx_transport: None, + deployment_params: Mapping[str, object], + sent_component: str, +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router("sagemaker_chat/chat-endpoint", deployment_params, "chat") + + await router.acompletion( + model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], model_id="client-component" + ) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == sent_component + + +@_INFERENCE_COMPONENT_PRECEDENCE +def test_a_client_model_id_cannot_move_a_sync_sagemaker_text_completion_off_the_deployments_component( + respx_mock: respx.MockRouter, deployment_params: Mapping[str, object], sent_component: str +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router( + "sagemaker_chat/chat-endpoint", deployment_params, "completion", model_name="sagemaker_chat/chat-endpoint" + ) + + router.text_completion(model="sagemaker_chat/chat-endpoint", prompt="hi", model_id="client-component") + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == sent_component + + +@pytest.mark.parametrize( + "client_header", + [ + MappingProxyType({"extra_headers": {"x-amzn-sagemaker-inference-component": "client-component"}}), + MappingProxyType({"headers": {"x-amzn-sagemaker-inference-component": "client-component"}}), + MappingProxyType( + { + "provider_specific_header": { + "custom_llm_provider": "sagemaker_chat", + "extra_headers": {"x-amzn-sagemaker-inference-component": "client-component"}, + } + } + ), + ], + ids=["extra_headers", "headers", "provider_specific_header"], +) +@pytest.mark.parametrize( + "deployment_pin", [_PINNED_COMPONENT, _HEADER_PINNED_COMPONENT], ids=["model_id_pin", "header_pin"] +) +@pytest.mark.asyncio +async def test_a_client_inference_component_header_cannot_move_a_pinned_sagemaker_chat_call( + respx_mock: respx.MockRouter, + _router_httpx_transport: None, + client_header: Mapping[str, object], + deployment_pin: Mapping[str, object], +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router("sagemaker_chat/chat-endpoint", deployment_pin, "chat") + + await router.acompletion(model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], **client_header) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == "deployment-component" + + +@pytest.mark.parametrize( + "request_component", + [MappingProxyType({}), MappingProxyType({"model_id": "deployment-component"})], + ids=["request_without_model_id", "request_with_the_pinned_model_id"], +) +@pytest.mark.asyncio +async def test_a_router_default_model_id_cannot_move_a_sagemaker_call_off_the_deployments_component( + respx_mock: respx.MockRouter, _router_httpx_transport: None, request_component: Mapping[str, str] +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router( + "sagemaker_chat/chat-endpoint", + _PINNED_COMPONENT, + "chat", + router_defaults=MappingProxyType({"model_id": "default-component"}), + ) + + await router.acompletion(model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], **request_component) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == "deployment-component" + + +@_INFERENCE_COMPONENT_PRECEDENCE +@pytest.mark.asyncio +async def test_a_client_model_id_cannot_move_a_sagemaker_embedding_call_off_the_deployments_component( + deployment_params: Mapping[str, object], sent_component: str +) -> None: + component_embeddings: Final = MappingProxyType({"deployment-component": [0.1], "client-component": [0.2]}) + + def invoke_endpoint(**request: str | bytes) -> Mapping[str, io.BytesIO]: + served: Final = component_embeddings[str(request["InferenceComponentName"])] + return { + "Body": io.BytesIO( + json.dumps({"object": "list", "data": [{"object": "embedding", "index": 0, "embedding": served}]}).encode() + ) + } + + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + router: Final = _sagemaker_router("sagemaker/openai/embed-endpoint", deployment_params, "embedding") + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + response: Final = await router.aembedding(model="sagemaker-model", input=["hello"], model_id="client-component") + + assert response.data[0]["embedding"] == component_embeddings[sent_component] + + +@pytest.mark.asyncio +async def test_a_header_pin_holds_when_the_deployment_names_sagemaker_through_custom_llm_provider( + respx_mock: respx.MockRouter, _router_httpx_transport: None +) -> None: + route: Final = _sagemaker_chat_route(respx_mock) + router: Final = _sagemaker_router( + "chat-endpoint", MappingProxyType({**_HEADER_PINNED_COMPONENT, "custom_llm_provider": "sagemaker_chat"}), "chat" + ) + + await router.acompletion( + model="sagemaker-model", messages=[{"role": "user", "content": "hi"}], model_id="client-component" + ) + + assert route.calls.last.request.headers["X-Amzn-SageMaker-Inference-Component"] == "deployment-component" + + +@pytest.mark.asyncio +async def test_a_component_header_on_a_non_sagemaker_deployment_leaves_the_clients_model_id_alone( + respx_mock: respx.MockRouter, _router_httpx_transport: None +) -> None: + route: Final = respx_mock.post("https://gateway.local/v1/chat/completions").respond(json=dict(_CHAT_COMPLETION_REPLY)) + router: Final = Router( + model_list=[ + { + "model_name": "gateway-model", + "litellm_params": { + "model": "openai/served-model", + "api_base": "https://gateway.local/v1", + "api_key": "test-key", + "extra_headers": {"X-Amzn-SageMaker-Inference-Component": "gateway-component"}, + }, + } + ] + ) + + await router.acompletion(model="gateway-model", messages=[{"role": "user", "content": "hi"}], model_id="client-value") + + assert json.loads(route.calls.last.request.content)["model_id"] == "client-value" + + def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): router: Final = Router( From 080e643edc2db94cf34824fe851721c320099bd7 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:19:25 -0700 Subject: [PATCH 34/34] fix(proxy): apply new model rows on every worker as soon as they are written (#49034) * fix(proxy): apply published model rows on every worker before the debounced resync Config sync messages now carry the ids of model rows a create, update, or upsert wrote. Every worker that receives one loads those rows by id right away, including while its debounced full resync is still waiting, so a group added one deployment at a time is complete on every worker as soon as the write returns. Deletes, bulk writes, and messages from older pods carry no ids and keep the full resync path * fix(proxy): load published model rows from the primary under the reconcile lock Read the by-id rows through the writer so a lagging read replica cannot put an old row back, and read them inside MODEL_RECONCILE_LOCK so a concurrent delete or reload stays the last word. Look the ids up in bounded chunks, reload DB credentials only when a row names one this worker lacks, and leave auto-router rows to the debounced full resync. Add an integration test that boots two proxies and needs the peer to fall back to a just-added backup within 4 s. * test(proxy): cover a published model row that names no credential * fix(proxy): keep the unknown model name read-through on the replica and outside the lock Only the by-id load that follows a published write reads the primary inside the reconcile lock. A request for a model name a worker does not know reads through the replica routing before taking the lock again, as on main, so unknown names put no load on the primary and do not wait behind a reload. * fix(proxy): keep access group edits from dropping staying deployments and batch published config ids * fix(proxy): read access group rows from the primary so quick edits are not lost to a lagging replica * fix(proxy): install ordinary rows queued with an auto router and check access group edits before writing A batch of published model ids that held one auto router skipped every row in it, so ordinary rows written at the same moment waited for the full reload. The by-id load now leaves only the auto router rows to that reload. An access group update now reads every deployment from the primary, answers 404 for an unknown group and 400 for an unknown model id or name before it writes anything, then removes the group from the deployments that leave before it adds it to the ones that join, so a failed write leaves the group narrower than asked, never wider. * fix(proxy): write an access group update in one transaction after a fresh read The update now checks model names first, reads every deployment from the primary, and commits every join and leave in one batch right after that read, so a failed write changes nothing and a field another admin set during validation is kept. Other workers hear about the rows only after the commit. * fix(proxy): lock the model rows while an access group update reads and rewrites them * test(proxy): run the model resync tests on the real installer and a real router The by-id resync tests now install rows through ProxyConfig's real installer and credential loader on a real Router, faking only the database, and the config sync test row model is frozen. * fix(proxy): lock only the rows an access group update touches and answer 404 before 400 An access group update locked every model row, so a lock on any unrelated row stalled it for 30 s and then failed it with a 500, and unrelated model edits on other proxies waited behind it. It now takes a per-group advisory lock, locks only the rows that carry the group or that it selects, and reads them again once locked. The group existence check runs before the model name check again, so a missing group answers 404 as on main. * fix(proxy): never wait on a locked row in an access group update The update now takes its row locks with NOWAIT. When another edit holds one of the group's rows, it rolls back, holding nothing, and tries again for about 10 s before answering 409. Holding rows while waiting deadlocked against team deletes and let a concurrent model edit undo the update's change. The per-group lock also moves to the two-key advisory space, apart from team ids. * fix(proxy): retry access group row locks for 30 s with SKIP LOCKED and lock only the rows model_ids select * fix(proxy): take the access group lock with pg_try_advisory_xact_lock and retry while another edit holds it --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 2 +- .../proxy/common_utils/config_sync_pubsub.py | 136 +++- .../common_utils/registry_read_through.py | 126 ++- ...model_access_group_management_endpoints.py | 315 ++++++-- .../model_management_endpoints.py | 6 +- litellm/proxy/proxy_server.py | 2 + ...backup_deployment_serves_peer_fallbacks.py | 109 +++ .../common_utils/test_config_sync_pubsub.py | 482 +++++++++--- .../test_registry_read_through.py | 347 +++++++++ .../test_access_group_management.py | 735 +++++++++++++++++- .../test_model_management_endpoints.py | 6 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +- 12 files changed, 2049 insertions(+), 222 deletions(-) create mode 100644 tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index bc64a4c7ef0..c76b9be7c33 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -1648,7 +1648,7 @@ }, "/access_group/{access_group}/update": { "put": { - "description": "Update an access group's model names.\n\nThis will:\n1. Remove the access group from all current deployments\n2. Add the access group to all deployments for the new model_names list\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found", + "description": "Update an access group's model names.\n\nThis will:\n1. Add the access group to all deployments for the new model_names list\n2. Remove the access group from every other deployment\n\nExample:\n```bash\ncurl -X PUT 'http://localhost:4000/access_group/production-models/update' \\\n -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model_names\": [\"gpt-4\", \"claude-3-sonnet\"]\n }'\n```\n\nParameters:\n- access_group: str - The access group name (URL path parameter)\n- model_names: List[str] - New list of model groups to include\n\nReturns:\n- NewModelGroupResponse with the updated access group details\n\nRaises:\n- HTTPException 400: If any model names don't exist\n- HTTPException 404: If access group not found\n- HTTPException 409: If another edit holds the group or one of its models for 30 s", "operationId": "update_access_group_access_group__access_group__update_put", "parameters": [ { diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py index da74851b2d8..e9aaa0e81e4 100644 --- a/litellm/proxy/common_utils/config_sync_pubsub.py +++ b/litellm/proxy/common_utils/config_sync_pubsub.py @@ -2,9 +2,12 @@ import asyncio import json import random import time -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import asdict, dataclass -from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # untyped prisma/redis boundary needs cast +from itertools import chain +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast # noqa: TID251 # untyped prisma/redis boundary + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.repositories.prisma_protocols import RowT_co, TableActions @@ -106,24 +109,69 @@ _pubsub_capable_client: Final = pubsub_capable_client @dataclass(frozen=True, slots=True) class _ConfigChangeMessage: object_type: str + model_ids: tuple[str, ...] = () -def _config_change_message_json(object_type: str) -> str: - return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type))) +@dataclass(frozen=True, slots=True) +class _PubSubEnvelope: + data: str | bytes -async def publish_config_change(redis_cache: "RedisCache | None", object_type: str) -> None: +class _WrittenModelRow(BaseModel): + model_config = ConfigDict(from_attributes=True, frozen=True) + + model_id: str + + +_CONFIG_CHANGE_MESSAGE: Final = TypeAdapter(_ConfigChangeMessage) +_PUBSUB_ENVELOPE: Final = TypeAdapter(_PubSubEnvelope) +_MODEL_TABLE_NAME: Final = "litellm_proxymodeltable" +_TARGETED_WRITE_ACTION_NAMES: Final[frozenset[str]] = frozenset({"create", "update", "upsert"}) + +PublishConfigChange: TypeAlias = Callable[[str, tuple[str, ...]], Awaitable[None]] + + +def _config_change_message_json(object_type: str, model_ids: tuple[str, ...]) -> str: + return json.dumps(asdict(_ConfigChangeMessage(object_type=object_type, model_ids=model_ids))) + + +def parse_config_change(message: object) -> _ConfigChangeMessage | None: + try: + payload: Final = ( + message if isinstance(message, (str, bytes)) else _PUBSUB_ENVELOPE.validate_python(message).data + ) + return _CONFIG_CHANGE_MESSAGE.validate_json(payload) + except ValidationError: + return None + + +def written_model_ids(object_type: str, action_name: str, result: object) -> tuple[str, ...]: + if object_type != _MODEL_TABLE_NAME or action_name not in _TARGETED_WRITE_ACTION_NAMES: + return () + try: + return (_WrittenModelRow.model_validate(result).model_id,) + except ValidationError: + return () + + +async def publish_config_change( + redis_cache: "RedisCache | None", object_type: str, model_ids: tuple[str, ...] = () +) -> None: if redis_cache is None: return try: client: Final = pubsub_capable_client(redis_cache) - await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type)) + await client.publish(config_sync_channel(redis_cache), _config_change_message_json(object_type, model_ids)) except Exception as e: # noqa: BLE001 # best-effort publish; writes must never fail on redis errors verbose_proxy_logger.warning("config sync publish for %s failed: %s", object_type, e) -async def publish_config_change_for_object_type(object_type: str) -> None: - await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type) +async def publish_config_change_for_object_type(object_type: str, model_ids: tuple[str, ...] = ()) -> None: + await publish_config_change(redis_cache=coordination_redis_cache(), object_type=object_type, model_ids=model_ids) + + +async def publish_model_rows_written(model_ids: tuple[str, ...]) -> None: + await publish_config_change_for_object_type(_MODEL_TABLE_NAME, model_ids) async def publish_config_param_change(param_name: str) -> None: @@ -139,7 +187,7 @@ async def publish_config_param_change(param_name: str) -> None: class _PublishOnWriteActions: __slots__ = ("_actions", "_object_type", "_publish") - def __init__(self, actions: object, object_type: str, publish: Callable[[str], Awaitable[None]]) -> None: + def __init__(self, actions: object, object_type: str, publish: PublishConfigChange) -> None: self._actions = actions self._object_type = object_type self._publish = publish @@ -157,7 +205,7 @@ class _PublishOnWriteActions: **kwargs: object, # kwargs-ok: transparent passthrough to untyped prisma action ) -> object: result: Final = await write_action(*args, **kwargs) - await publish(object_type) + await publish(object_type, written_model_ids(object_type, name, result)) return result return _write_then_publish @@ -166,7 +214,7 @@ class _PublishOnWriteActions: def wrap_table_actions_for_config_sync( actions: "TableActions[RowT_co]", table_name: str, - publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type, + publish: PublishConfigChange = publish_config_change_for_object_type, ) -> "TableActions[RowT_co]": if table_name not in _CONFIG_SYNCED_TABLE_NAMES: return actions @@ -174,8 +222,14 @@ def wrap_table_actions_for_config_sync( return cast("TableActions[RowT_co]", wrapped) # cast-ok: dynamic write-through proxy keeps the wrapped row type +def _published_model_ids(messages: tuple[object, ...]) -> tuple[str, ...]: + changes: Final = (parse_config_change(message) for message in messages) + return tuple(dict.fromkeys(chain.from_iterable(change.model_ids for change in changes if change is not None))) + + class ConfigSyncSubscriber: __slots__ = ( + "_apply_model_ids", "_backoff_initial_seconds", "_backoff_max_seconds", "_debounce_seconds", @@ -195,6 +249,7 @@ class ConfigSyncSubscriber: self, redis_cache: "RedisCache", resync_callbacks: tuple[Callable[[], Awaitable[None]], ...], + apply_model_ids: Callable[[tuple[str, ...]], Awaitable[None]] | None = None, debounce_seconds: float = CONFIG_SYNC_DEBOUNCE_SECONDS, jitter_max_seconds: float = CONFIG_SYNC_JITTER_MAX_SECONDS, min_resync_interval_seconds: float = CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS, @@ -206,6 +261,7 @@ class ConfigSyncSubscriber: ) -> None: self._redis_cache = redis_cache self._resync_callbacks = resync_callbacks + self._apply_model_ids = apply_model_ids self._debounce_seconds = debounce_seconds self._jitter_max_seconds = jitter_max_seconds self._min_resync_interval_seconds = min_resync_interval_seconds @@ -266,32 +322,64 @@ class ConfigSyncSubscriber: message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=_POLL_TIMEOUT_SECONDS) if message is None: continue + await self._apply(pubsub, message) await self._debounce_and_resync(pubsub) - async def _debounce_and_resync(self, pubsub: _ConfigSyncPubSub) -> None: - await self._sleep(self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds)) - await self._wait_for_min_resync_interval() + async def _debounce_and_resync(self, pubsub: ConfigSyncPubSub) -> None: + await self._apply_until( + pubsub, self._monotonic() + self._debounce_seconds + self._rng.uniform(0.0, self._jitter_max_seconds) + ) + await self._apply_until(pubsub, self._next_resync_at()) await self._drain_pending(pubsub) await self._run_resync_callbacks() self._last_resync_at = self._monotonic() - async def _wait_for_min_resync_interval(self) -> None: + def _next_resync_at(self) -> float: if self._last_resync_at is None: - return - seconds_until_next_resync = self._min_resync_interval_seconds - (self._monotonic() - self._last_resync_at) - if seconds_until_next_resync <= 0: - return - verbose_proxy_logger.debug( - "config sync resync throttled for %.1fs to cap fleet-wide reload rate", - seconds_until_next_resync, - ) - await self._sleep(seconds_until_next_resync) + return self._monotonic() + next_resync_at: Final = self._last_resync_at + self._min_resync_interval_seconds + throttled_for: Final = next_resync_at - self._monotonic() + if throttled_for > 0: + verbose_proxy_logger.debug( + "config sync resync throttled for %.1fs to cap fleet-wide reload rate", + throttled_for, + ) + return next_resync_at + + async def _apply_until(self, pubsub: ConfigSyncPubSub, deadline: float) -> None: + while True: + remaining = deadline - self._monotonic() + if remaining <= 0: + return + message = await pubsub.get_message( + ignore_subscribe_messages=True, timeout=min(remaining, _POLL_TIMEOUT_SECONDS) + ) + if message is not None: + await self._apply(pubsub, message) @staticmethod async def _drain_pending(pubsub: ConfigSyncPubSub) -> None: while await pubsub.get_message(ignore_subscribe_messages=True, timeout=0) is not None: pass + @staticmethod + async def _ready_messages(pubsub: ConfigSyncPubSub) -> AsyncIterator[object]: + while (message := await pubsub.get_message(ignore_subscribe_messages=True, timeout=0)) is not None: + yield message + + async def _apply(self, pubsub: ConfigSyncPubSub, message: object) -> None: + apply: Final = self._apply_model_ids + if apply is None: + return + ready: Final = (message, *[ready_message async for ready_message in self._ready_messages(pubsub)]) + model_ids: Final = _published_model_ids(ready) + if not model_ids: + return + try: + await apply(model_ids) + except Exception as e: # noqa: BLE001 # the debounced full resync retries what a targeted apply could not + verbose_proxy_logger.warning("config sync targeted apply of %s failed: %s", model_ids, e) + async def _run_resync_callbacks(self) -> None: for callback in self._resync_callbacks: try: diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py index 22df11429c0..487c6feb68d 100644 --- a/litellm/proxy/common_utils/registry_read_through.py +++ b/litellm/proxy/common_utils/registry_read_through.py @@ -15,12 +15,13 @@ or the first routed request on this replica would pick a tier it has not loaded. import asyncio import time from collections.abc import Awaitable, Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper if TYPE_CHECKING: @@ -33,8 +34,14 @@ if TYPE_CHECKING: ) from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.repositories.prisma_protocols import TableActions from litellm.types.agents import AgentResponse + _ModelTable: TypeAlias = TableActions[prisma_models.LiteLLM_ProxyModelTable] + _ModelRows: TypeAlias = Sequence[prisma_models.LiteLLM_ProxyModelTable] + +ModelLoadOutcome: TypeAlias = Literal["loaded", "missing", "no_router", "auto_router"] + READ_THROUGH_MISS_TTL_SECONDS: Final = 2.0 READ_THROUGH_RESYNC_WINDOW_SECONDS: Final = 5.0 READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20 @@ -123,47 +130,114 @@ async def _resync_model_deployments(model_name: str) -> bool: return False prisma_client: Final = proxy_server.prisma_client assert prisma_client is not None - table: Final = ModelRepository(prisma_client).table - name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name} - id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name} - rows: Final = await table.find_many(where=name_filter) or await table.find_many(where=id_filter) + rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table) if not rows: return False - router: Final = proxy_server.llm_router - if router is None: - await proxy_server.proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj - ) - return proxy_server.llm_router is not None - if _holds_an_auto_router(rows): - await proxy_server.proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj - ) - return _model_is_loaded(model_name) async with proxy_server.MODEL_RECONCILE_LOCK: + outcome: Final = await _install_model_rows(rows, _always) + if outcome in ("no_router", "auto_router"): + await _reload_every_model() + return outcome == "loaded" or _model_is_loaded(model_name) + + +async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient + from litellm.repositories.model_repository import ModelRepository + + if not _db_backed_registries_enabled("models"): + return + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table + async with proxy_server.MODEL_RECONCILE_LOCK: + rows: Final = await _find_rows_by_ids(model_ids, writer_table) + _ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential) + + +async def _find_rows_by_name_or_id(model_name: str, table: "_ModelTable") -> "_ModelRows": + name_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_name": model_name} + id_filter: Final[LiteLLM_ProxyModelTableWhereInput] = {"model_id": model_name} + return await table.find_many(where=name_filter) or await table.find_many(where=id_filter) + + +async def _find_rows_by_ids(model_ids: tuple[str, ...], table: "_ModelTable") -> "_ModelRows": + from litellm.repositories.chunked_in import find_many_in + + return list(await find_many_in(table, "model_id", model_ids)) + + +async def _install_model_rows( + rows: "_ModelRows", + needs_credentials: Callable[["_ModelRows"], bool], +) -> ModelLoadOutcome: + from litellm.proxy import proxy_server + + router: Final = proxy_server.llm_router + if not rows: + return "missing" + if router is None: + return "no_router" + if _holds_an_auto_router(rows): + return "auto_router" + if needs_credentials(rows): + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client) - proxy_server.proxy_config._add_deployment(db_models=rows) - proxy_server.llm_model_list = router.registry().deployments() + proxy_server.proxy_config._add_deployment(db_models=rows) + proxy_server.llm_model_list = router.registry().deployments() + return "loaded" + + +async def _reload_every_model() -> None: + from litellm.proxy import proxy_server + + prisma_client: Final = proxy_server.prisma_client + assert prisma_client is not None + await proxy_server.proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj + ) + + +def _always(rows: "_ModelRows") -> bool: return True -def _holds_an_auto_router(rows: Sequence["prisma_models.LiteLLM_ProxyModelTable"]) -> bool: +def _holds_an_auto_router(rows: "_ModelRows") -> bool: return any(_names_an_auto_router(row.litellm_params) for row in rows) +def _without_auto_routers(rows: "_ModelRows") -> "_ModelRows": + return list(row for row in rows if not _names_an_auto_router(row.litellm_params)) + + +def _names_an_unloaded_credential(rows: "_ModelRows") -> bool: + return any(_names_an_unloaded_credential_param(row.litellm_params) for row in rows) + + _STORED_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object]) -def _names_an_auto_router(stored_litellm_params: object) -> bool: +def _decrypted_param(stored_litellm_params: object, key: str) -> str | None: try: params: Final = _STORED_LITELLM_PARAMS.validate_python(stored_litellm_params) except ValidationError: - return False - stored_model: Final = params.get("model") - if not isinstance(stored_model, str): - return False - model: Final = decrypt_value_helper(value=stored_model, key="model", return_original_value=True) - return isinstance(model, str) and model.startswith("auto_router/") + return None + stored_value: Final = params.get(key) + if not isinstance(stored_value, str): + return None + value: Final = decrypt_value_helper(value=stored_value, key=key, return_original_value=True) + return value if isinstance(value, str) else None + + +def _names_an_auto_router(stored_litellm_params: object) -> bool: + model: Final = _decrypted_param(stored_litellm_params, "model") + return model is not None and model.startswith("auto_router/") + + +def _names_an_unloaded_credential_param(stored_litellm_params: object) -> bool: + credential_name: Final = _decrypted_param(stored_litellm_params, "litellm_credential_name") + return credential_name is not None and CredentialAccessor.find_credential(credential_name) is None async def _resync_guardrails(guardrail_name: str) -> bool: diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py index 2d88882a2ac..3a2de12d108 100644 --- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py @@ -12,23 +12,30 @@ Endpoints here: - DELETE /access_group/{access_group}/budget - Clear an access group's shared budget """ +import asyncio +import contextlib +import itertools import json -from collections.abc import Mapping, Sequence -from datetime import datetime +import time +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from datetime import datetime, timedelta from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Protocol from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.config_sync_pubsub import publish_model_rows_written from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, model_access_group_cache_key, model_access_group_registry_cache_key, ) +from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper from litellm.proxy.management_endpoints.common_utils import validate_budget_duration # Clear cache and reload models to pick up the access group changes @@ -116,6 +123,28 @@ class _ModelTableClient(Protocol): async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... +class _ModelTableBatchActions(Protocol): + def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ... + + +class _ModelTableBatch(Protocol): + @property + def litellm_proxymodeltable(self) -> _ModelTableBatchActions: ... + + +class _ModelTableTxActions(Protocol): + async def find_many(self) -> Sequence[_DeploymentRow]: ... + + +class _ModelTableTx(Protocol): + @property + def litellm_proxymodeltable(self) -> _ModelTableTxActions: ... + + async def query_raw(self, query: str, *args: object) -> object: ... + + def batch_(self) -> contextlib.AbstractAsyncContextManager[_ModelTableBatch]: ... + + class _BudgetRow(Protocol): @property def budget_id(self) -> str: ... @@ -168,7 +197,42 @@ class _ModelAccessGroupBudgetTableClient(Protocol): def _model_table(prisma_client: PrismaClient) -> _ModelTableClient: - return ModelRepository(prisma_client).table + return ModelRepository(WriterPinnedClient(prisma_client.db)).table + + +_TRY_LOCK_ACCESS_GROUP: Final = ( + "SELECT pg_try_advisory_xact_lock(hashtext('model_access_group'), hashtext($1)) AS locked" +) +_LOCK_MODEL_ROWS: Final = ( + 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = ANY($1::text[]) ' + "ORDER BY model_id FOR UPDATE SKIP LOCKED" +) +_MEMBERSHIP_TX_TIMEOUT: Final = timedelta(seconds=30) +_LOCK_RETRY_BUDGET_SECONDS: Final = _MEMBERSHIP_TX_TIMEOUT.total_seconds() +_LOCK_RETRY_BACKOFF_SECONDS: Final = (0.05, 0.1, 0.2, 0.4, 0.8) +_LOCK_RETRY_LONGEST_DELAY_SECONDS: Final = 1.0 + + +class _LockedModelRow(BaseModel): + model_config = ConfigDict(frozen=True) + + model_id: str + + +_LOCKED_MODEL_ROWS: Final = TypeAdapter(tuple[_LockedModelRow, ...]) + + +class _AccessGroupLock(BaseModel): + model_config = ConfigDict(frozen=True) + + locked: bool + + +_ACCESS_GROUP_LOCK: Final = TypeAdapter(tuple[_AccessGroupLock]) + + +def _model_table_tx(prisma_client: PrismaClient) -> contextlib.AbstractAsyncContextManager[_ModelTableTx]: + return writer_wrapper(prisma_client.db).tx(timeout=_MEMBERSHIP_TX_TIMEOUT) def _model_access_group_budget_table(prisma_client: PrismaClient) -> _ModelAccessGroupBudgetTableClient: @@ -379,11 +443,10 @@ async def _tag_deployment_with_access_group( """Write `access_group` into one deployment's model_info; returns the (model_id, updated model_info) pair when a write happened, None when the deployment already carried the group.""" - updated_model_info, was_modified = add_access_group_to_deployment( - model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), - access_group=access_group, + updated_model_info: Final = _model_info_with_access_group( + model_id=model_id, model_info=model_info, access_group=access_group ) - if not was_modified: + if updated_model_info is None: return None await _model_table(prisma_client).update( where={"model_id": model_id}, @@ -393,6 +456,141 @@ async def _tag_deployment_with_access_group( return (model_id, updated_model_info) +def _model_info_with_access_group(model_id: str, model_info: object, access_group: str) -> Mapping[str, object] | None: + updated_model_info, was_modified = add_access_group_to_deployment( + model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), + access_group=access_group, + ) + return updated_model_info if was_modified else None + + +def _model_info_without_access_group( + model_id: str, model_info: object, access_group: str +) -> Mapping[str, object] | None: + updated_model_info, was_modified = remove_access_group_from_deployment( + model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), + access_group=access_group, + ) + return updated_model_info if was_modified else None + + +def _membership_rewrite(deployment: _DeploymentRow, selected: bool, access_group: str) -> Mapping[str, object] | None: + if selected: + return _model_info_with_access_group( + model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group + ) + return _model_info_without_access_group( + model_id=deployment.model_id, model_info=deployment.model_info, access_group=access_group + ) + + +def _membership_rewrites( + deployments: Sequence[_DeploymentRow], selected_model_ids: frozenset[str], access_group: str +) -> tuple[tuple[str, Mapping[str, object]], ...]: + rewrites: Final = ( + ( + deployment.model_id, + _membership_rewrite( + deployment=deployment, + selected=deployment.model_id in selected_model_ids, + access_group=access_group, + ), + ) + for deployment in deployments + ) + return tuple((model_id, model_info) for model_id, model_info in rewrites if model_info is not None) + + +def _carries_access_group(model_info: object, access_group: str) -> bool: + access_groups: Final = (model_info_as_mapping(model_info) or {}).get("access_groups") + return isinstance(access_groups, list) and access_group in access_groups + + +def _model_ids_to_lock( + deployments: Sequence[_DeploymentRow], access_group: str, data: UpdateModelGroupRequest +) -> frozenset[str]: + selected_model_ids: Final = frozenset(data.model_ids or ()) + selected_model_names: Final = frozenset(() if data.model_ids else data.model_names or ()) + return frozenset( + deployment.model_id + for deployment in deployments + if deployment.model_id in selected_model_ids + or deployment.model_name in selected_model_names + or _carries_access_group(deployment.model_info, access_group) + ) + + +async def _raise_404_unless_the_primary_carries(access_group: str, prisma_client: PrismaClient) -> None: + if access_group not in _access_groups_of_deployments(await _model_table(prisma_client).find_many()): + raise _http_error(404, f"Access group '{access_group}' not found") + + +async def _rewrite_membership_unless_locked( + prisma_client: PrismaClient, access_group: str, data: UpdateModelGroupRequest +) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]] | None: + async with _model_table_tx(prisma_client) as tx: + (group_lock,) = _ACCESS_GROUP_LOCK.validate_python(await tx.query_raw(_TRY_LOCK_ACCESS_GROUP, access_group)) + if not group_lock.locked: + return None + model_ids_to_lock: Final = _model_ids_to_lock( + deployments=await tx.litellm_proxymodeltable.find_many(), access_group=access_group, data=data + ) + locked_rows: Final = _LOCKED_MODEL_ROWS.validate_python( + await tx.query_raw(_LOCK_MODEL_ROWS, sorted(model_ids_to_lock)) + ) + if frozenset(row.model_id for row in locked_rows) != model_ids_to_lock: + return None + deployments: Final = tuple( + deployment + for deployment in await tx.litellm_proxymodeltable.find_many() + if deployment.model_id in model_ids_to_lock + ) + if access_group not in _access_groups_of_deployments(deployments): + raise HTTPException( + status_code=404, + detail={"error": f"Access group '{access_group}' not found"}, + ) + selected_deployments: Final = _selected_deployments_or_400(deployments=deployments, data=data) + rewrites: Final = _membership_rewrites( + deployments=deployments, + selected_model_ids=frozenset(deployment.model_id for deployment in selected_deployments), + access_group=access_group, + ) + if rewrites: + async with tx.batch_() as batch: + for model_id, model_info in rewrites: + batch.litellm_proxymodeltable.update( + where={"model_id": model_id}, data={"model_info": json.dumps(model_info)} + ) + return selected_deployments, rewrites + + +def _lock_retry_delays() -> Iterator[float]: + return itertools.chain(_LOCK_RETRY_BACKOFF_SECONDS, itertools.repeat(_LOCK_RETRY_LONGEST_DELAY_SECONDS)) + + +async def _rewrite_membership_once_unlocked( + prisma_client: PrismaClient, + access_group: str, + data: UpdateModelGroupRequest, + retry_budget_seconds: float = _LOCK_RETRY_BUDGET_SECONDS, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + clock: Callable[[], float] = time.monotonic, +) -> tuple[tuple[_DeploymentRow, ...], tuple[tuple[str, Mapping[str, object]], ...]]: + deadline: Final = clock() + retry_budget_seconds + for delay in _lock_retry_delays(): + if ( + rewritten := await _rewrite_membership_unless_locked( + prisma_client=prisma_client, access_group=access_group, data=data + ) + ) is not None: + return rewritten + if clock() + delay > deadline: + break + await sleep(delay) + raise _http_error(409, f"Another edit holds access group '{access_group}' or one of its models, try again") + + def _readable_model_info_or_raise(model_id: str, model_info: object) -> Mapping[str, object]: """These helpers rewrite the model_info column wholesale, so a present-but-unreadable value must refuse loudly rather than be silently replaced with a fresh object; an @@ -412,11 +610,10 @@ async def _strip_access_group_from_deployment( """Remove `access_group` from one deployment's model_info; returns the (model_id, updated model_info) pair when a write happened, None when the deployment did not carry the group.""" - updated_model_info, was_modified = remove_access_group_from_deployment( - model_info=dict(_readable_model_info_or_raise(model_id=model_id, model_info=model_info)), - access_group=access_group, + updated_model_info: Final = _model_info_without_access_group( + model_id=model_id, model_info=model_info, access_group=access_group ) - if not was_modified: + if updated_model_info is None: return None await _model_table(prisma_client).update( where={"model_id": model_id}, @@ -493,6 +690,35 @@ async def update_specific_deployments_with_access_group( return tuple(pair for pair in tagged if pair is not None) +def _selected_deployments_or_400( + deployments: Sequence[_DeploymentRow], data: UpdateModelGroupRequest +) -> tuple[_DeploymentRow, ...]: + if data.model_ids: + known_model_ids: Final = frozenset(deployment.model_id for deployment in deployments) + unknown_model_id: Final = next( + (model_id for model_id in data.model_ids if model_id not in known_model_ids), None + ) + if unknown_model_id is not None: + raise HTTPException( + status_code=400, + detail={"error": f"Deployment with model_id '{unknown_model_id}' not found in Database."}, + ) + selected_model_ids: Final = frozenset(data.model_ids) + return tuple(deployment for deployment in deployments if deployment.model_id in selected_model_ids) + model_names: Final = data.model_names or () + known_model_names: Final = frozenset(deployment.model_name for deployment in deployments) + unknown_model_name: Final = next((name for name in model_names if name not in known_model_names), None) + if unknown_model_name is not None: + raise HTTPException( + status_code=400, + detail={ + "error": f"Can't find model '{unknown_model_name}' in Database. Access group management is only supported for database models." + }, + ) + selected_model_names: Final = frozenset(model_names) + return tuple(deployment for deployment in deployments if deployment.model_name in selected_model_names) + + async def _find_deployment_or_400(model_id: str, prisma_client: PrismaClient) -> object: deployment: Final = await _model_table(prisma_client).find_unique(where={"model_id": model_id}) if deployment is None: @@ -536,10 +762,10 @@ async def get_all_access_groups_from_db( Returns: Dict[str, AccessGroupInfo]: Dictionary mapping access_group name to info """ - # Get all deployments - deployments: Final = await ModelRepository(prisma_client).table.find_many() + return dict(_access_groups_of_deployments(await ModelRepository(prisma_client).table.find_many())) - # Build access group map + +def _access_groups_of_deployments(deployments: Sequence[_DeploymentRow]) -> Mapping[str, AccessGroupInfo]: model_names_by_group: Final[dict[str, list[str]]] = {} for deployment in deployments: @@ -835,8 +1061,8 @@ async def update_access_group( Update an access group's model names. This will: - 1. Remove the access group from all current deployments - 2. Add the access group to all deployments for the new model_names list + 1. Add the access group to all deployments for the new model_names list + 2. Remove the access group from every other deployment Example: ```bash @@ -858,6 +1084,7 @@ async def update_access_group( Raises: - HTTPException 400: If any model names don't exist - HTTPException 404: If access group not found + - HTTPException 409: If another edit holds the group or one of its models for 30 s """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -879,26 +1106,14 @@ async def update_access_group( detail={"error": "Either model_names or model_ids must be provided and non-empty"}, ) - use_model_ids: Final = has_model_ids - - # Validation: Check if access group exists try: - access_groups_map: Final = await get_all_access_groups_from_db(prisma_client=prisma_client) - if access_group not in access_groups_map: - raise HTTPException( - status_code=404, - detail={"error": f"Access group '{access_group}' not found"}, - ) + await _raise_404_unless_the_primary_carries(access_group=access_group, prisma_client=prisma_client) except HTTPException: raise except Exception as e: - raise HTTPException( - status_code=500, - detail={"error": f"Failed to check access group existence: {e}"}, - ) + raise _http_error(500, f"Failed to check access group existence: {e}") - # Validation: Check if all new models exist (only if using model_names path) - if not use_model_ids and has_model_names: + if not has_model_ids and has_model_names: assert data.model_names is not None missing_models: Final = await _missing_models_after_read_through( model_names=data.model_names, @@ -912,43 +1127,19 @@ async def update_access_group( ) try: - # Step 1: Remove access group from ALL DB deployments (skip config models) - all_deployments: Final = await _model_table(prisma_client).find_many() - - stripped: Final = [ - await _strip_access_group_from_deployment( - model_id=deployment.model_id, - model_info=deployment.model_info, - access_group=access_group, - prisma_client=prisma_client, - ) - for deployment in all_deployments - ] - stripped_pairs: Final = tuple(pair for pair in stripped if pair is not None) - - # Step 2: Add access group using the appropriate method - if use_model_ids: - assert data.model_ids is not None - updated_pairs = await update_specific_deployments_with_access_group( - model_ids=data.model_ids, - access_group=access_group, - prisma_client=prisma_client, - ) - else: - assert data.model_names is not None - updated_pairs = await update_deployments_with_access_group( - model_names=data.model_names, - access_group=access_group, - prisma_client=prisma_client, - ) - models_updated: Final = len(updated_pairs) + selected_deployments, rewrites = await _rewrite_membership_once_unlocked( + prisma_client=prisma_client, access_group=access_group, data=data + ) + if rewrites: + await publish_model_rows_written(tuple(model_id for model_id, _ in rewrites)) + models_updated: Final = len(selected_deployments) # Clear cache and reload models to pick up the access group changes live_before_reload: Final = live_model_ids_snapshot() reload_outcome: Final = await clear_cache() _raise_http_if_reload_degraded_serving( before=live_before_reload, - written_models=list({**dict(stripped_pairs), **dict(updated_pairs)}.items()), + written_models=rewrites, access_group=access_group, still_desired=reload_outcome.still_desired, live_after=reload_outcome.live_after, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index cdfe3c90182..74fa8e3b3c1 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -779,7 +779,11 @@ async def _auto_router_capability_slot( limit=limit, ) yield tables.litellm_proxymodeltable - await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") + await publish_config_change( + redis_cache=coordination_redis_cache(), + object_type="litellm_proxymodeltable", + model_ids=() if model_id is None else (model_id,), + ) ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING: Final = "enforce_rpm_tpm_on_model_add" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b0ccf74c63b..3e004f1ced7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -549,6 +549,7 @@ from litellm.proxy.common_utils.periodic_reload_schedule import ( write_reload_interval, ) from litellm.proxy.common_utils.proxy_state import ProxyState +from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob from litellm.proxy.common_utils.responses_stream_errors import ResponsesStreamErrorState from litellm.proxy.common_utils.scheduled_job_stagger import ( @@ -8750,6 +8751,7 @@ class ProxyConfig: subscriber: Final = ConfigSyncSubscriber( redis_cache=redis_cache, resync_callbacks=(_resync_config_from_db,), + apply_model_ids=resync_model_deployments_by_ids, ) self.config_sync_subscriber = subscriber subscriber.start() diff --git a/tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py b/tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py new file mode 100644 index 00000000000..942bed6c165 --- /dev/null +++ b/tests/integration/management/test_added_backup_deployment_serves_peer_fallbacks.py @@ -0,0 +1,109 @@ +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Final + +import httpx +import pytest +from pydantic import TypeAdapter +from redis import Redis + +from tests.integration._support.client import Gateway, eventually, list_value, object_value, string_value +from tests.integration._support.database import scratch_database +from tests.integration._support.process import graceful_stop_seconds, owned_proxy +from tests.integration._support.redis_process import owned_redis + +pytestmark: Final = [pytest.mark.timeout(int(4 * graceful_stop_seconds() + 120))] + +COORDINATION_CONFIG: Final = Path(__file__).resolve().parents[1] / "coordination_redis_proxy_config.yaml" +CONFIG_SYNC_CHANNEL: Final = "litellm_proxy.config_change" +SUBSCRIBER_COUNTS: Final = TypeAdapter(tuple[tuple[str, int], ...]) +PERIODIC_RELOAD_SECONDS: Final = 3600 +PEER_PICKUP_SECONDS: Final = 4 +PROVIDER_KEY: Final = "integration-provider-key" + + +@contextmanager +def _proxy_pair(gateway: Gateway, directory: Path) -> Iterator[tuple[Gateway, Gateway]]: + with scratch_database("integration_model_add_sync") as database, owned_redis(directory) as coordination: + shared_stores: Final = { + "DATABASE_URL": database, + "REDIS_HOST": coordination.host, + "REDIS_PORT": str(coordination.port), + "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": str(PERIODIC_RELOAD_SECONDS), + } + with ( + owned_proxy(gateway, directory / "writer", shared_stores, config=COORDINATION_CONFIG) as writer, + owned_proxy(gateway, directory / "peer", shared_stores, config=COORDINATION_CONFIG) as peer, + Redis(host=coordination.host, port=coordination.port, socket_timeout=2) as redis, + ): + eventually( + lambda: SUBSCRIBER_COUNTS.validate_python(redis.pubsub_numsub(CONFIG_SYNC_CHANNEL))[0][1], + lambda count: count >= 2, + seconds=30, + ) + yield writer, peer + + +@contextmanager +def _failing_upstream_model(gateway: Gateway, upstream_model: str) -> Iterator[None]: + script: Final = f"{gateway.upstream_url}/__scripts/{upstream_model}" + httpx.post(script, json={"statuses": [500]}, timeout=5, trust_env=False).raise_for_status() + try: + yield + finally: + httpx.delete(script, timeout=5, trust_env=False).raise_for_status() + + +def _add_deployment(writer: Gateway, group: str, upstream_model: str, order: int) -> None: + writer.post( + "/model/new", + { + "model_name": group, + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_key": PROVIDER_KEY, + "api_base": f"{writer.upstream_url}/v1", + "order": order, + "num_retries": 0, + }, + }, + ) + + +def _listed_models(proxy: Gateway) -> frozenset[str]: + return frozenset(string_value(object_value(entry)["id"]) for entry in list_value(proxy.get("/v1/models")["data"])) + + +def _chat(proxy: Gateway, group: str) -> httpx.Response: + return proxy.request( + "POST", + "/v1/chat/completions", + {"model": group, "messages": [{"role": "user", "content": f"order fallback {uuid.uuid4().hex}"}]}, + ) + + +def test_a_backup_added_on_one_proxy_catches_the_peers_failing_primary_right_away( + gateway: Gateway, tmp_path: Path +) -> None: + group: Final = f"integration-{uuid.uuid4().hex}" + primary: Final = f"integration-primary-{uuid.uuid4().hex}" + backup: Final = f"integration-backup-{uuid.uuid4().hex}" + with _proxy_pair(gateway, tmp_path) as (writer, peer), _failing_upstream_model(gateway, primary): + _add_deployment(writer, group, primary, order=1) + eventually(lambda: _listed_models(peer), lambda listed: group in listed, seconds=30) + + _add_deployment(writer, group, backup, order=2) + on_writer: Final = _chat(writer, group) + on_peer: Final = eventually( + lambda: _chat(peer, group), + lambda response: response.status_code == 200, + seconds=PEER_PICKUP_SECONDS, + return_last_on_timeout=True, + ) + + assert on_writer.status_code == 200, on_writer.text + assert object_value(on_writer.json())["model"] == backup, on_writer.text + assert on_peer.status_code == 200, on_peer.text + assert object_value(on_peer.json())["model"] == backup, on_peer.text diff --git a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py index 92cfb6a71e5..cbb37c964c6 100644 --- a/tests/unit/proxy/common_utils/test_config_sync_pubsub.py +++ b/tests/unit/proxy/common_utils/test_config_sync_pubsub.py @@ -5,6 +5,7 @@ from typing import Callable, Coroutine, Final, Iterable, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pydantic import BaseModel, ConfigDict from redis.asyncio import Redis import litellm @@ -102,8 +103,8 @@ class _ScriptedPubSubClient: class _QueuePubSub: - def __init__(self, initial_messages: Iterable[str] = ()) -> None: - self.queue: "asyncio.Queue[str]" = asyncio.Queue() + def __init__(self, initial_messages: Iterable[object] = ()) -> None: + self.queue: "asyncio.Queue[object]" = asyncio.Queue() for message in initial_messages: self.queue.put_nowait(message) self.subscribed_channels: List[str] = [] @@ -112,7 +113,7 @@ class _QueuePubSub: async def subscribe(self, *channels: str) -> None: self.subscribed_channels.extend(channels) - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: if timeout == 0: try: return self.queue.get_nowait() @@ -128,7 +129,7 @@ class _QueuePubSub: class _BrokenPubSub(_QueuePubSub): - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: raise ConnectionError("connection lost") @@ -138,11 +139,11 @@ class _CloseFailingBrokenPubSub(_BrokenPubSub): class _EmptyPollsThenMessagePubSub(_QueuePubSub): - def __init__(self, empty_polls: int, initial_messages: Iterable[str] = ()) -> None: + def __init__(self, empty_polls: int, initial_messages: Iterable[object] = ()) -> None: super().__init__(initial_messages=initial_messages) self.remaining_empty_polls = empty_polls - async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[str]: + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: if timeout != 0 and self.remaining_empty_polls > 0: self.remaining_empty_polls -= 1 return None @@ -157,6 +158,38 @@ class _FakeClock: return self.now +class _VirtualTimePubSub(_QueuePubSub): + """Timed polls move the injected clock forward instead of waiting on the wall clock. + + A delivery scheduled at a clock time is returned by the first poll whose window + reaches it, with the clock set to that time, so a test asserts when each resync + and targeted apply ran in subscriber time. + """ + + def __init__(self, clock: _FakeClock, deliveries: Iterable[Tuple[float, object]] = ()) -> None: + super().__init__() + self._clock = clock + self.pending: List[Tuple[float, object]] = sorted(deliveries, key=lambda delivery: delivery[0]) + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> Optional[object]: + await asyncio.sleep(0) + due_by = self._clock.now + timeout + if self.pending and self.pending[0][0] <= due_by: + delivered_at, message = self.pending.pop(0) + self._clock.now = max(self._clock.now, delivered_at) + return message + self._clock.now = due_by + return None + + +def _change(*model_ids: str) -> str: + return json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": list(model_ids)}) + + +def _redis_delivery(message: str) -> dict: + return {"type": "message", "pattern": None, "channel": b"litellm_proxy.config_change", "data": message.encode()} + + class _ScriptedPubSubRedisClient(Redis): def __init__(self, pubsubs: Iterable[_QueuePubSub]) -> None: self._scripted_pubsubs = iter(pubsubs) @@ -204,7 +237,16 @@ async def test_publish_sends_object_type_json_on_channel() -> None: assert len(client.published) == 1 channel, message = client.published[0] assert channel == "litellm_proxy.config_change" - assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + assert json.loads(message) == {"object_type": "litellm_proxymodeltable", "model_ids": []} + + +async def test_publish_carries_the_written_model_ids() -> None: + client = _RecordingRedisClient() + cache = _FakeRedisCache(client) + + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-1", "m-2")) + + assert json.loads(client.published[0][1]) == {"object_type": "litellm_proxymodeltable", "model_ids": ["m-1", "m-2"]} async def test_publish_uses_namespaced_channel() -> None: @@ -235,7 +277,7 @@ async def test_publish_reaches_cluster_derived_pubsub_clients() -> None: await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable") assert client.published == [ - (CONFIG_SYNC_CHANNEL, json.dumps({"object_type": "litellm_proxymodeltable"})) + (CONFIG_SYNC_CHANNEL, json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": []})) ] @@ -312,29 +354,210 @@ class _MaxJitterRandom(random.Random): return b -async def test_debounce_sleep_adds_jitter_from_injected_rng() -> None: - pubsub = _QueuePubSub(initial_messages=[json.dumps({"object_type": "litellm_proxymodeltable"})]) - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - sleeps: List[float] = [] - fired = asyncio.Event() +def _virtual_time_subscriber( + pubsub: _VirtualTimePubSub, + clock: _FakeClock, + events: List[str], + resynced: asyncio.Event, + *, + debounce_seconds: float = 0.0, + jitter_max_seconds: float = 0.0, + min_resync_interval_seconds: float = 10.0, + rng: Optional[random.Random] = None, + expected_resyncs: int = 1, + apply_error: Optional[Exception] = None, +) -> ConfigSyncSubscriber: + async def resync() -> None: + events.append(f"resync@{clock.now}") + if sum(event.startswith("resync@") for event in events) >= expected_resyncs: + resynced.set() - async def recording_sleep(seconds: float) -> None: - sleeps.append(seconds) + async def apply(model_ids: Tuple[str, ...]) -> None: + events.append(f"apply:{','.join(model_ids)}@{clock.now}") + if apply_error is not None: + raise apply_error - subscriber = ConfigSyncSubscriber( - redis_cache=cache, - resync_callbacks=(_recording_callback([], "resync", fired),), - debounce_seconds=1.0, - jitter_max_seconds=4.0, - rng=_MaxJitterRandom(), - sleep=recording_sleep, + return ConfigSyncSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])), + resync_callbacks=(resync,), + apply_model_ids=apply, + debounce_seconds=debounce_seconds, + jitter_max_seconds=jitter_max_seconds, + min_resync_interval_seconds=min_resync_interval_seconds, + rng=rng, + monotonic=clock, ) - subscriber.start() - await asyncio.wait_for(fired.wait(), timeout=5) - await subscriber.stop() - assert sleeps == [5.0] +async def _run_until_resynced(subscriber: ConfigSyncSubscriber, resynced: asyncio.Event) -> None: + subscriber.start() + try: + await asyncio.wait_for(resynced.wait(), timeout=5) + finally: + await subscriber.stop() + + +async def test_debounce_waits_out_the_jitter_from_injected_rng() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change())]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber( + pubsub, clock, events, resynced, debounce_seconds=1.0, jitter_max_seconds=4.0, rng=_MaxJitterRandom() + ) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["resync@1005.0"] + + +async def test_published_model_ids_apply_on_receipt_while_the_full_resync_still_debounces() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber( + pubsub, clock, events, resynced, debounce_seconds=1.0, jitter_max_seconds=4.0, rng=_MaxJitterRandom() + ) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1@1000.0", "resync@1005.0"] + + +async def test_model_ids_published_inside_the_throttle_wait_apply_without_waiting() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub( + clock, deliveries=[(1000.0, _change()), (1004.0, _change("m-2")), (1006.0, _change("m-3", "m-4"))] + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["resync@1000.0", "apply:m-2@1004.0", "apply:m-3,m-4@1006.0", "resync@1010.0"] + + +async def test_model_ids_already_waiting_apply_together_in_one_call() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub( + clock, deliveries=[(1000.0, _change("m-1")), (1000.0, _change("m-2", "m-1")), (1000.0, _change("m-3"))] + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1,m-2,m-3@1000.0", "resync@1000.0"] + + +async def test_redis_shaped_message_applies_its_model_ids() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _redis_delivery(_change("m-1")))]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1@1000.0", "resync@1000.0"] + + +async def test_messages_without_usable_model_ids_only_schedule_the_full_resync() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub( + clock, + deliveries=[ + (1000.0, "change"), + (1004.0, json.dumps({"object_type": "litellm_proxymodeltable"})), + (1005.0, json.dumps({"object_type": "litellm_proxymodeltable", "model_ids": "m-1"})), + (1006.0, {"type": "message", "pattern": None, "channel": b"c", "data": 7}), + (1007.0, None), + ], + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["resync@1000.0", "resync@1010.0"] + + +async def test_failing_targeted_apply_still_runs_the_full_resync() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, apply_error=RuntimeError("db down")) + + await _run_until_resynced(subscriber, resynced) + + assert events == ["apply:m-1@1000.0", "resync@1000.0"] + + +async def test_subscriber_without_an_apply_callback_only_resyncs() -> None: + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, _change("m-1"))]) + resyncs: List[str] = [] + fired = asyncio.Event() + subscriber = ConfigSyncSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])), + resync_callbacks=(_recording_callback(resyncs, "resync", fired),), + debounce_seconds=0.0, + jitter_max_seconds=0.0, + monotonic=clock, + ) + + await _run_until_resynced(subscriber, fired) + + assert resyncs == ["resync"] + + +async def test_peer_holds_every_published_row_before_its_next_full_resync() -> None: + publisher = _RecordingRedisClient() + cache = _FakeRedisCache(publisher) + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-order-1",)) + await publish_config_change(redis_cache=cache, object_type="litellm_proxymodeltable", model_ids=("m-order-2",)) + on_the_wire = [_redis_delivery(message) for _, message in publisher.published] + clock = _FakeClock() + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, on_the_wire[0]), (1007.0, on_the_wire[1])]) + db_rows_written_at = {"m-order-1": 1000.0, "m-order-2": 1007.0} + registry: set = set() + snapshots: List[Tuple[str, float, frozenset]] = [] + resynced = asyncio.Event() + + async def apply(model_ids: Tuple[str, ...]) -> None: + registry.update(model_ids) + snapshots.append(("apply", clock.now, frozenset(registry))) + + async def resync() -> None: + registry.update(row for row, written_at in db_rows_written_at.items() if written_at <= clock.now) + snapshots.append(("resync", clock.now, frozenset(registry))) + if sum(kind == "resync" for kind, _, _ in snapshots) == 2: + resynced.set() + + subscriber = ConfigSyncSubscriber( + redis_cache=_FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])), + resync_callbacks=(resync,), + apply_model_ids=apply, + debounce_seconds=1.0, + jitter_max_seconds=5.0, + min_resync_interval_seconds=10.0, + rng=_MaxJitterRandom(), + monotonic=clock, + ) + + await _run_until_resynced(subscriber, resynced) + + assert snapshots == [ + ("apply", 1000.0, frozenset({"m-order-1"})), + ("resync", 1006.0, frozenset({"m-order-1"})), + ("apply", 1007.0, frozenset({"m-order-1", "m-order-2"})), + ("resync", 1016.0, frozenset({"m-order-1", "m-order-2"})), + ] def test_default_jitter_window_is_nonzero() -> None: @@ -345,92 +568,43 @@ def test_default_min_resync_interval_caps_reload_rate() -> None: assert CONFIG_SYNC_MIN_RESYNC_INTERVAL_SECONDS > CONFIG_SYNC_JITTER_MAX_SECONDS -def _throttled_subscriber( - cache: object, - events: List[str], - fired: asyncio.Event, - clock: _FakeClock, - min_resync_interval_seconds: float = 10.0, -) -> ConfigSyncSubscriber: - async def recording_sleep(seconds: float) -> None: - events.append(f"sleep:{seconds}") - await asyncio.sleep(0) - - async def resync() -> None: - events.append("resync") - fired.set() - - return ConfigSyncSubscriber( - redis_cache=cache, - resync_callbacks=(resync,), - debounce_seconds=0.0, - jitter_max_seconds=0.0, - min_resync_interval_seconds=min_resync_interval_seconds, - sleep=recording_sleep, - monotonic=clock, - ) - - async def test_resync_arriving_inside_min_interval_waits_out_the_remainder() -> None: - pubsub = _QueuePubSub() - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - events: List[str] = [] - fired = asyncio.Event() clock = _FakeClock() - subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, "change"), (1004.0, "change")]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) - subscriber.start() - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - fired.clear() - clock.now += 4.0 - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - await subscriber.stop() + await _run_until_resynced(subscriber, resynced) - assert events == ["sleep:0.0", "resync", "sleep:0.0", "sleep:6.0", "resync"] + assert events == ["resync@1000.0", "resync@1010.0"] async def test_resync_after_min_interval_elapsed_is_not_throttled() -> None: - pubsub = _QueuePubSub() - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - events: List[str] = [] - fired = asyncio.Event() clock = _FakeClock() - subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + pubsub = _VirtualTimePubSub(clock, deliveries=[(1000.0, "change"), (1030.0, "change")]) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) - subscriber.start() - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - fired.clear() - clock.now += 30.0 - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - await subscriber.stop() + await _run_until_resynced(subscriber, resynced) - assert events == ["sleep:0.0", "resync", "sleep:0.0", "resync"] + assert events == ["resync@1000.0", "resync@1030.0"] async def test_writes_during_the_throttle_wait_collapse_into_the_next_resync() -> None: - pubsub = _QueuePubSub() - cache = _FakeRedisCache(_ScriptedPubSubRedisClient([pubsub])) - events: List[str] = [] - fired = asyncio.Event() clock = _FakeClock() - subscriber = _throttled_subscriber(cache=cache, events=events, fired=fired, clock=clock) + pubsub = _VirtualTimePubSub( + clock, deliveries=[(1000.0, "change")] + [(1004.0 + offset, "change") for offset in range(5)] + ) + events: List[str] = [] + resynced = asyncio.Event() + subscriber = _virtual_time_subscriber(pubsub, clock, events, resynced, expected_resyncs=2) - subscriber.start() - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - fired.clear() - for _ in range(5): - pubsub.queue.put_nowait("change") - await asyncio.wait_for(fired.wait(), timeout=5) - await asyncio.sleep(0.1) - await subscriber.stop() + await _run_until_resynced(subscriber, resynced) - assert events.count("resync") == 2 - assert pubsub.queue.empty() + assert events == ["resync@1000.0", "resync@1010.0"] + assert pubsub.pending == [] async def test_polls_without_messages_do_not_trigger_resyncs() -> None: @@ -646,13 +820,39 @@ class _AllWritesTableActions: return action -def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str], Coroutine[None, None, None]]: - async def publish(object_type: str) -> None: +def _recording_publish(calls: List[Tuple[str, str]]) -> Callable[[str, Tuple[str, ...]], Coroutine[None, None, None]]: + async def publish(object_type: str, model_ids: Tuple[str, ...]) -> None: calls.append(("publish", object_type)) return publish +def _recording_publish_with_ids( + calls: List[Tuple[str, Tuple[str, ...]]], +) -> Callable[[str, Tuple[str, ...]], Coroutine[None, None, None]]: + async def publish(object_type: str, model_ids: Tuple[str, ...]) -> None: + calls.append((object_type, model_ids)) + + return publish + + +class _ModelRow(BaseModel): + model_config = ConfigDict(frozen=True) + + model_id: str + model_name: str + + +class _ModelRowTableActions: + def __getattr__(self, name: str) -> Callable[..., Coroutine[None, None, object]]: + async def action(*args: object, **kwargs: object) -> object: + if name.endswith("_many"): + return 2 + return _ModelRow(model_id="m-1", model_name="gpt-5.2") + + return action + + def test_wrapper_passes_through_unsynced_tables() -> None: actions = object() @@ -722,6 +922,46 @@ async def test_wrapper_publishes_for_every_write_action(action_name: str) -> Non assert publish_calls == [("publish", "litellm_guardrailstable")] +@pytest.mark.parametrize( + "action_name, expected_ids", + [ + ("create", ("m-1",)), + ("update", ("m-1",)), + ("upsert", ("m-1",)), + ("delete", ()), + ("create_many", ()), + ("update_many", ()), + ("delete_many", ()), + ], +) +async def test_wrapper_publishes_the_written_model_id_for_row_writes( + action_name: str, expected_ids: Tuple[str, ...] +) -> None: + publish_calls: List[Tuple[str, Tuple[str, ...]]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_ModelRowTableActions(), + table_name="litellm_proxymodeltable", + publish=_recording_publish_with_ids(publish_calls), + ) + + await getattr(wrapped, action_name)(data={}) + + assert publish_calls == [("litellm_proxymodeltable", expected_ids)] + + +async def test_wrapper_publishes_no_model_ids_for_other_tables() -> None: + publish_calls: List[Tuple[str, Tuple[str, ...]]] = [] + wrapped = wrap_table_actions_for_config_sync( + actions=_ModelRowTableActions(), + table_name="litellm_guardrailstable", + publish=_recording_publish_with_ids(publish_calls), + ) + + await wrapped.create(data={}) + + assert publish_calls == [("litellm_guardrailstable", ())] + + async def test_model_repository_write_publishes_via_live_coordination_cache() -> None: from litellm.proxy import proxy_server from litellm.proxy.proxy_server import _set_redis_usage_cache @@ -747,7 +987,7 @@ async def test_model_repository_write_publishes_via_live_coordination_cache() -> assert len(client.published) == 1 channel, message = client.published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "litellm_proxymodeltable"} + assert json.loads(message) == {"object_type": "litellm_proxymodeltable", "model_ids": ["m-1"]} async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None: @@ -774,7 +1014,7 @@ async def test_ui_settings_write_publishes_via_live_coordination_cache() -> None assert len(client.published) == 1 channel, message = client.published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": "litellm_uisettings"} + assert json.loads(message) == {"object_type": "litellm_uisettings", "model_ids": []} async def _publish_calls_for_invalidated_param(param_name: str) -> List[Tuple[str, str]]: @@ -799,7 +1039,7 @@ async def test_invalidate_config_param_publishes_params_a_resync_applies(param_n assert len(published) == 1 channel, message = published[0] assert channel == CONFIG_SYNC_CHANNEL - assert json.loads(message) == {"object_type": param_name} + assert json.loads(message) == {"object_type": param_name, "model_ids": []} @pytest.mark.parametrize("param_name", _STARTUP_ONLY_CONFIG_PARAM_NAMES) @@ -932,6 +1172,46 @@ async def test_proxy_config_subscriber_resyncs_deployments_only() -> None: assert subscriber._task is None +async def test_proxy_config_subscriber_applies_published_model_ids_from_the_db( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig + + rows = [MagicMock()] + prisma_client = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=rows) + router = MagicMock() + router.registry.return_value.deployments.return_value = ["deployment"] + installed: List[object] = [] + + def install(db_models: object) -> None: + installed.append(db_models) + + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", None) + monkeypatch.setattr(proxy_server.proxy_config, "get_credentials", AsyncMock()) + monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", install) + cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()])) + config = ProxyConfig() + + config.start_config_sync_subscriber(prisma_client=prisma_client, proxy_logging_obj=MagicMock(), redis_cache=cache) + subscriber = config.config_sync_subscriber + assert subscriber is not None + apply = subscriber._apply_model_ids + assert apply is not None + await apply(("m-1", "m-2")) + await config.stop_config_sync_subscriber() + + prisma_client.db.litellm_proxymodeltable.find_many.assert_awaited_once_with( + where={"model_id": {"in": ["m-1", "m-2"]}} + ) + assert installed == [rows] + assert proxy_server.llm_model_list == ["deployment"] + + async def test_proxy_config_does_not_start_subscriber_without_coordination_redis() -> None: from litellm.proxy.proxy_server import ProxyConfig diff --git a/tests/unit/proxy/common_utils/test_registry_read_through.py b/tests/unit/proxy/common_utils/test_registry_read_through.py index 04729167707..da7b1f497c9 100644 --- a/tests/unit/proxy/common_utils/test_registry_read_through.py +++ b/tests/unit/proxy/common_utils/test_registry_read_through.py @@ -1,4 +1,5 @@ import asyncio +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import pytest @@ -6,7 +7,10 @@ import pytest from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough if TYPE_CHECKING: + from litellm import Router + from litellm.proxy._types import LiteLLM_ProxyModelTable from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + from litellm.types.utils import CredentialItem def nothing_loaded(_key: str) -> bool: @@ -487,6 +491,178 @@ async def test_resync_model_deployments_loads_db_credentials_before_reconciling_ installed.assert_called_once_with(db_models=rows, credential={"api_key": "sk-from-db"}) +@pytest.fixture +def stored_params_salt(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-stored-model-rows") + + +def _stored_model_row(model_id: str, stored_params: Mapping[str, str]) -> "LiteLLM_ProxyModelTable": + from litellm.proxy._types import LiteLLM_ProxyModelTable + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + return LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="gpt-failover-group", + litellm_params={key: encrypt_value_helper(value) for key, value in stored_params.items()}, + model_info={"id": model_id}, + ) + + +def _backup_row(model_id: str, model: str = "openai/gpt-backup") -> "LiteLLM_ProxyModelTable": + return _stored_model_row(model_id, {"model": model, "api_key": "sk-backup"}) + + +def _router_serving(*model_ids: str) -> "Router": + from litellm import Router + + return Router( + model_list=[ + { + "model_name": "gpt-failover-group", + "litellm_params": {"model": "openai/gpt-primary", "api_key": "sk-primary"}, + "model_info": {"id": model_id}, + } + for model_id in model_ids + ] + ) + + +def _served_ids(deployments: "Sequence[Mapping[str, Mapping[str, object]]] | None") -> frozenset[object]: + return frozenset(deployment["model_info"]["id"] for deployment in deployments or ()) + + +class _ProxyModelTable: + def __init__(self, rows: "tuple[LiteLLM_ProxyModelTable, ...]") -> None: + self.rows = rows + self.reads = 0 + + async def find_many( + self, where: Mapping[str, Mapping[str, Sequence[str]]] + ) -> "tuple[LiteLLM_ProxyModelTable, ...]": + self.reads += 1 + return tuple(row for row in self.rows if row.model_id in where["model_id"]["in"]) + + async def create(self, data: Mapping[str, object]) -> None: + raise AssertionError(f"a model load wrote a row: {data}") + + +class _CredentialsTable: + def __init__(self, rows: "tuple[CredentialItem, ...]") -> None: + self.rows = rows + + async def find_many(self) -> "tuple[CredentialItem, ...]": + return self.rows + + +class _Db: + def __init__( + self, litellm_proxymodeltable: _ProxyModelTable, litellm_credentialstable: _CredentialsTable | None = None + ) -> None: + self.litellm_proxymodeltable = litellm_proxymodeltable + self.litellm_credentialstable = litellm_credentialstable or _CredentialsTable(()) + self.other_tables_read: list[str] = [] + + def __getattr__(self, table_name: str) -> object: + self.other_tables_read.append(table_name) + raise AttributeError(table_name) + + +class _PrismaClientWithoutConfigRows: + def __init__(self, db: object) -> None: + self.db = db + + async def get_generic_data(self, key: str, value: str, table_name: str) -> None: + return None + + +def _serve_db_models(monkeypatch: pytest.MonkeyPatch, db: object, router: "Router | None") -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", _PrismaClientWithoutConfigRows(db)) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", None) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_installs_only_the_published_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + router: Final = _router_serving("primary-id") + _serve_db_models( + monkeypatch, _Db(_ProxyModelTable((_backup_row("m-1"), _backup_row("m-2"), _backup_row("m-3")))), router + ) + + await resync_model_deployments_by_ids(("m-1", "m-2")) + + assert _served_ids(router.registry().deployments()) == {"primary-id", "m-1", "m-2"} + assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "m-1", "m-2"} + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_leaves_the_router_alone_for_deleted_rows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + router: Final = _router_serving("primary-id") + _serve_db_models(monkeypatch, _Db(_ProxyModelTable(())), router) + + await resync_model_deployments_by_ids(("deleted-id",)) + + assert _served_ids(router.registry().deployments()) == {"primary-id"} + assert proxy_server.llm_model_list is None + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_without_a_router_leaves_the_reload_to_the_debounced_resync( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + table: Final = _ProxyModelTable((_backup_row("m-1"),)) + db: Final = _Db(table) + _serve_db_models(monkeypatch, db, None) + + await resync_model_deployments_by_ids(("m-1",)) + + assert proxy_server.llm_router is None + assert table.reads == 1 + assert db.other_tables_read == [] + + +@pytest.mark.asyncio +async def test_resync_model_deployments_by_ids_respects_supported_db_objects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock( + side_effect=AssertionError("db hit for an object type this replica does not load") + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["guardrails"]}) + + await resync_model_deployments_by_ids(("gated-out-id",)) + + prisma_client.db.litellm_proxymodeltable.find_many.assert_not_awaited() + + @pytest.mark.asyncio async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch): from unittest.mock import AsyncMock, MagicMock @@ -786,3 +962,174 @@ async def test_resync_model_deployments_reconciles_every_db_row_when_the_missed_ assert await _resync_model_deployments(router_name) is router_holds_the_name_after_reconcile reconciled.assert_called_once_with(prisma_client=prisma_client, proxy_logging_obj=proxy_server.proxy_logging_obj) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_reads_the_writer_not_a_lagging_replica( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from unittest.mock import MagicMock + + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer: Final = MagicMock(name="writer_prisma") + writer.litellm_proxymodeltable = _ProxyModelTable((_backup_row("backup-id"),)) + replica_table: Final = _ProxyModelTable(()) + replica: Final = MagicMock(name="lagging_replica_prisma") + replica.litellm_proxymodeltable = replica_table + router: Final = _router_serving("primary-id") + _serve_db_models( + monkeypatch, + RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=writer, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=replica, iam_token_db_auth=False), + ), + router, + ) + + await resync_model_deployments_by_ids(("backup-id",)) + + assert router.registry().has_deployment("backup-id") + assert replica_table.reads == 0 + + +@pytest.mark.asyncio +async def test_unknown_model_name_read_through_reads_the_replica_without_waiting_for_a_reload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + + writer: Final = MagicMock(name="writer_prisma") + writer.litellm_proxymodeltable = SimpleNamespace(find_many=AsyncMock(return_value=[]), create=AsyncMock()) + replica: Final = MagicMock(name="replica_prisma") + replica.litellm_proxymodeltable = SimpleNamespace(find_many=AsyncMock(return_value=[]), create=AsyncMock()) + prisma_client: Final = MagicMock() + prisma_client.db = RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=writer, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=replica, iam_token_db_auth=False), + ) + reload_in_progress: Final = asyncio.Lock() + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) + monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress) + + async with reload_in_progress: + found: Final = await asyncio.wait_for(_resync_model_deployments("no-such-model"), timeout=2) + + assert found is False + writer.litellm_proxymodeltable.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_installs_the_row_a_concurrent_reload_left_current( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + table: Final = _ProxyModelTable((_backup_row("m-1", model="openai/gpt-before-update"),)) + router: Final = _router_serving("primary-id") + _serve_db_models(monkeypatch, _Db(table), router) + reload_in_progress: Final = asyncio.Lock() + monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress) + + async with reload_in_progress: + targeted_load: Final = asyncio.create_task(resync_model_deployments_by_ids(("m-1",))) + for _ in range(5): + await asyncio.sleep(0) + assert not router.registry().has_deployment("m-1") + table.rows = (_backup_row("m-1", model="openai/gpt-after-update"),) + await targeted_load + + installed: Final = router.registry().deployment("m-1") + assert installed is not None + assert installed.litellm_params.model == "openai/gpt-after-update" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +@pytest.mark.parametrize( + ("stored_credential_name", "credentials_on_this_worker", "api_key_this_worker_uses"), + [ + ("shared-openai", (), "sk-from-db"), + ("shared-openai", ("another-openai",), "sk-from-db"), + ("shared-openai", ("shared-openai",), "sk-on-this-worker"), + (None, (), None), + ], +) +async def test_resync_model_deployments_by_ids_reloads_credentials_only_for_one_this_worker_lacks( + monkeypatch: pytest.MonkeyPatch, + stored_credential_name: str | None, + credentials_on_this_worker: tuple[str, ...], + api_key_this_worker_uses: str | None, +) -> None: + import litellm + from litellm.litellm_core_utils.credential_accessor import CredentialAccessor + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + from litellm.types.utils import CredentialItem + + model_params: Final = {"model": "openai/gpt-backup"} + row: Final = _stored_model_row( + "backup-id", + model_params if stored_credential_name is None else model_params | {"litellm_credential_name": stored_credential_name}, + ) + stored_credential: Final = CredentialItem( + credential_name="shared-openai", + credential_values={"api_key": encrypt_value_helper("sk-from-db")}, + credential_info={}, + ) + router: Final = _router_serving("primary-id") + monkeypatch.setattr( + litellm, + "credential_list", + [ + CredentialItem(credential_name=name, credential_values={"api_key": "sk-on-this-worker"}, credential_info={}) + for name in credentials_on_this_worker + ], + ) + _serve_db_models(monkeypatch, _Db(_ProxyModelTable((row,)), _CredentialsTable((stored_credential,))), router) + + await resync_model_deployments_by_ids(("backup-id",)) + + credential: Final = CredentialAccessor.find_credential("shared-openai") + assert (credential.credential_values.get("api_key") if credential else None) == api_key_this_worker_uses + assert router.registry().has_deployment("backup-id") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("stored_params_salt") +async def test_resync_model_deployments_by_ids_installs_the_ordinary_rows_queued_with_an_auto_router( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import resync_model_deployments_by_ids + + table: Final = _ProxyModelTable( + ( + _stored_model_row("auto-router-id", {"model": "auto_router/complexity-router"}), + _backup_row("backup-id"), + ) + ) + db: Final = _Db(table) + router: Final = _router_serving("primary-id") + _serve_db_models(monkeypatch, db, router) + + await resync_model_deployments_by_ids(("auto-router-id", "backup-id")) + + assert _served_ids(router.registry().deployments()) == {"primary-id", "backup-id"} + assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "backup-id"} + assert table.reads == 1 + assert db.other_tables_read == [] + assert not proxy_server.MODEL_RECONCILE_LOCK.locked() diff --git a/tests/unit/proxy/management_endpoints/test_access_group_management.py b/tests/unit/proxy/management_endpoints/test_access_group_management.py index c36576beaa9..d79af0af519 100644 --- a/tests/unit/proxy/management_endpoints/test_access_group_management.py +++ b/tests/unit/proxy/management_endpoints/test_access_group_management.py @@ -687,33 +687,152 @@ class _FakeAccessGroupBudgetTable: return self.rows.pop(access_group_name, None) +class _FakeClock: + def __init__(self) -> None: + self.now = 0.0 + self.slept: list[float] = [] + + def __call__(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.now += seconds + + class _FakeModelTable: - def __init__(self, journal: list[str], deployments) -> None: + def __init__(self, journal: list[str], deployments, failing_model_ids=frozenset()) -> None: self.journal = journal self.deployments = list(deployments) + self.failing_model_ids = failing_model_ids self.updates: list[tuple[dict, dict]] = [] + self.rows_locked_by_another_session: frozenset[str] = frozenset() + self.group_locked_by_another_session = False + self.edits_the_other_session_commits: dict[str, dict] = {} + self.other_session_commits_after_refusals: int | None = None + self.lock_refusals = 0 async def find_many(self, where=None, **kwargs): - return list(self.deployments) + model_names = (where or {}).get("model_name", {}).get("in") + return [d for d in self.deployments if model_names is None or d.model_name in model_names] + + def lock_group(self, wait): + if not self.group_locked_by_another_session: + return True + if wait: + raise TimeoutError("lock_timeout expired waiting for the access group lock") + self._refuse_lock() + return False + + def lock_rows(self, model_ids, skip_locked): + held = self.rows_locked_by_another_session.intersection(model_ids) + if not held: + return list(model_ids) + if not skip_locked: + raise TimeoutError(f"timed out waiting for the row lock on {sorted(held)}") + self._refuse_lock() + return [model_id for model_id in model_ids if model_id not in held] + + def _refuse_lock(self): + self.lock_refusals += 1 + if self.lock_refusals == self.other_session_commits_after_refusals: + self._other_session_commits() + + def _other_session_commits(self): + self.deployments = [ + _FakeDeployment( + model_id=d.model_id, + model_name=d.model_name, + model_info=self.edits_the_other_session_commits.get(d.model_id, d.model_info), + ) + for d in self.deployments + ] + self.rows_locked_by_another_session = frozenset() + self.group_locked_by_another_session = False async def find_unique(self, where, include=None): return next((d for d in self.deployments if d.model_id == where["model_id"]), None) async def update(self, where, data, include=None): + if where["model_id"] in self.failing_model_ids: + raise RuntimeError(f"database write to {where['model_id']} failed") self.journal.append(f"model_table.update:{where['model_id']}") self.updates.append((dict(where), dict(data))) return None + async def create(self, data, **kwargs): + raise AssertionError("access group endpoints never create deployments") + + +class _FakeModelTableBatch: + """Queues model-table updates and applies them all or none when the batch closes, like a Prisma batch.""" + + def __init__(self, model_table: _FakeModelTable) -> None: + self.model_table = model_table + self.queued: list[tuple[dict, dict]] = [] + self.litellm_proxymodeltable = SimpleNamespace(update=self._queue_update) + + def _queue_update(self, where, data): + self.queued.append((dict(where), dict(data))) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + if exc_type is not None: + return False + failing = [where["model_id"] for where, _ in self.queued if where["model_id"] in self.model_table.failing_model_ids] + if failing: + raise RuntimeError(f"batch write to {failing} failed, nothing applied") + for where, data in self.queued: + await self.model_table.update(where=where, data=data) + return False + + +class _FakeModelTableTx: + """An interactive transaction on the primary that journals the row lock and the read it guards.""" + + def __init__(self, journal: list[str], model_table: _FakeModelTable) -> None: + self.journal = journal + self.model_table = model_table + self.litellm_proxymodeltable = SimpleNamespace(find_many=self._find_many) + + async def query_raw(self, query, *args): + if "advisory_xact_lock" in query: + self.journal.append(f"access_group.lock:{args[0]}") + return [{"locked": self.model_table.lock_group(wait="pg_try_" not in query)}] + if "FOR UPDATE" in query: + model_ids = sorted(args[0] if args else (d.model_id for d in self.model_table.deployments)) + self.journal.append(f"model_table.lock:{','.join(model_ids)}") + locked = self.model_table.lock_rows(model_ids, skip_locked="SKIP LOCKED" in query) + return [{"model_id": model_id} for model_id in locked] + self.journal.append(f"model_table.query:{query}") + return [] + + async def _find_many(self): + self.journal.append("model_table.read") + return await self.model_table.find_many() + + def batch_(self): + return _FakeModelTableBatch(self.model_table) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + class _FakePrismaClient: - def __init__(self, journal: list[str], deployments=()) -> None: + def __init__(self, journal: list[str], deployments=(), failing_model_ids=frozenset()) -> None: self.budget_table = _FakeBudgetTable(journal) self.access_group_budget_table = _FakeAccessGroupBudgetTable(journal, self.budget_table) - self.model_table = _FakeModelTable(journal, deployments) + self.model_table = _FakeModelTable(journal, deployments, failing_model_ids) self.db = SimpleNamespace( litellm_budgettable=self.budget_table, litellm_modelaccessgroupbudgettable=self.access_group_budget_table, litellm_proxymodeltable=self.model_table, + tx=lambda timeout=None: _FakeModelTableTx(journal, self.model_table), ) def jsonify_object(self, data): @@ -784,6 +903,12 @@ def _proxy_with_stubbed_reload(prisma): yield +def _serve_model_groups(router, model_groups): + router.registry.return_value.names.side_effect = lambda model_name=None, *, kind="groups", **_: ( + list(model_groups) if kind == "groups" else [] + ) + + def _eviction_journal(access_group): """Both auth cache keys, in the order a write path has to evict them.""" from litellm.proxy.common_utils.user_api_key_cache import ( @@ -1290,3 +1415,605 @@ async def test_deleting_an_access_group_that_never_had_a_budget_still_evicts(): assert response.models_updated == 1 assert prisma.access_group_budget_table.rows == {} _assert_evicted_after_write(journal, "prod-models", "access_group_budget.delete:prod-models") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selection", + [{"model_ids": ["stays", "joins"]}, {"model_names": ["gpt-stays", "gpt-joins"]}], +) +async def test_updating_an_access_group_writes_only_the_deployments_that_join_or_leave_it(selection): + import json + + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"]) + response = await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(**selection), + user_api_key_dict=_admin(), + ) + + written_access_groups = { + where["model_id"]: json.loads(data["model_info"])["access_groups"] + for where, data in prisma.model_table.updates + } + assert written_access_groups == {"joins": ["prod-models"], "leaves": []} + assert response.models_updated == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "selection", + [{"model_ids": ["stays", "joins"]}, {"model_names": ["gpt-stays", "gpt-joins"]}], +) +async def test_updating_an_access_group_reads_the_primary_not_a_lagging_replica(selection): + import json + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.db.prisma_client import PrismaWrapper + from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + primary = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + replica_before_the_last_edit = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves", access_groups=()), + _deployment(model_id="joins", model_name="gpt-joins"), + ], + ) + primary.db = RoutingPrismaWrapper( + writer=PrismaWrapper(original_prisma=primary.db, iam_token_db_auth=False), + reader=PrismaWrapper(original_prisma=replica_before_the_last_edit.db, iam_token_db_auth=False), + ) + + with _proxy_with_stubbed_reload(primary): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"]) + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(**selection), + user_api_key_dict=_admin(), + ) + + written_access_groups = { + where["model_id"]: json.loads(data["model_info"])["access_groups"] + for where, data in primary.model_table.updates + } + assert written_access_groups == {"joins": ["prod-models"], "leaves": []} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failing_model_id", ["leaves", "joins"]) +async def test_a_failed_access_group_update_leaves_every_deployment_unchanged(failing_model_id): + from fastapi import HTTPException + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + deployments = [ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ] + prisma = _FakePrismaClient([], deployments=deployments, failing_model_ids=frozenset({failing_model_id})) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-leaves", "gpt-joins"]) + with pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 500 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_an_access_group_update_keeps_a_model_info_field_another_admin_set_while_names_were_validated(): + import json + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + def another_admin_edits_joins_while_names_are_checked(model_name=None, *, kind="groups", **_): + if kind != "groups": + return [] + prisma.model_table.deployments = [ + deployment + if deployment.model_id != "joins" + else _FakeDeployment( + model_id="joins", + model_name="gpt-joins", + model_info={"access_groups": [], "description": "set by another admin"}, + ) + for deployment in prisma.model_table.deployments + ] + return ["gpt-stays", "gpt-joins"] + + with _proxy_with_stubbed_reload(prisma): + proxy_server.llm_router.registry.return_value.names.side_effect = ( + another_admin_edits_joins_while_names_are_checked + ) + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_names=["gpt-stays", "gpt-joins"]), + user_api_key_dict=_admin(), + ) + + written = {where["model_id"]: json.loads(data["model_info"]) for where, data in prisma.model_table.updates} + assert written["joins"]["description"] == "set by another admin" + assert written["joins"]["access_groups"] == ["prod-models"] + + +@pytest.mark.asyncio +async def test_an_access_group_update_locks_the_rows_it_reads_and_tells_other_workers_after_the_commit(): + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + journal: list[str] = [] + prisma = _FakePrismaClient( + journal, + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + async def record_publish(model_ids): + journal.append(f"publish:{','.join(sorted(model_ids))}") + + with ( + _proxy_with_stubbed_reload(prisma), + patch( # test-quality-ok: the endpoint publishes through its module-level import; Redis is the real seam + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.publish_model_rows_written", + new=record_publish, + ), + ): + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"]), + user_api_key_dict=_admin(), + ) + + assert journal == [ + "access_group.lock:prod-models", + "model_table.read", + "model_table.lock:joins,leaves,stays", + "model_table.read", + "model_table.update:leaves", + "model_table.update:joins", + "publish:joins,leaves", + ] + + +@pytest.mark.asyncio +async def test_an_access_group_update_that_changes_no_membership_writes_and_announces_nothing(): + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + journal: list[str] = [] + prisma = _FakePrismaClient( + journal, + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="also-stays", model_name="gpt-also-stays"), + _deployment(model_id="outside", model_name="gpt-outside", access_groups=()), + ], + ) + + async def record_publish(model_ids): + journal.append(f"publish:{','.join(sorted(model_ids))}") + + with ( + _proxy_with_stubbed_reload(prisma), + patch( # test-quality-ok: the endpoint publishes through its module-level import; Redis is the real seam + "litellm.proxy.management_endpoints.model_access_group_management_endpoints.publish_model_rows_written", + new=record_publish, + ), + ): + response = await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "also-stays"]), + user_api_key_dict=_admin(), + ) + + assert response.models_updated == 2 + assert journal == [ + "access_group.lock:prod-models", + "model_table.read", + "model_table.lock:also-stays,stays", + "model_table.read", + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ignored_model_names", [None, ["gpt-unrelated"]]) +async def test_an_access_group_update_goes_through_while_another_session_holds_a_model_row_outside_the_group( + ignored_model_names, +): + import json + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + _deployment(model_id="unrelated", model_name="gpt-unrelated", access_groups=("other-group",)), + ], + ) + prisma.model_table.rows_locked_by_another_session = frozenset({"unrelated"}) + + with _proxy_with_stubbed_reload(prisma): + response = await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"], model_names=ignored_model_names), + user_api_key_dict=_admin(), + ) + + written_access_groups = { + where["model_id"]: json.loads(data["model_info"])["access_groups"] + for where, data in prisma.model_table.updates + } + assert written_access_groups == {"joins": ["prod-models"], "leaves": []} + assert response.models_updated == 2 + + +@pytest.mark.asyncio +async def test_an_access_group_update_retries_a_locked_row_and_keeps_the_edit_committed_there(): + import json + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + prisma.model_table.rows_locked_by_another_session = frozenset({"joins"}) + prisma.model_table.edits_the_other_session_commits = { + "joins": {"access_groups": [], "description": "saved while the update waited"} + } + prisma.model_table.other_session_commits_after_refusals = 1 + + with _proxy_with_stubbed_reload(prisma): + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays", "joins"]), + user_api_key_dict=_admin(), + ) + + written = {where["model_id"]: json.loads(data["model_info"]) for where, data in prisma.model_table.updates} + assert written == {"joins": {"access_groups": ["prod-models"], "description": "saved while the update waited"}} + + +@pytest.mark.asyncio +async def test_an_access_group_emptied_while_the_update_retried_a_locked_row_answers_404_and_writes_nothing(): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="leaves", model_name="gpt-leaves"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + prisma.model_table.rows_locked_by_another_session = frozenset({"leaves"}) + prisma.model_table.edits_the_other_session_commits = {"leaves": {"access_groups": []}} + prisma.model_table.other_session_commits_after_refusals = 1 + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["joins"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 404 + assert prisma.model_table.updates == [] + + +def _another_session_holds_a_member_row(model_table): + model_table.rows_locked_by_another_session = frozenset({"leaves"}) + + +def _another_edit_holds_the_group(model_table): + model_table.group_locked_by_another_session = True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"] +) +async def test_an_access_group_update_answers_409_and_writes_nothing_while_another_session_keeps_a_lock_it_needs( + hold, +): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + _rewrite_membership_once_unlocked, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + ], + ) + hold(prisma.model_table) + clock = _FakeClock() + + with pytest.raises(HTTPException) as raised: + await _rewrite_membership_once_unlocked( + prisma_client=prisma, + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays"]), + retry_budget_seconds=0.5, + sleep=clock.sleep, + clock=clock, + ) + + assert raised.value.status_code == 409 + assert 0 < clock.now <= 0.5 + assert prisma.model_table.lock_refusals == len(clock.slept) + 1 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "hold", [_another_session_holds_a_member_row, _another_edit_holds_the_group], ids=["member_row", "group"] +) +async def test_an_access_group_update_waits_out_a_lock_held_for_over_ten_seconds_and_writes_the_edit(hold): + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + _rewrite_membership_once_unlocked, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="leaves", model_name="gpt-leaves"), + ], + ) + hold(prisma.model_table) + prisma.model_table.other_session_commits_after_refusals = 20 + clock = _FakeClock() + + _, rewrites = await _rewrite_membership_once_unlocked( + prisma_client=prisma, + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays"]), + sleep=clock.sleep, + clock=clock, + ) + + assert clock.now > 10 + assert [model_id for model_id, _ in rewrites] == ["leaves"] + assert [where["model_id"] for where, _ in prisma.model_table.updates] == ["leaves"] + + +@pytest.mark.asyncio +async def test_updating_a_missing_access_group_answers_404_before_checking_the_model_names(): + from fastapi import HTTPException + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], deployments=[_deployment(model_id="other", model_name="gpt-other", access_groups=("other-group",))] + ) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-other"]) + with pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_names=["no-such-model"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 404 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_an_access_group_update_answers_500_when_the_deployments_cannot_be_read(): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient([], deployments=[_deployment(model_id="stays")]) + prisma.model_table.find_many = AsyncMock(side_effect=RuntimeError("database unreachable")) + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["stays"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 500 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_ids", [["no-such-deployment"], ["joins", "no-such-deployment"]]) +async def test_updating_an_access_group_with_an_unknown_model_id_writes_no_deployment(model_ids): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[_deployment(model_id="stays"), _deployment(model_id="joins", access_groups=())], + ) + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=model_ids), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 400 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_updating_an_access_group_no_deployment_carries_answers_404_and_writes_nothing(): + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient([], deployments=[_deployment(model_id="other", access_groups=("other-group",))]) + + with _proxy_with_stubbed_reload(prisma), pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_ids=["other"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 404 + assert prisma.model_table.updates == [] + + +@pytest.mark.asyncio +async def test_updating_an_access_group_with_a_model_name_only_the_config_serves_writes_no_deployment(): + from fastapi import HTTPException + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_endpoints.model_access_group_management_endpoints import ( + update_access_group, + ) + from litellm.types.proxy.management_endpoints.model_management_endpoints import ( + UpdateModelGroupRequest, + ) + + prisma = _FakePrismaClient( + [], + deployments=[ + _deployment(model_id="stays", model_name="gpt-stays"), + _deployment(model_id="joins", model_name="gpt-joins", access_groups=()), + ], + ) + + with _proxy_with_stubbed_reload(prisma): + _serve_model_groups(proxy_server.llm_router, ["gpt-stays", "gpt-joins", "config-only-model"]) + with pytest.raises(HTTPException) as raised: + await update_access_group( + access_group="prod-models", + data=UpdateModelGroupRequest(model_names=["gpt-joins", "config-only-model"]), + user_api_key_dict=_admin(), + ) + + assert raised.value.status_code == 400 + assert prisma.model_table.updates == [] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 14d93f368c1..872d1607351 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -6508,7 +6508,11 @@ class TestStrategyRouterWriteValidation: assert fake.tx_obj.raw_calls == [] return assert handle is fake.tx_obj.litellm_proxymodeltable - published.assert_awaited_once_with(redis_cache=None, object_type="litellm_proxymodeltable") + published.assert_awaited_once_with( + redis_cache=None, + object_type="litellm_proxymodeltable", + model_ids=(model_id,) if model_id else (), + ) (lock_sql, lock_params), (count_sql, count_params) = fake.tx_obj.raw_calls assert "pg_advisory_xact_lock($1)" in lock_sql and "count" not in lock_sql assert lock_params == (AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13a064517a4..7d9e9839a5f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -699,8 +699,8 @@ export interface paths { * @description Update an access group's model names. * * This will: - * 1. Remove the access group from all current deployments - * 2. Add the access group to all deployments for the new model_names list + * 1. Add the access group to all deployments for the new model_names list + * 2. Remove the access group from every other deployment * * Example: * ```bash @@ -722,6 +722,7 @@ export interface paths { * Raises: * - HTTPException 400: If any model names don't exist * - HTTPException 404: If access group not found + * - HTTPException 409: If another edit holds the group or one of its models for 30 s */ put: operations["update_access_group_access_group__access_group__update_put"]; post?: never;