From ea51e77dc5bd718e16de370ae32badf955d3ce7f Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:39:56 -0700 Subject: [PATCH 01/25] fix(responses)!: keep the proxy's error body on litellm.responses errors from litellm_proxy models (#49073) * fix(responses): keep the proxy's error body on litellm.responses errors from litellm_proxy models litellm.responses() with a litellm_proxy/ model raised BadRequestError with body None when the proxy answered a 4xx, while litellm.completion() on the same proxy carries the decoded error object. The litellm_proxy Responses config now decodes the error JSON into the exception body the way the OpenAI SDK does, so body, code, type and the message match chat completions on sync, async and streaming calls. * fix(proxy): relay a downstream proxy's provider_specific_fields in error responses A LiteLLM proxy routing to another LiteLLM proxy now copies the downstream's provider_specific_fields (which guardrail fired) from the exception body into its own error answer, on chat and Responses alike. Adds tests for the two _openai_sdk_error_body branches the existing tests left open. * fix(proxy): redact internal details from relayed provider_specific_fields * fix(proxy): redact relayed provider_specific_fields per key so credential-named values stay hidden * fix(proxy): redact numeric credentials in relayed provider_specific_fields --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/_logging.py | 28 +++++ .../litellm_proxy/responses/transformation.py | 32 ++++++ litellm/proxy/common_request_processing.py | 4 +- .../common_utils/openai_error_payload.py | 16 +++ .../code_coverage_tests/recursive_detector.py | 1 + .../responses/test_transformation.py | 84 +++++++++++++++ .../proxy/test_common_request_processing.py | 101 ++++++++++++++++++ 7 files changed, 265 insertions(+), 1 deletion(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 5279cb233b4..6a38eea2b12 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( + REDACTED, python_redact_string, python_redact_structured_value, redact_internal_details, + redact_structured_value, ) from litellm.litellm_core_utils.secret_redaction import ( redact_string as redact_secret_string, @@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str: return redact_internal_details(value) +_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object]) +_CLIENT_ITEMS: Final = TypeAdapter(list[object]) + + +def _redacted_client_field(key: str | None, value: object) -> object: + if isinstance(value, str): + return redact_internal_details(redact_structured_value(key, value)) + if isinstance(value, dict): + return { + child_key: _redacted_client_field(child_key, child) + for child_key, child in _CLIENT_FIELDS.validate_python(value).items() + } + if isinstance(value, (list, tuple)): + return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)] + if not isinstance(value, (int, float)): + return value + text: Final = str(value) + return value if redact_structured_value(key, text) == text else REDACTED + + +def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object: + if not _ENABLE_SECRET_REDACTION: + return fields + return _redacted_client_field(None, dict(fields)) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index 3092b49b743..fc94eb30504 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod This config enables pass-through behavior to the proxy's /v1/responses endpoint. """ +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + + +def _openai_sdk_error_body( + error_message: str, +) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict + try: + decoded: Final = _ERROR_BODY.validate_json(error_message) + except ValidationError: + return None + nested: Final = decoded.get("error") + return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded + class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "responses") + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + body=_openai_sdk_error_body(error_message), + ) + def supports_native_websocket(self) -> bool: """LiteLLM Proxy does not support native WebSocket for Responses API""" return False diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 666269be2eb..7cd752c2297 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -114,6 +114,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( error_status_code, openai_error_param, openai_error_type, + relayed_provider_specific_fields, ) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, @@ -4462,7 +4463,8 @@ class ProxyBaseLLMRequestProcessing: param=openai_error_param(e), openai_code=getattr(e, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None), + provider_specific_fields=getattr(e, "provider_specific_fields", None) + or relayed_provider_specific_fields(e), headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 3c91e87ee51..f92fbeef9a1 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -8,7 +8,9 @@ from types import MappingProxyType from typing import Final from fastapi import HTTPException, status +from pydantic import TypeAdapter +from litellm._logging import redact_internal_details_from_client_fields from litellm.constants import STRINGIFIED_NONE from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded @@ -26,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) _FAILOVER_REFUSALS: Final = ( @@ -77,6 +81,18 @@ def openai_error_param(exc: object) -> str | None: return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None +def relayed_provider_specific_fields( + exc: object, +) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict + body: Final = attribute_of(exc, "body") + if not isinstance(body, dict): + return None + fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields") + if not isinstance(fields, dict): + return None + return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields))) + + def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException: match exc: case ProxyException(): diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..c943ff2cbd0 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -81,6 +81,7 @@ IGNORE_FUNCTIONS = [ "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_redacted_client_field", # bounded by the nesting depth of a JSON-decoded error body (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index 3d774b79eed..5537d9d1916 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable, Iterable from typing import Final import httpx @@ -94,3 +95,86 @@ def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop "user": "end-user-1", "temperature": 0.2, } + + +_GUARDRAIL_BLOCK: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call"}, + } +} + + +def _proxy_rejection(route: str, response: httpx.Response, call: Callable[[], object]) -> litellm.BadRequestError: + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://my-proxy.example.com/{route}").mock(return_value=response) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +def _consume(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +_PROXY_PARAMS: Final = {"api_base": "https://my-proxy.example.com/", "api_key": "sk-proxy-key"} + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +def test_proxy_400_reaches_responses_caller_with_the_same_body_as_chat_completions(stream: bool) -> None: + chat_error: Final = _proxy_rejection( + "chat/completions", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_PROXY_PARAMS + ), + ) + responses_error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: ( + _consume(litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", stream=True, **_PROXY_PARAMS)) + if stream + else litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_PROXY_PARAMS) + ), + ) + assert responses_error.body == chat_error.body == _GUARDRAIL_BLOCK["error"] + assert (responses_error.code, responses_error.type) == (chat_error.code, chat_error.type) + assert str(responses_error) == str(chat_error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_without_a_json_object_body_still_raises_with_its_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, text="upstream proxy rejected the request"), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body is None + assert "upstream proxy rejected the request" in str(error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_object_without_an_error_member_is_the_body_as_sent() -> None: + sent: Final = {"message": "upstream proxy rejected the request", "type": "invalid_request_error"} + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=sent), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body == sent + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_whose_error_member_is_text_still_raises_with_that_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json={"error": "upstream proxy rejected the request"}), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert "upstream proxy rejected the request" in str(error) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index cc1fc5cd6c3..aeab7a3d348 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -14,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx import pytest +import respx from fastapi import HTTPException, Request, Response, status from fastapi.responses import JSONResponse, StreamingResponse @@ -12516,3 +12517,103 @@ def test_maybe_get_model_id_prefers_litellm_metadata_over_caller_metadata_withou def test_maybe_get_model_id_returns_none_before_the_router_picked_a_deployment(): assert ProxyBaseLLMRequestProcessing(data={"metadata": {}}).maybe_get_model_id(None) is None + + +_DOWNSTREAM_GUARDRAIL_FIELDS: Final = MappingProxyType( + {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call", "pattern": "secret_project_codename"} +) + + +def _downstream_proxy_rejection( + route: str, call: Callable[[], object], fields: Mapping[str, object] = _DOWNSTREAM_GUARDRAIL_FIELDS +) -> Exception: + body: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": dict(fields), + } + } + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://downstream-proxy.example.com/{route}").mock(return_value=httpx.Response(400, json=body)) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +_DOWNSTREAM: Final = MappingProxyType( + {"api_base": "https://downstream-proxy.example.com/", "api_key": "sk-downstream-key"} +) + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + ("route", "call"), + [ + ( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + ), + ( + "chat/completions", + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_DOWNSTREAM + ), + ), + ], +) +async def test_a_chained_proxy_relays_which_downstream_guardrail_blocked_the_request( + route: str, call: Callable[[], object] +) -> None: + rejection: Final = _downstream_proxy_rejection(route, call) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + assert relayed.value.to_dict()["provider_specific_fields"] == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + + +_DOWNSTREAM_LEAKABLE_VALUES: Final = ( + "guard-scan.internal.corp", + "/srv/secret/app/guard.py", + "10.0.4.17", + "abcd1234efgh5678", + "cs-0123456789", + "tok-0123456789abcdef", + "4815162342108", +) + + +@pytest.mark.usefixtures("httpx_transport") +async def test_a_chained_proxy_redacts_secrets_and_internal_details_from_relayed_guardrail_fields() -> None: + fields: Final = { + **_DOWNSTREAM_GUARDRAIL_FIELDS, + "exception": "connect to http://guard-scan.internal.corp:9000/v1/check failed at /srv/secret/app/guard.py", + "api_key": "abcd1234efgh5678", + "access_token": ["tok-0123456789abcdef"], + "scanner": { + "hosts": ["10.0.4.17:8080"], + "client_secret": "cs-0123456789", + "api_secret": 4815162342108, + "attempts": 2, + }, + } + rejection: Final = _downstream_proxy_rejection( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + fields, + ) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + answered: Final = json.loads(json.dumps(relayed.value.to_dict()["provider_specific_fields"])) + assert {key: answered[key] for key in _DOWNSTREAM_GUARDRAIL_FIELDS} == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + assert answered["scanner"]["attempts"] == 2 + assert [value for value in _DOWNSTREAM_LEAKABLE_VALUES if value in json.dumps(answered)] == [] From 2b4cb92c8b0e470b1fd8a4ae0d91c8193353abde Mon Sep 17 00:00:00 2001 From: Alon Sokolovski Date: Fri, 30 Oct 2026 02:42:11 +0200 Subject: [PATCH 02/25] fix(context_management)!: run clear_tool_uses in the gateway on the Responses API path (#47861) * fix(context_management): run clear_tool_uses in the gateway on the Responses API path /v1/messages to openai/ models goes through the Responses API, which translated only compact_20260112 (to its native compaction) and dropped every other edit without a warning, so clear_tool_uses_20250919 never cleared anything. Edits without a Responses API form now run in the same polyfill as the chat completions path, before the call, and their applied_edits come back on the response and the final message_delta. compact stays native. * fix(context_management): check the sync polyfill result instead of casting it The strict ruff gate (TID251) bans typing.cast. * test(context_management): cover the Responses path's polyfill error handling A polyfill crash is a 500 before the call, a rejected edit under drop_params is skipped, and the sync stream reports its edits. normalize_spec never raises for a dict spec, so its guard is gone. * fix(context_management): count unreadable blocks and check tool_result text before clearing clear_tool_uses now maps blocks the token counter cannot read (server tool, compaction) to JSON text, so an input_tokens trigger no longer 500s on that history. The Responses API path checks malformed tool_result text on the original messages before clearing, as the chat completions path does. Also removes comments that repeated the code and makes the test fixtures immutable. * fix(context_management): count only blocks the model receives and keep list specs unsent clear_tool_uses now leaves blocks token_counter cannot read, like server tool results and compaction, out of its count instead of counting them as JSON text. No adapter sends them to the model, so an input_tokens trigger no longer fires on tokens the model never sees. On the Responses API path, an OpenAI-style list context_management stays unsent, as it was before, instead of reaching OpenAI's native compaction, where a threshold under 1000 is a 400. On the chat completions path, a clear edit listed before compact_20260112 now runs on the turns after the latest compaction block, as Anthropic does, instead of counting and clearing turns the compaction block drops. The crash test that replaced apply_context_management with a mock is removed. * fix(context_management): apply clear edits in OpenAI count_tokens and skip nested off-spec blocks when counting * fix(context_management): run edits on the history drop_params sends, so count_tokens and /v1/messages clear alike * fix(context_management): count clear_at_least freed tokens over sent blocks only * fix(context_management): count tools with JSON Schema shorthands and reject a non-object trigger or keep A tool whose schema has an array without items, a boolean subschema, or a search_result block without a title made the token counter raise, so a clear edit answered 500. A string trigger or keep also answered 500, even under drop_params. Both now count or get the API's 400, and drop_params drops the malformed field. * fix(token_counter): count a malformed tool schema keyword as absent and move shorthand framed-estimate cases * fix(count_tokens): pass the system prompt into clearing so count_tokens matches inference * fix(context_management): keep a malformed system prompt from failing clear_tool_uses on the Responses API A number, an object, or a list holding non-object items as system made the polyfill answer 500. The polyfill now keeps a list's object items and treats any other shape as no system, on both /v1/messages and count_tokens, so the two paths still count the same text --------- Co-authored-by: AlonSoko <25619863+AlonSoko@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 25 +- .../anthropic/messages/adapters/handler.py | 3 +- .../messages/context_management/dispatcher.py | 5 +- .../editors/clear_tool_uses.py | 75 +++- .../context_management/editors/compact.py | 8 +- .../messages/responses_adapters/handler.py | 153 ++++++++- .../responses_adapters/streaming_iterator.py | 10 +- .../responses/count_tokens/token_counter.py | 45 ++- .../litellm_core_utils/test_token_counter.py | 51 +++ .../test_clear_tool_uses.py | 130 ++++++- .../context_management/test_compact.py | 6 +- .../context_management/test_dispatcher.py | 55 +++ .../test_responses_adapters_handler.py | 325 +++++++++++++++++- .../count_tokens/test_token_counter.py | 120 ++++++- .../vertex_ai/count_tokens/test_context.py | 2 +- tests/unit/proxy/test_proxy_token_counter.py | 54 ++- 16 files changed, 993 insertions(+), 74 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 591d77bdb16..5ad4e130e62 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1224,10 +1224,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type: if expected_cls is None: raise ValueError(f"Unknown Anthropic content type: '{content_type}'") - missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content] - if missing: - raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}") - return expected_cls @@ -1552,12 +1548,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[ def _format_object_parameters(parameters, indent): properties: Final = parameters.get("properties") - if not properties: + if not isinstance(properties, Mapping) or not properties: return "" - required_params: Final = parameters.get("required", []) + required: Final = parameters.get("required") + required_params: Final = tuple(required) if isinstance(required, list) else () lines: Final = [] for key, props in properties.items(): - description = props.get("description") + description = props.get("description") if isinstance(props, Mapping) else None if description: lines.append(f"// {description}") question = "?" @@ -1568,19 +1565,21 @@ def _format_object_parameters(parameters, indent): def _format_type(props, indent): + if not isinstance(props, Mapping): + return "any" type: Final = props.get("type") + enum: Final = props.get("enum") if type == "string": - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "string" elif type == "array": - # items is required, OpenAI throws an error if it's missing - return f"{_format_type(props['items'], indent)}[]" + return f"{_format_type(props.get('items'), indent)}[]" elif type == "object": return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}" elif type in ["integer", "number"]: - if "enum" in props: - return " | ".join([f'"{item}"' for item in props["enum"]]) + if isinstance(enum, list): + return " | ".join([f'"{item}"' for item in enum]) return "number" elif type == "boolean": return "boolean" diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 71ccfbaf915..704bec9056f 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -160,6 +160,7 @@ async def prepare_context_managed_request( """Apply client compaction history, then optional context_management polyfill.""" from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, + slice_around_compaction_block, ) # Skip the client-history pre-processing when a ``compact_20260112`` @@ -175,7 +176,7 @@ async def prepare_context_managed_request( if polyfill_will_run: history_result: PolyfillResult | None = None - working_messages: _AnthropicMessages = messages + working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0]) working_system: _AnthropicSystem = system else: history_result = apply_client_compaction_block_history( diff --git a/litellm/llms/anthropic/messages/context_management/dispatcher.py b/litellm/llms/anthropic/messages/context_management/dispatcher.py index 1557cf206ac..2b713646d50 100644 --- a/litellm/llms/anthropic/messages/context_management/dispatcher.py +++ b/litellm/llms/anthropic/messages/context_management/dispatcher.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.types.llms.anthropic import AppliedEdit +from ..convertible_blocks import drop_non_string_tool_result_texts from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE from .editors import ( CheckedEdit, @@ -103,7 +104,7 @@ async def apply_context_management( Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400 before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under - ``drop_invalid_edits``. + ``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it. The dispatcher is async so async editors (``compact_20260112``) can ``await`` the configured summarization model. Sync editors run in a @@ -119,7 +120,7 @@ async def apply_context_management( raise first_error well_formed_edits: Final = tuple(check.edit for check in checked) - current_messages = messages + current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits)) current_system = system aggregated_applied: Final[list[AppliedEdit]] = [] aggregated_compaction_block = None diff --git a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py index c4f7966f429..fabc112d443 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py +++ b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py @@ -39,6 +39,21 @@ if TYPE_CHECKING: T = TypeVar("T") # api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep _TINY_RESULT_BYTES: Final = 46 +_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset( + { + "text", + "image_url", + "image", + "document", + "file", + "tool_use", + "tool_result", + "thinking", + "redacted_thinking", + "tool_reference", + "search_result", + } +) ToolNames = Literal["all"] | frozenset[str] @@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel): class _ClearToolUsesFields(BaseModel): model_config = ConfigDict(frozen=True) + trigger: Mapping[str, object] | None = None + keep: Mapping[str, object] | None = None clear_at_least: _ClearAtLeast | None = None exclude_tools: tuple[str, ...] | None = None clear_tool_inputs: bool | tuple[str, ...] | None = None _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) -# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently +# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently _API_MESSAGES: Final = MappingProxyType( { "tuple_type": "Input should be a valid array", + "dict_type": "Input should be an object", "model_type": "Input does not match the expected shape.", } ) @@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]: def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int: """Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it.""" - before: Final = [result, use] if clears_input else [result] - after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)] + before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,) + after: Final[Sequence[Mapping[str, object]]] = ( + (_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),) + ) counts: Final = tuple( - litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after) + _count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after) ) return max(counts[0] - counts[1], 0) @@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle return tuple(candidates[:batch_end]) +def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]: + nested: Final = _parse(_BLOCKS, block.get("content")) + if nested is None: + return block + return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]} + + +def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]: + blocks: Final = _parse(_BLOCKS, message.get("content")) + if blocks is None: + return message + return { + **message, + "content": [ + _without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES + ], + } + + +def _count_tokens( + model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None +) -> int: + return litellm.token_counter( + model=model, + messages=[_countable_message(message) for message in messages], + tools=cast("list[ChatCompletionToolParam] | None", tools), + ) + + def _trigger_met( trigger: dict[str, object], model: str, @@ -218,11 +267,7 @@ def _trigger_met( if not isinstance(threshold, int): threshold = DEFAULT_INPUT_TOKENS_TRIGGER - message_tokens: Final = litellm.token_counter( - model=model, - messages=messages, - tools=cast("list[ChatCompletionToolParam] | None", tools), - ) + message_tokens: Final = _count_tokens(model, messages, tools) if message_tokens > threshold: return True, message_tokens current_tokens: Final = message_tokens + count_system_tokens(model, system) @@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919( [_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited ) - tokens_before: Final = ( - counted_tokens - if counted_tokens is not None - else litellm.token_counter( - model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) - ) + tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools) kept_after, _ = history_after_client_compaction(final_messages, system) - tokens_after: Final = litellm.token_counter( - model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools) - ) + tokens_after: Final = _count_tokens(model, kept_after, tools) cleared_input_tokens: Final = max(tokens_before - tokens_after, 0) return final_messages, { diff --git a/litellm/llms/anthropic/messages/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py index 0bf8cac7344..d1cfcc2f2bc 100644 --- a/litellm/llms/anthropic/messages/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -137,7 +137,7 @@ def _find_latest_compaction_index( return None, None -def _slice_around_compaction_block( +def slice_around_compaction_block( messages: Sequence[_MsgT], ) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]: """Apply Anthropic's "drop everything before the compaction block" rule. @@ -194,7 +194,7 @@ def history_after_client_compaction( system: str | Sequence[Mapping[str, object]] | None, ) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]: """Return the history the API keeps after the latest compaction block, without it, and the system with its summary.""" - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) if block is None: return list(messages), system tail: Final = _strip_compaction_blocks(sliced) @@ -615,7 +615,7 @@ def apply_client_compaction_block_history( tail is forwarded unchanged (with compaction blocks stripped) so recent turns the summary does not cover are preserved. """ - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) if prior_compaction_block is None: return None @@ -686,7 +686,7 @@ async def apply_compact_20260112( # opt-in gate below so that even when summarization is disabled we still # strip Anthropic-only ``compaction`` blocks from messages going to # non-Anthropic backends (which would reject them). - effective_messages, prior_compaction_block = _slice_around_compaction_block(messages) + effective_messages, prior_compaction_block = slice_around_compaction_block(messages) prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None augmented_system: str | list[dict[str, object]] | None = system if isinstance(prior_summary_text, str) and prior_summary_text: diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 9b22170c9db..0eef3fd87c0 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -10,11 +10,15 @@ from typing import Final, TypeAlias from pydantic import TypeAdapter, ValidationError import litellm +from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.asyncify import ( + run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_for_wire, @@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, AnthropicOutputConfig, AnthropicOutputSchema, + ContextManagementResponse, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import request_safeguards_evaluator +from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management +from ..context_management.constants import COMPACT_EDIT_TYPE +from ..context_management.dispatcher import normalize_spec from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name @@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]] _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) +_SYSTEM_ITEMS: Final = TypeAdapter(list[object]) +_SPEC: Final = TypeAdapter(dict[str, object]) +_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]] +AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() +def _split_context_management( + spec: AnthropicContextManagementSpec, + kwargs: Mapping[str, object], +) -> _SplitEdits: + """Return (the edits the Responses API runs natively, the edits the gateway polyfill runs). + + The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other + edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as + it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays + unsent, as it was before the polyfill ran here. + """ + if not isinstance(spec, dict): + return None, () + if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")): + return spec, () + edits: Final = normalize_spec(spec) or [] + native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE] + return ({"edits": native} if native else None), tuple( + edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE + ) + + +def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None: + if system is None or isinstance(system, str): + return system + try: + items: Final = _SYSTEM_ITEMS.validate_python(system) + except ValidationError: + return None + return tuple(block for block in map(as_str_mapping, items) if block is not None) + + +async def _polyfilled( + *, + model: str, + messages: AnthropicRequestMessages, + tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None, + system: object, + edits: Sequence[Mapping[str, object]], + kwargs: Mapping[str, object], +) -> PolyfillResult | None: + """Run the edits the Responses API can't run natively; raise like the chat completions path does.""" + if not edits: + return None + reject_malformed_tool_result_text(messages, model, kwargs) + drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params")) + blocks: Final = _system_blocks(system) + try: + return await apply_context_management( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=_BLOCK_LIST.validate_python(tools) if tools else None, + system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks), + context_management_spec={"edits": [dict(edit) for edit in edits]}, + drop_invalid_edits=drop_invalid_edits, + ) + except AnthropicContextManagementError: + raise + except Exception as e: + verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e) + raise AnthropicContextManagementError( + status_code=500, message=f"context_management polyfill failed: {e}" + ) from e + + +async def count_view_messages( + *, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]]: + """The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively.""" + if context_management is None or not isinstance(context_management.spec, dict): + return messages + _, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {}) + result: Final = await _polyfilled( + model=model, + messages=_BLOCK_LIST.validate_python(messages), + tools=tools, + system=system, + edits=edits, + kwargs={"drop_params": context_management.drop_invalid_edits}, + ) + return messages if result is None else result.messages + + +def _with_applied_edits( + response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None +) -> AnthropicMessagesResponse: + applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None + if not applied_edits: + return response + return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))} + + def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None: logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs) if logging_obj is not None: @@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format: AnthropicOutputSchema | None = None, **kwargs: object, ) -> AnthropicMessagesResponse | AsyncIterator[bytes]: + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + polyfill_result: Final = await _polyfilled( + model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs + ) responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler: model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), safeguards_evaluator=safeguards_evaluator, + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator) + served: Final = await with_safeguard_results( + _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator + ) _record_bridged_response(responses_kwargs, result) return served @@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler: max_tokens: int, messages: AnthropicRequestMessages, model: str, - context_management: dict[str, object] | None = None, + context_management: AnthropicContextManagementSpec = None, metadata: dict[str, object] | None = None, output_config: AnthropicOutputConfig | None = None, stop_sequences: list[str] | None = None, @@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler: ) # Sync path + native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs) + ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped + run_async_function( + _polyfilled, + model=model, + messages=messages, + tools=tools, + system=system, + edits=polyfilled_edits, + kwargs=kwargs, + ) + if polyfilled_edits + else None + ) + polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None responses_kwargs: Final = _build_responses_kwargs( max_tokens=max_tokens, - messages=messages, + messages=polyfill_result.messages if polyfill_result is not None else messages, model=model, - context_management=context_management, + context_management=native_context_management, metadata=metadata, output_config=output_config, stop_sequences=stop_sequences, @@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler: responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider")), litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs), + applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None, ) return wrapper.async_anthropic_sse_wrapper() if not isinstance(result, ResponsesAPIResponse): raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") - return _ADAPTER.translate_response(result) + return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result) diff --git a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index 44f233f2254..dae96ba4ce1 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -2,7 +2,7 @@ ## Translates OpenAI call to Anthropic `/v1/messages` format import asyncio from collections import deque -from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping +from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import anyio @@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import ( partial_responses_incomplete_event_on_disconnect, stream_error_status_and_message, ) +from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.base import LiteLLMBaseModel @@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper: model: str, litellm_logging_obj: "LiteLLMLoggingObject | None" = None, safeguards_evaluator: "SafeguardsEvaluator | None" = None, + applied_edits: Sequence[AppliedEdit] | None = None, ) -> None: self.responses_stream = responses_stream + self._applied_edits: Final = tuple(applied_edits or ()) self.model = model self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator @@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper: "type": "message_delta", "delta": message_delta_payload, "usage": dict(anthropic_usage), + **( + {"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))} + if self._applied_edits + else {} + ), } ) self._chunk_queue.append({"type": "message_stop"}) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index 6ae61e15953..87a75dae95e 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -15,7 +15,9 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -95,6 +97,32 @@ def _count_error(model_to_use: str, request_model: str, message: str, status_cod ) +async def _sent_messages( + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + system: object, + messages_format: TokenCountMessagesFormat, + context_management: CountContextManagement | None, +) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError: + match messages_format: + case "openai": + return messages + case "anthropic": + try: + return await count_view_messages( + model=model, + messages=messages, + tools=tools, + system=system, + context_management=context_management, + ) + except AnthropicContextManagementError as error: + return error + case _: + assert_never(messages_format) + + class OpenAITokenCounter(BaseTokenCounter): """Token counter implementation for OpenAI provider using the Responses API.""" @@ -175,13 +203,28 @@ class OpenAITokenCounter(BaseTokenCounter): rejected_before_provider_call=True, ) + sent_messages: Final = await _sent_messages( + model_to_use, messages, tools, system, messages_format, context_management + ) + if isinstance(sent_messages, AnthropicContextManagementError): + return TokenCountResponse( + total_tokens=0, + request_model=request_model, + model_used=model_to_use, + tokenizer_type="openai_api", + error=True, + error_message=sent_messages.message, + status_code=sent_messages.status_code, + rejected_before_provider_call=True, + ) + resolved: Final = _count_credential(deployment, request_model or model_to_use) if isinstance(resolved, OpenAIError): return _count_error(model_to_use, request_model, resolved.message, resolved.status_code) input_items, instructions, counted_tools, text = self._responses_request_parts( model_to_use=model_to_use, - messages=messages, + messages=sent_messages, tools=tools, system=system, messages_format=messages_format, diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d97970753f8..f03c7421899 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -523,6 +523,43 @@ def test_token_counter_skips_non_mapping_tools(): assert mixed_tokens == valid_tokens +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [({"type": "array"}, {"type": "array", "items": {}}), (True, {})], + ids=["array_without_items", "boolean_subschema"], +) +def test_token_counter_counts_json_schema_shorthands_like_their_explicit_form(shorthand: object, explicit: object) -> None: + assert _count_with_property_schema(shorthand) == _count_with_property_schema(explicit) + + +@pytest.mark.parametrize( + ("malformed", "without_keyword"), + [ + ( + {"type": "object", "properties": {"id": {"type": "string"}}, "required": True}, + {"type": "object", "properties": {"id": {"type": "string"}}}, + ), + ({"type": "object", "properties": [{"id": {"type": "string"}}]}, {"type": "object"}), + ({"type": "string", "enum": None}, {"type": "string"}), + ({"type": "integer", "enum": 3}, {"type": "integer"}), + ], + ids=["required_true", "properties_list", "string_enum_null", "integer_enum_number"], +) +def test_token_counter_counts_a_malformed_schema_keyword_as_absent(malformed: object, without_keyword: object) -> None: + assert _count_with_property_schema(malformed) == _count_with_property_schema(without_keyword) + + +def _count_with_property_schema(property_schema: object) -> int: + tool: Final = { + "type": "function", + "function": { + "name": "read_files", + "parameters": {"type": "object", "properties": {"paths": property_schema}}, + }, + } + return token_counter_new(model="gpt-4o", messages=[{"role": "user", "content": "Read the config files"}], tools=[tool]) + + _WEATHER_FUNCTION: Final = { "name": "get_weather", "description": "Get the current weather for a city", @@ -2132,6 +2169,20 @@ def test_token_counter_counts_anthropic_search_result_text() -> None: ) +def test_token_counter_counts_a_search_result_without_a_title() -> None: + from litellm.litellm_core_utils.token_counter import _count_content_list + + search_result: Final = { + "type": "search_result", + "source": "https://example.invalid/notes", + "content": [{"type": "text", "text": "The code word is 7f3a."}], + } + + assert _count_content_list(len, [search_result], False, None) == len( + "https://example.invalid/notesThe code word is 7f3a." + ) + + @pytest.mark.parametrize("nested", (False, True)) def test_unknown_content_type_remains_a_value_error_with_an_optional_fallback(nested: bool) -> None: from litellm.litellm_core_utils.token_counter import _count_content_list diff --git a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index f03ffb25b2b..4cbf4cfa5f6 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -217,6 +217,106 @@ def test_cleared_input_tokens_is_nonnegative(): assert applied["cleared_input_tokens"] >= 0 +SERVER_TOOL_TURN: Final = { + "role": "assistant", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "Mumbai rain"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300}], + }, + ], +} + + +def _clear_above( + messages: list[dict[str, object]], threshold: int +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": threshold}, + "keep": {"type": "tool_uses", "value": 1}, + }, + ) + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends() -> None: + history: Final = _make_history(n_pairs=3) + sent_tokens: Final = litellm.token_counter(model=MODEL, messages=history) + turn: Final = SERVER_TOOL_TURN + messages: Final = [history[0], turn, *history[1:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert applied["cleared_tool_uses"] == 2 + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + assert cleared[1] == turn + + +def _first_result_with(blocks: list[dict[str, object]]) -> dict[str, object]: + return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_00", "content": blocks}]} + + +def test_an_input_tokens_trigger_skips_blocks_no_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + sent_tokens: Final = litellm.token_counter( + model=MODEL, messages=[*history[:2], _first_result_with([result_text]), *history[3:]] + ) + messages: Final = [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + + untouched, not_applied = _clear_above(messages, sent_tokens + 100) + cleared, applied = _clear_above(messages, sent_tokens - 100) + + assert not_applied is None + assert untouched == messages + assert applied is not None + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + +def _clear_at_least_one_token( + messages: list[dict[str, object]], +) -> tuple[list[dict[str, object]], AppliedEdit | None]: + return apply_clear_tool_uses_20250919( + model=MODEL, + messages=messages, + tools=None, + system=None, + edit_spec={ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 0}, + "keep": {"type": "tool_uses", "value": 1}, + "clear_at_least": {"type": "input_tokens", "value": 1}, + }, + ) + + +def test_clear_at_least_frees_only_blocks_an_adapter_sends_inside_a_tool_result() -> None: + history: Final = _make_history(n_pairs=3) + result_text: Final = {"type": "text", "text": "Result 0: " + "x" * 200} + search_result: Final = {"type": "web_search_result", "url": "https://example.com", "title": "rain " * 300} + + _, sent_only = _clear_at_least_one_token([*history[:2], _first_result_with([result_text]), *history[3:]]) + cleared, applied = _clear_at_least_one_token( + [*history[:2], _first_result_with([result_text, search_result]), *history[3:]] + ) + + assert applied is not None + assert applied == sent_only + assert _cleared_ids(cleared) == {"toolu_00", "toolu_01"} + + def _blocks_of(messages: list[dict[str, object]], block_type: str) -> tuple[dict[str, object], ...]: contents: Final = (message["content"] for message in messages) blocks: Final = chain.from_iterable(content for content in contents if isinstance(content, list)) @@ -559,6 +659,9 @@ async def _malformed_spec_error(edits: list[dict[str, object]]) -> AnthropicCont ("exclude_tools", "get_weather"), ("exclude_tools", [1]), ("clear_tool_inputs", "get_weather"), + ("trigger", "soon"), + ("trigger", [1]), + ("keep", 5), ], ) async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, value: object) -> None: @@ -576,12 +679,15 @@ async def test_malformed_fields_are_a_400_even_below_the_trigger(field: str, val ("clear_at_least", 1200, "clear_at_least: Input does not match the expected shape."), ("clear_at_least", {"type": "input_tokens", "value": "1200"}, "value: Input should be a valid integer"), ("clear_tool_inputs", 5, "clear_tool_inputs.bool: Input should be a valid boolean, unable to interpret input"), + ("trigger", "soon", "trigger: Input should be an object"), + ("keep", 5, "keep: Input should be an object"), ], ) async def test_malformed_field_400_names_the_edit_and_field_as_the_api_does( field: str, value: object, message: str ) -> None: - # api.anthropic.com, measured 2026-10-19, answers these four with these messages for the edit at index 1. + # api.anthropic.com answers these with these messages for the edit at index 1 (trigger and keep measured + # 2026-10-28, the rest 2026-10-19). error = await _malformed_spec_error( [{"type": "clear_thinking_20251015"}, {"type": "clear_tool_uses_20250919", field: value}] ) @@ -625,6 +731,28 @@ async def test_drop_params_drops_only_the_malformed_field(field: str, value: obj assert _tool_inputs(result.messages) == {"toolu_00": {}, "toolu_01": {}, "toolu_02": {"location": "City2"}} +@pytest.mark.parametrize(("field", "value"), [("trigger", "soon"), ("keep", 5)]) +async def test_drop_params_treats_a_malformed_trigger_or_keep_as_never_sent(field: str, value: object) -> None: + async def run(edit: dict[str, object], drop_invalid_edits: bool) -> PolyfillResult | None: + return await _run_polyfill_if_enabled( + model=MODEL, + messages=_make_history(n_pairs=5), + tools=None, + system=None, + context_management_spec={"edits": [edit]}, + litellm_metadata={}, + additional_drop_params=None, + llm_router=None, + drop_invalid_edits=drop_invalid_edits, + ) + + malformed: Final = {**_VALID_EDIT, field: value} + with pytest.raises(AnthropicContextManagementError, match=rf"\.{field}: Input should be an object$"): + await run(malformed, drop_invalid_edits=False) + never_sent: Final = {key: spec for key, spec in _VALID_EDIT.items() if key != field} + assert await run(malformed, drop_invalid_edits=True) == await run(never_sent, drop_invalid_edits=True) + + async def test_malformed_edit_is_rejected_before_a_compact_summary_is_paid_for( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/anthropic/messages/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py index dff931a6074..32899a8e3c8 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.context_management.editors.compact import ( _augment_system_with_summary, _extract_summary_text, _select_last_user_question, - _slice_around_compaction_block, + slice_around_compaction_block, _strip_compaction_blocks, apply_client_compaction_block_history, apply_compact_20260112, @@ -153,7 +153,7 @@ def test_applied_edits_for_response_includes_compact_when_block_present(): def test_slice_around_compaction_block_found(): messages = _messages_with_compaction("my summary") - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is not None assert block["type"] == "compaction" assert block["content"] == "my summary" @@ -164,7 +164,7 @@ def test_slice_around_compaction_block_found(): def test_slice_around_compaction_block_not_found(): messages = _simple_messages() - sliced, block = _slice_around_compaction_block(messages) + sliced, block = slice_around_compaction_block(messages) assert block is None assert sliced is messages # same object, no copy diff --git a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index fabb05c2e61..e55af602013 100644 --- a/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -5,6 +5,7 @@ Unit tests for the context_management polyfill dispatcher. from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) +from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts MODEL = "xai/grok-4" @@ -160,3 +161,57 @@ async def test_sync_editor_counts_tokens_off_the_event_loop(): assert result.messages == messages assert_loop_stayed_free(took, lags) + + +def _history_with_null_text_beside_a_short_first_result(): + return [ + {"role": "user", "content": "Hi"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t0", "name": "f", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t0", + "content": [{"type": "text", "text": "ok"}, {"type": "text", "text": None}], + } + ], + }, + *_history_with_two_tool_pairs()[1:], + ] + + +async def test_drop_invalid_edits_clears_as_if_malformed_tool_result_text_was_never_sent(): + spec = { + "edits": [ + { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 2}, + "keep": {"type": "tool_uses", "value": 1}, + } + ] + } + history = _history_with_null_text_beside_a_short_first_result() + + from_raw = await apply_context_management( + model=MODEL, + messages=history, + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + from_sent = await apply_context_management( + model=MODEL, + messages=list(drop_non_string_tool_result_texts(history, True)), + tools=None, + system=None, + context_management_spec=spec, + drop_invalid_edits=True, + ) + + assert from_raw.applied_edits == from_sent.applied_edits + assert from_raw.messages == from_sent.messages diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index bba5413fbdf..382917fc756 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -2,7 +2,8 @@ import datetime import json import os import sys -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -761,3 +762,325 @@ async def test_recomputed_cost_of_a_served_bridge_response_prices_the_reasoning_ assert logging_obj.cost_breakdown is not None assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(expected_output) assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(expected_reasoning) + + +TEXT_RESPONSE: Final = { + "id": "resp_cleared", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} +CLEAR_EDIT: Final = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} +COMPACT_EDIT: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}} + + +def _read(i: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + { + "role": "assistant", + "content": [{"type": "tool_use", "id": f"toolu_{i}", "name": "Bash", "input": {"command": f"cat {i}"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{i}", "content": f"log {i} " * 40}], + }, + ) + + +def _three_reads() -> list[dict[str, object]]: + return [{"role": "user", "content": "Read the logs."}, *chain.from_iterable(_read(i) for i in range(3))] + + +async def _cleared_bridge( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + *, + stream: bool, + edits: list[dict[str, object]], + drop_params: bool = False, + system: object = None, +) -> tuple[object, Mapping[str, object]]: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses") + if stream: + upstream.respond(status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + else: + upstream.respond(status_code=200, json=TEXT_RESPONSE) + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + context_management={"edits": edits}, + drop_params=drop_params, + system=system, + ) + body: Final = json.loads(upstream.calls.last.request.content) + if not stream: + return response, body + return [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in response], body + + +@pytest.mark.asyncio +async def test_clear_tool_uses_clears_old_results_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT)]) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert "context_management" not in body + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_streamed_clear_tool_uses_reports_its_edits_on_the_final_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + events, _ = await _cleared_bridge(respx_mock, monkeypatch, stream=True, edits=[dict(CLEAR_EDIT)]) + + assert isinstance(events, list) + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_compact_stays_native_while_clear_tool_uses_runs_in_the_gateway( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[dict(CLEAR_EDIT), dict(COMPACT_EDIT)] + ) + + assert body["context_management"] == [{"type": "compaction", "compact_threshold": 150000}] + assert isinstance(served, dict) + assert [edit["type"] for edit in served["context_management"]["applied_edits"]] == ["clear_tool_uses_20250919"] + + +@pytest.mark.asyncio +async def test_a_malformed_clear_tool_uses_edit_is_a_400_before_the_responses_api_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError + + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + with pytest.raises(AnthropicContextManagementError) as raised: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management={"edits": [{**CLEAR_EDIT, "exclude_tools": "Bash"}]}, + ) + assert raised.value.status_code == 400 + assert not upstream.called + + +@pytest.mark.asyncio +async def test_a_malformed_text_in_a_result_clearing_would_drop_is_still_a_400( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + reads: Final = _three_reads() + malformed_oldest_result: Final = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_0", "content": [{"type": "text", "text": None}]}], + } + with pytest.raises(litellm.BadRequestError) as rejected: + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=[*reads[:2], malformed_oldest_result, *reads[3:]], + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + assert rejected.value.status_code == 400 + assert "tool_result toolu_0 content.0.text: Input should be a valid string" in rejected.value.message + assert not upstream.called + + +def test_the_sync_path_clears_old_results_too(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + + served: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + + body: Final = json.loads(upstream.calls.last.request.content) + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +@pytest.mark.asyncio +async def test_an_openai_style_context_management_list_is_still_not_sent_to_the_responses_api( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, json=TEXT_RESPONSE + ) + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + + served: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + custom_llm_provider="openai", + context_management=[{"type": "compaction", "compact_threshold": 500}], + ) + + assert "context_management" not in json.loads(upstream.calls.last.request.content) + assert isinstance(served, dict) + assert "context_management" not in served + + +@pytest.mark.asyncio +async def test_under_drop_params_a_malformed_edit_still_clears_without_its_bad_field( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + served, body = await _cleared_bridge( + respx_mock, monkeypatch, stream=False, edits=[{**CLEAR_EDIT, "exclude_tools": "Bash"}], drop_params=True + ) + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + assert [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] == [ + True, + True, + False, + ] + assert isinstance(served, dict) + assert served["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +def _cleared_outputs(body: Mapping[str, object]) -> list[bool]: + from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER + + outputs: Final = { + item["call_id"]: item["output"] for item in body["input"] if item.get("type") == "function_call_output" + } + return [CLEARED_TOOL_RESULT_PLACEHOLDER in json.dumps(outputs[f"toolu_{i}"]) for i in range(3)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [7, {"type": "text", "text": "Be brief."}, ["Be brief."], ["Be brief.", {"type": "text", "text": "Be kind."}]], + ids=["number", "object", "strings", "mixed_list"], +) +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_clear_tool_uses_still_clears_when_the_system_prompt_is_not_text_or_a_block_list( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, stream: bool +) -> None: + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=stream, edits=[dict(CLEAR_EDIT)], system=system) + + assert _cleared_outputs(body) == [True, True, False] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("system", "expect_cleared"), + [ + (["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], [True, True, False]), + (["Follow the policy. " * 1500], [False, False, False]), + ], + ids=["text_block_counts", "stray_string_does_not"], +) +async def test_input_tokens_trigger_counts_the_text_blocks_of_a_system_list_with_stray_items( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, system: object, expect_cleared: list[bool] +) -> None: + trigger_between_messages_and_system: Final = litellm.token_counter(model="gpt-4o", messages=_three_reads()) + 1000 + edit: Final = {**CLEAR_EDIT, "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}} + + _, body = await _cleared_bridge(respx_mock, monkeypatch, stream=False, edits=[edit], system=system) + + assert _cleared_outputs(body) == expect_cleared + + +@pytest.mark.asyncio +async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-clear-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").respond( + status_code=200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY + ) + + sse: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=_three_reads(), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=True, + custom_llm_provider="openai", + context_management={"edits": [dict(CLEAR_EDIT)]}, + ) + events: Final = [ + json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], sse) + ] + message_delta: Final = next(event for event in events if event["type"] == "message_delta") + assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 diff --git a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py index 17c1b35e211..29c25c5fdbf 100644 --- a/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py +++ b/tests/unit/llms/openai/responses/count_tokens/test_token_counter.py @@ -2,6 +2,7 @@ import copy import json import operator from collections.abc import Mapping +from itertools import chain from typing import Final import httpx @@ -11,9 +12,10 @@ from pydantic import JsonValue, ValidationError import litellm from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.llms.anthropic.messages.context_management.constants import CLEARED_TOOL_RESULT_PLACEHOLDER from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse COUNT_URL: Final = "https://api.openai.com/v1/responses/input_tokens" @@ -484,6 +486,122 @@ async def test_anthropic_format_tool_result_text_block_without_text_still_counts assert {"type": "function_call_output", "call_id": "toolu_log", "output": "build ok\n\ndone"} in sent["input"] +def _tool_pair(index: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + {"role": "assistant", "content": [{"type": "tool_use", "id": f"toolu_{index}", "name": "log", "input": {}}]}, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": f"toolu_{index}", "content": f"log {index} " * 40}], + }, + ) + + +TOOL_PAIRS: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "Compare the logs."}, + *chain.from_iterable(_tool_pair(index) for index in range(3)), +] + + +async def _count_with_edit( + monkeypatch: pytest.MonkeyPatch, edit: dict[str, JsonValue], drop_invalid_edits: bool, system: object = None +) -> tuple[TokenCountResponse | None, respx.Route]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 9})) + result: Final = await OpenAITokenCounter().count_tokens( + model_to_use="gpt-4o", + messages=copy.deepcopy(TOOL_PAIRS), + contents=None, + deployment={"litellm_params": {"api_key": "sk-test"}}, + request_model="openai/gpt-4o", + system=system, + messages_format="anthropic", + context_management=CountContextManagement(spec={"edits": [edit]}, drop_invalid_edits=drop_invalid_edits), + ) + return result, route + + +def _sent_outputs(route: respx.Route) -> dict[str, str]: + sent: Final = json.loads(route.calls.last.request.content) + return {item["call_id"]: item["output"] for item in sent["input"] if item["type"] == "function_call_output"} + + +CLEAR_EDIT: Final[dict[str, JsonValue]] = { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "tool_uses", "value": 1}, + "keep": {"type": "tool_uses", "value": 1}, +} + + +@pytest.mark.asyncio +async def test_anthropic_format_counts_the_history_the_clear_edit_leaves_for_inference( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, CLEAR_EDIT, drop_invalid_edits=False) + + assert result is not None + assert (result.error, result.total_tokens) == (False, 9) + assert _sent_outputs(route) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "system", + [ + "Follow the policy. " * 1500, + [{"type": "text", "text": "Follow the policy. " * 1500}], + ["Stray.", {"type": "text", "text": "Follow the policy. " * 1500}], + ], + ids=["string", "blocks", "blocks_beside_a_stray_string"], +) +async def test_anthropic_format_input_tokens_trigger_counts_the_system_prompt_like_inference( + monkeypatch: pytest.MonkeyPatch, system: object +) -> None: + message_tokens: Final = litellm.token_counter(model="gpt-4o", messages=TOOL_PAIRS) + trigger_between_messages_and_system: Final = message_tokens + 1000 + edit: Final[dict[str, JsonValue]] = { + **CLEAR_EDIT, + "trigger": {"type": "input_tokens", "value": trigger_between_messages_and_system}, + } + + _, without_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False) + _, with_system = await _count_with_edit(monkeypatch, edit, drop_invalid_edits=False, system=system) + + assert _sent_outputs(without_system)["toolu_0"] == "log 0 " * 40 + assert _sent_outputs(with_system) == { + "toolu_0": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_1": CLEARED_TOOL_RESULT_PLACEHOLDER, + "toolu_2": "log 2 " * 40, + } + + +@pytest.mark.asyncio +async def test_anthropic_format_malformed_clear_edit_answers_400_without_calling_openai( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=False) + + assert route.called is False + assert result is not None + assert (result.error, result.status_code) == (True, 400) + assert "exclude_tools" in str(result.error_message) + + +@pytest.mark.asyncio +async def test_anthropic_format_drop_params_counts_the_clear_without_its_malformed_field( + monkeypatch: pytest.MonkeyPatch, +) -> None: + result, route = await _count_with_edit(monkeypatch, {**CLEAR_EDIT, "exclude_tools": "log"}, drop_invalid_edits=True) + + assert result is not None + assert result.error is False + assert _sent_outputs(route)["toolu_0"] == CLEARED_TOOL_RESULT_PLACEHOLDER + + @pytest.mark.asyncio async def test_count_uses_the_global_openai_key_chat_would_use(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index b2a0b735b0f..38eb87debfd 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -649,7 +649,7 @@ async def test_anthropic_view_counts_the_unedited_body_without_an_error_log_when assert not any(record.levelno >= logging.ERROR for record in caplog.records) -async def test_anthropic_view_counts_the_turns_after_a_compaction_block_when_an_edit_before_compact_fails() -> None: +async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_when_clear_runs_before_compact() -> None: clear_before_compact: Final[JsonValue] = { "edits": [ { diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 43c017468ff..c182cd1ec75 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2064,22 +2064,6 @@ async def test_framing_keeps_the_400_for_blocks_the_provider_counter_cannot_conv }, id="web_search_blocks", ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [{"name": "search", "input_schema": {"type": "object", "properties": {"query": True}}}], - }, - id="boolean_subschema_tool", - ), - pytest.param( - { - "messages": [{"role": "user", "content": "What is new?"}], - "tools": [ - {"name": "search", "input_schema": {"type": "object", "properties": {"tags": {"type": "array"}}}} - ], - }, - id="array_schema_without_items", - ), pytest.param( {"messages": [], "contents": [{"role": "user", "parts": [{"text": "What is new?"}]}]}, id="google_contents", @@ -2138,6 +2122,44 @@ async def test_framed_estimate_that_cannot_read_the_body_answers_the_provider_er assert readable.count_source == "local_estimate" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("shorthand", "explicit"), + [ + pytest.param({"type": "array"}, {"type": "array", "items": {}}, id="array_schema_without_items"), + pytest.param(True, {}, id="boolean_subschema"), + ], +) +async def test_framed_estimate_counts_a_json_schema_shorthand_like_its_explicit_form( + monkeypatch: pytest.MonkeyPatch, shorthand: object, explicit: dict[str, object] +): + counter: Final = _ScriptedCounter({"us.moonshotai.kimi-k3": _provider_reply("kimi", status_code=400)}) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router(model_list=[{"model_name": "kimi", "litellm_params": {"model": "bedrock/us.moonshotai.kimi-k3"}}]), + ) + monkeypatch.setattr( + litellm.proxy.proxy_server, "_get_provider_token_counter", lambda deployment, model: (counter, model, "bedrock") + ) + + async def count(schema: object) -> TokenCountResponse: + return await token_counter( + request=TokenCountRequest( + model="kimi", + messages=[{"role": "user", "content": "What is new?"}], + tools=[{"name": "search", "input_schema": {"type": "object", "properties": {"tags": schema}}}], + ), + call_endpoint=True, + ) + + from_shorthand: Final = await count(shorthand) + from_explicit: Final = await count(explicit) + + assert from_shorthand.count_source == "local_estimate" + assert from_shorthand.total_tokens == from_explicit.total_tokens + + @pytest.mark.asyncio @pytest.mark.parametrize( "contents", From a6fda5a17e1b330c42890ab6fd3cf1c8bacf77c2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 17:45:37 -0700 Subject: [PATCH 03/25] bump: litellm-enterprise 0.1.80 -> 0.1.81, litellm-proxy-extras 0.4.111 -> 0.4.112 (#49149) --- enterprise/pyproject.toml | 4 ++-- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 4653f919e17..7db2de1f874 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.80" +version = "0.1.81" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.80" +version = "0.1.81" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 6de44be0c12..a12e1e27cfc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.111" +version = "0.4.112" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.111" +version = "0.4.112" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 44dfa498904..6e4732b4f7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,8 +84,8 @@ proxy = [ "mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.111", - "litellm-enterprise==0.1.80", + "litellm-proxy-extras==0.4.112", + "litellm-enterprise==0.1.81", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/uv.lock b/uv.lock index 5be97af9ed2..b3e8ddc492c 100644 --- a/uv.lock +++ b/uv.lock @@ -5478,12 +5478,12 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.80" +version = "0.1.81" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.111" +version = "0.4.112" source = { editable = "litellm-proxy-extras" } dependencies = [ { name = "psycopg" }, From bbc68e21f92f017853859d76fb43ca5e4641a2f2 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:48:38 -0700 Subject: [PATCH 04/25] feat(dashscope): add image edits and text-to-speech (#48998) * feat(dashscope): add image edits and text-to-speech * fix(dashscope): bill reported speech characters and input images, follow the chat region for edits, keep the call timeout on the audio download * fix(dashscope): bill deployment image prices as main does and route edits to the host speech uses A deployment that sets its own input_cost_per_image, output_cost_per_image or input_cost_per_pixel bills through the default image calculator again, so an override is never read as a per-input-image price and pixel pricing on qwen-image-3.0-pro no longer raises. Edits now resolve the caller's api_base or DASHSCOPE_API_BASE through the same API-root helper speech uses, on any host, with DASHSCOPE_API_BASE_IMAGE still winning. An image response whose usage is null or not an object keeps its images instead of raising. * fix(images): bill a deployment's own prices on image edits through the proxy The proxy builds the logging object before the router picks a deployment, so image_edit never handed the deployment's price fields to its cost calculation and billed the cost-map price instead. Pass the custom pricing params with the pre-call update, as the Responses API does. * fix(images): bill per-pixel image prices at a DashScope W*H size DashScope's mapped size reads 1328*1328, which the cost size reader rejected, so a deployment's input_cost_per_pixel billed every DashScope generation or edit at the 1024x1024 default. * fix(dashscope): bill image prices by their meaning, report the rendered size, prefer the deployment edit base, fetch audio over https * fix(dashscope): read the rendered image size from output_width and output_height * test(images): bill a fal edit at its deployment's own image price * fix(dashscope): keep the qwen-image-3.0 input-image fee out of the shared cost map row Released proxies read the cost map from main and bill input_cost_per_image as the flat price of each DashScope image, so the row's $0.003 input fee billed every qwen-image-3.0 image there at a tenth of its $0.03 price. A deployment can still set input_cost_per_image to bill DashScope's input images --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 10 +- litellm/images/main.py | 3 +- .../litellm_core_utils/llm_cost_calc/utils.py | 24 +- .../base_llm/text_to_speech/transformation.py | 12 + litellm/llms/custom_httpx/llm_http_handler.py | 2 +- litellm/llms/dashscope/common_utils.py | 22 +- litellm/llms/dashscope/image_edit/__init__.py | 0 .../dashscope/image_edit/transformation.py | 160 +++++++ .../image_generation/cost_calculator.py | 19 + .../image_generation/transformation.py | 94 ++-- .../llms/dashscope/text_to_speech/__init__.py | 0 .../text_to_speech/transformation.py | 297 +++++++++++++ litellm/main.py | 9 +- ...odel_prices_and_context_window_backup.json | 67 ++- .../provider_endpoints_support_backup.json | 4 +- litellm/utils.py | 57 ++- model_prices_and_context_window.json | 67 ++- provider_endpoints_support.json | 4 +- ...st_dashscope_image_edit_and_speech_wire.py | 259 +++++++++++ tests/unit/images/test_image_edit.py | 119 +++++ ...est_dashscope_image_edit_transformation.py | 358 +++++++++++++++ ...dashscope_text_to_speech_transformation.py | 415 ++++++++++++++++++ .../test_text_to_speech_transformation.py | 38 ++ .../test_get_complete_url_trailing_slash.py | 1 + tests/unit/test_dashscope_image_generation.py | 61 ++- 25 files changed, 2009 insertions(+), 93 deletions(-) create mode 100644 litellm/llms/dashscope/image_edit/__init__.py create mode 100644 litellm/llms/dashscope/image_edit/transformation.py create mode 100644 litellm/llms/dashscope/image_generation/cost_calculator.py create mode 100644 litellm/llms/dashscope/text_to_speech/__init__.py create mode 100644 litellm/llms/dashscope/text_to_speech/transformation.py create mode 100644 tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py create mode 100644 tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py create mode 100644 tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a4a9a5bee39..defe1e63b86 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1887,7 +1887,10 @@ def completion_cost( video_resolution=video_resolution, ) elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = litellm.utils.count_characters(text=prompt) + prompt_characters = _speech_billed_characters( + completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, + prompt, + ) elif call_type in _TRANSCRIPTION_CALL_TYPES: # Check _hidden_params first (duration stored there to # avoid polluting the response body), then fall back to @@ -2366,6 +2369,11 @@ def response_cost_calculator( raise e +def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: + reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None + return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) + + def _video_input_image_cost( model: str, custom_llm_provider: str | None, diff --git a/litellm/images/main.py b/litellm/images/main.py index 4aad5210028..165543aadba 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -53,7 +53,7 @@ from litellm.main import ( from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import ImageGenerationRequestQuality -from litellm.types.router import GenericLiteLLMParams +from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, LlmProviders, @@ -1010,6 +1010,7 @@ def image_edit( optional_params=dict(image_edit_request_params), litellm_params={ **image_edit_request_params, + **litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True), "litellm_call_id": litellm_call_id, "model_info": model_info, "vertex_location": litellm_params.vertex_location, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..404dfab0a6d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( "output_cost_per_image_token", ) + +def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool: + return pricing is not None and ( + "input_cost_per_pixel" in pricing + or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing) + ) + + # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead # of being rebuilt for every model_info key on every call. Longest-first so a @@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None: return value if isinstance(value, int) else None -_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+") +_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+") def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None: @@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None: value: Final = _requested_image_param(optional_params, "size") - return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None + return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None def get_web_search_requests(server_tool_use: object) -> int | None: @@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol): def _model_priced_image_cost_calculator( - custom_llm_provider: str | None, model: str + custom_llm_provider: str | None, model: str, pricing: ModelInfo | None ) -> _ModelPricedImageCostCalculator | None: match custom_llm_provider: case litellm.LlmProviders.RECRAFT.value: @@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator( from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None + case litellm.LlmProviders.DASHSCOPE.value: + if _deployment_prices_images_without_an_output_price(pricing): + return None + from litellm.llms.dashscope.image_generation.cost_calculator import ( + cost_calculator as dashscope_image_cost_calculator, + ) + + return dashscope_image_cost_calculator case _: return None @@ -2178,7 +2194,7 @@ class CostCalculatorUtils: ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) - model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model) + model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing) if model_priced_calculator is not None: return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing) diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py index be49098b62b..6df83966bf8 100644 --- a/litellm/llms/base_llm/text_to_speech/transformation.py +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC): Transform provider response to standard format """ + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + return self.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + ) + def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1f25235a651..c92315e53b3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -12849,7 +12849,7 @@ class BaseLLMHTTPHandler: provider_config=text_to_speech_provider_config, ) - speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response( + speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response( model=model, raw_response=response, logging_obj=logging_obj, diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 5ad2aa17a3a..d3e1fb8eada 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic" DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1" DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1" DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks" -DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com" _STANDARD_DASHSCOPE_HOSTS: Final = frozenset( @@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool: return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/") +def dashscope_api_root(api_base: str) -> str: + endpoints: Final = resolve_dashscope_endpoints(api_base) + if endpoints is not None: + return endpoints.origin + trimmed: Final = api_base.rstrip("/") + if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH): + return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH) + root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH) + return root if separator else trimmed + + +def is_aliyun_host(host: str | None) -> bool: + return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com")) + + def resolve_dashscope_openai_api_base( api_base: str | None, env_var: str = "DASHSCOPE_API_BASE", @@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None: if api_base is None: return None parsed: Final = urlparse(api_base) - host: Final = parsed.hostname or "" - on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com") - if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: + if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: return None return f"{parsed.scheme}://{parsed.netloc}" diff --git a/litellm/llms/dashscope/image_edit/__init__.py b/litellm/llms/dashscope/image_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/image_edit/transformation.py b/litellm/llms/dashscope/image_edit/transformation.py new file mode 100644 index 00000000000..0d34017a321 --- /dev/null +++ b/litellm/llms/dashscope/image_edit/transformation.py @@ -0,0 +1,160 @@ +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +import httpx +from httpx._types import RequestFiles +from pydantic import TypeAdapter +from typing_extensions import override + +import litellm +from litellm.images.utils import image_to_data_url +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_CHAT_COMPATIBLE_PATH, + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + dashscope_api_root, + is_dashscope_native_endpoint_url, + resolve_dashscope_endpoints, +) +from litellm.llms.dashscope.image_generation.transformation import ( + DEFAULT_API_BASE, + OPENAI_TO_DASHSCOPE_SIZE, + DashScopeImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +MAX_INPUT_IMAGES: Final = 3 +DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed") + +_IMAGES: Final = TypeAdapter(tuple[object, ...]) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + +def _deployment_edit_url(api_base: str) -> str: + derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and ( + resolve_dashscope_endpoints(api_base) is not None + or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH) + ) + return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base + + +class DashScopeImageEditConfig(BaseImageEditConfig): + def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None: + super().__init__() + self._generation_config: Final = generation_config or DashScopeImageGenerationConfig() + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + return list(DASHSCOPE_IMAGE_EDIT_PARAMS) + + @override + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: mirrors override contract + return { + key: _dashscope_size(value) if key == "size" else value + for key, value in image_edit_optional_params.items() + if key in DASHSCOPE_IMAGE_EDIT_PARAMS + } + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return self._generation_config.validate_environment( + headers=dict(headers), + model=model, + messages=[], + optional_params={}, + litellm_params=dict(litellm_params or {}), + api_key=api_key, + api_base=api_base, + ) + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + if api_base is not None: + return _deployment_edit_url(api_base) + env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE") + derived: Final = ( + f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None + ) + return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE + + @override + def use_multipart_form_data(self) -> bool: + return False + + @override + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: object, + image_edit_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract + candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,) + images: Final = tuple(candidate for candidate in candidates if candidate is not None) + if len(images) > MAX_INPUT_IMAGES: + raise litellm.BadRequestError( + message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}", + model=model, + llm_provider="dashscope", + ) + prompt_content: Final = ({"text": prompt},) if prompt is not None else () + return { + "model": model, + "input": { + "messages": [ + { + "role": "user", + "content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content], + } + ] + }, + "parameters": dict(image_edit_optional_request_params), + }, () + + @override + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> ImageResponse: + return self._generation_config.transform_image_generation_response( + model=model, + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) diff --git a/litellm/llms/dashscope/image_generation/cost_calculator.py b/litellm/llms/dashscope/image_generation/cost_calculator.py new file mode 100644 index 00000000000..f699ef7e1c5 --- /dev/null +++ b/litellm/llms/dashscope/image_generation/cost_calculator.py @@ -0,0 +1,19 @@ +from typing import Final + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo + + +def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: + prices: Final = resolve_image_model_info( + model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info + ) + output_cost_per_image: Final = prices.get("output_cost_per_image") + if output_cost_per_image is None: + raise ValueError(f"No output_cost_per_image found for dashscope/{model}") + input_images: Final = image_response.hidden_params.get("billable_input_image_count") + input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * ( + input_images if isinstance(input_images, int) else 0 + ) + return output_cost_per_image * len(image_response.data or ()) + input_image_cost diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index 2f528b5a2c3..d1b5a7b40e7 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -23,7 +23,7 @@ Response format: } """ -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Maps OpenAI size strings (WxH) to DashScope size strings (W*H) -OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { +OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = { "256x256": "256*256", "512x512": "512*512", "1024x1024": "1024*1024", @@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { } +def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]: + contents: Final = _JSON_OBJECTS.validate_python(message.get("content", [])) + return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url) + + +def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]: + output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) + choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) + for choice in choices: + yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {}))) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + class DashScopeImageGenerationConfig(BaseImageGenerationConfig): """ Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro, @@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict: + ) -> dict[str, object]: supported_params: Final = self.get_supported_openai_params(model) - mapped: Final[dict] = {} - for k, v in non_default_params.items(): - if k in optional_params: - continue - if k not in supported_params: - continue - if k == "size": - # Convert "WxH" → "W*H" - mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*")) - else: - mapped[k] = v - return mapped + requested: Final = { + k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params + } + return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()} def _resolve_api_key(self, api_key: str | None) -> str: resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY") @@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], stream: bool | None = None, ) -> str: image_env_api_base: Final = self._image_api_base_env() @@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}" headers["Content-Type"] = "application/json" return headers @@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): raw_response: httpx.Response, model_response: ImageResponse, logging_obj: LiteLLMLoggingObj, - request_data: dict, - optional_params: dict, - litellm_params: dict, + request_data: dict[str, object], + optional_params: dict[str, object], + litellm_params: dict[str, object], encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, @@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): headers=raw_response.headers, ) - if not model_response.data: - model_response.data = [] - - output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) - choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) - for choice in choices: - message = _JSON_OBJECT.validate_python(choice.get("message", {})) - content_list = _JSON_OBJECTS.validate_python(message.get("content", [])) - for content_item in content_list: - image_url = content_item.get("image") - if image_url: - model_response.data.append(ImageObject(url=image_url)) - - return model_response + images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))] + usage: Final = response_object.get("usage") + usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {} + width: Final = usage_fields.get("output_width", usage_fields.get("width")) + height: Final = usage_fields.get("output_height", usage_fields.get("height")) + image_response: Final = model_response.model_copy( + update={ + "data": images, + **({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}), + } + ) + input_image_count: Final = usage_fields.get("input_image_count") + if isinstance(input_image_count, int) and input_image_count > 0: + image_response.hidden_params = { + **model_response.hidden_params, + "billable_input_image_count": input_image_count, + } + return image_response diff --git a/litellm/llms/dashscope/text_to_speech/__init__.py b/litellm/llms/dashscope/text_to_speech/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/text_to_speech/transformation.py b/litellm/llms/dashscope/text_to_speech/transformation.py new file mode 100644 index 00000000000..053682040bd --- /dev/null +++ b/litellm/llms/dashscope/text_to_speech/transformation.py @@ -0,0 +1,297 @@ +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from itertools import chain +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import override + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped + get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped +) +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + DashScopeError, + dashscope_api_root, + is_aliyun_host, + missing_dashscope_family_key_message, + resolve_dashscope_family_api_key, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com" +_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-" +_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None]) + + +@dataclass(frozen=True, slots=True) +class _TextToSpeechFamily: + path: str + response_formats: frozenset[str] + sends_format: bool + takes_instructions: bool + + +MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily( + path=DASHSCOPE_MULTIMODAL_GENERATION_PATH, + response_formats=frozenset({"wav"}), + sends_format=False, + takes_instructions=False, +) +SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily( + path="/api/v1/services/audio/tts/SpeechSynthesizer", + response_formats=frozenset({"mp3", "wav", "opus", "pcm"}), + sends_format=True, + takes_instructions=True, +) + + +class _Audio(BaseModel): + model_config = ConfigDict(frozen=True) + url: str + + +class _Output(BaseModel): + model_config = ConfigDict(frozen=True) + audio: _Audio + + +class _Usage(BaseModel): + model_config = ConfigDict(frozen=True) + input_tokens: int | None = None + output_tokens: int | None = None + characters: int | None = None + + +class _SpeechResponse(BaseModel): + model_config = ConfigDict(frozen=True) + output: _Output | None = None + usage: _Usage | None = None + code: str | None = None + message: str | None = None + + +def text_to_speech_family(model: str) -> _TextToSpeechFamily: + if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX): + return SPEECH_SYNTHESIZER_FAMILY + return MULTIMODAL_GENERATION_FAMILY + + +def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None: + match key: + case "response_format" if value in family.response_formats: + return {"format": value} if family.sends_format else {} + case "instructions" if family.takes_instructions: + return {"instruction": value} + case _: + return None + + +def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it", + model=model, + llm_provider="dashscope", + ) + + +def _speech_response(raw_response: httpx.Response) -> _SpeechResponse: + try: + return _SpeechResponse.model_validate_json(raw_response.content) + except ValidationError as e: + raise DashScopeError( + status_code=502, + message=f"Unexpected DashScope text-to-speech response: {e}", + headers=raw_response.headers, + ) from e + + +def _https_on_aliyun(url: str) -> str: + parsed: Final = urlparse(url) + return ( + parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url + ) + + +def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]: + parsed: Final = _speech_response(raw_response) + if parsed.output is None: + raise DashScopeError( + status_code=max(raw_response.status_code, 400), + message=parsed.message or parsed.code or raw_response.text, + headers=raw_response.headers, + ) + return _https_on_aliyun(parsed.output.audio.url), parsed.usage + + +def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None: + extension: Final = raw_response.request.extensions.get("timeout") + if extension is None: + return None + phases: Final = _TIMEOUT_PHASES.validate_python(extension) + return httpx.Timeout( + connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool") + ) + + +def _token_usage(usage: _Usage | None) -> Usage | None: + if usage is None or usage.input_tokens is None or usage.output_tokens is None: + return None + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ) + + +def _refused_audio_url(error: SSRFError) -> DashScopeError: + return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}") + + +def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent: + if audio_response.status_code != 200: + raise DashScopeError( + status_code=502, + message=f"DashScope audio download failed with HTTP {audio_response.status_code}", + headers=audio_response.headers, + ) + binary: Final = HttpxBinaryResponseContent(audio_response) + binary.usage = _token_usage(usage) + if usage is not None and usage.characters is not None: + binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters} + return binary + + +def _shared_async_client() -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE) + + +class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig): + def __init__( + self, + sync_client: Callable[[], HTTPHandler] = get_httpx_client, + async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client, + ) -> None: + super().__init__() + self._sync_client: Final = sync_client + self._async_client: Final = async_client + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())] + + @override + def map_openai_params( + self, + model: str, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract + drop_params: bool = False, + kwargs: Mapping[str, object] | None = None, + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items()) + unsupported: Final = ( + *(f"{key}={value!r}" for key, value, field in fields if field is None), + *(("voice as an object",) if isinstance(voice, dict) else ()), + ) + if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))): + raise _unsupported(model, unsupported) + supported: Final = tuple(field for _, _, field in fields if field is not None) + return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported)) + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key) + if resolved_api_key is None: + raise ValueError(missing_dashscope_family_key_message("dashscope")) + return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"} + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE") + root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE + return f"{root}{text_to_speech_family(model).path}" + + @override + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + headers: dict[str, str], # mutable-ok: mirrors override contract + ) -> TextToSpeechRequestData: + voice_field: Final = {"voice": voice} if voice is not None else {} + return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}} + + @override + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response)) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = await async_safe_get( + self._async_client(), audio_url, timeout=_request_timeout(raw_response) + ) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract + ) -> BaseLLMException: + return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers)) diff --git a/litellm/main.py b/litellm/main.py index 0162742de18..d469aa629fe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8826,8 +8826,9 @@ def speech( extra_headers: Final = kwargs.get("extra_headers", None) model_info: Final = kwargs.get("model_info", None) shared_session: Final = kwargs.get("shared_session", None) + caller_api_base: Final = api_base or base_url model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url + model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base ) kwargs.pop("tags", []) @@ -8882,13 +8883,13 @@ def speech( custom_llm_provider=custom_llm_provider, ) response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None - if custom_llm_provider == "edenai": - litellm_params_dict["api_base"] = api_base + if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None: + litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base response = base_llm_http_handler.text_to_speech_handler( model=model, input=input, voice=voice if isinstance(voice, str) else None, - text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(), + text_to_speech_provider_config=text_to_speech_provider_config, text_to_speech_optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params_dict, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6bbb63b6fc..50cfc6007ca 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1d487f33835..069ecfb82b2 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, diff --git a/litellm/utils.py b/litellm/utils.py index e46e8ca1d52..904d154f8df 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10923,19 +10923,8 @@ class ProviderConfigManager: return None @staticmethod - def get_provider_image_edit_config( - model: str, - provider: LlmProviders, - ) -> BaseImageEditConfig | None: - if LlmProviders.OPENAI == provider: - from litellm.llms.openai.image_edit import OpenAIImageEditConfig - - return OpenAIImageEditConfig() - elif LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config - - return get_hosted_vllm_image_edit_config(model=model) - elif LlmProviders.AZURE == provider: + def _get_model_independent_image_edit_config(provider: LlmProviders) -> BaseImageEditConfig | None: + if LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, ) @@ -10953,6 +10942,36 @@ class ProviderConfigManager: ) return BlackForestLabsImageEditConfig() + elif LlmProviders.LITELLM_PROXY == provider: + from litellm.llms.litellm_proxy.image_edit.transformation import ( + LiteLLMProxyImageEditConfig, + ) + + return LiteLLMProxyImageEditConfig() + elif LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.image_edit.transformation import ( + DashScopeImageEditConfig, + ) + + return DashScopeImageEditConfig() + return None + + @staticmethod + def get_provider_image_edit_config( + model: str, + provider: LlmProviders, + ) -> BaseImageEditConfig | None: + model_independent_config: Final = ProviderConfigManager._get_model_independent_image_edit_config(provider) + if model_independent_config is not None: + return model_independent_config + if LlmProviders.OPENAI == provider: + from litellm.llms.openai.image_edit import OpenAIImageEditConfig + + return OpenAIImageEditConfig() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config + + return get_hosted_vllm_image_edit_config(model=model) elif LlmProviders.FAL_AI == provider: from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config @@ -10965,12 +10984,6 @@ class ProviderConfigManager: from litellm.llms.gemini.image_edit import get_gemini_image_edit_config return get_gemini_image_edit_config(model) - elif LlmProviders.LITELLM_PROXY == provider: - from litellm.llms.litellm_proxy.image_edit.transformation import ( - LiteLLMProxyImageEditConfig, - ) - - return LiteLLMProxyImageEditConfig() elif LlmProviders.VERTEX_AI == provider: from litellm.llms.vertex_ai.image_edit import ( get_vertex_ai_image_edit_config, @@ -11192,6 +11205,12 @@ class ProviderConfigManager: return AWSPollyTextToSpeechConfig() elif litellm.LlmProviders.EDENAI == provider: return litellm.EdenAITextToSpeechConfig() + elif litellm.LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.text_to_speech.transformation import ( + DashScopeTextToSpeechConfig, + ) + + return DashScopeTextToSpeechConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6bbb63b6fc..50cfc6007ca 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d487f33835..069ecfb82b2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, diff --git a/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py new file mode 100644 index 00000000000..ea16633d81d --- /dev/null +++ b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py @@ -0,0 +1,259 @@ +import base64 +import json +import os +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment, list_value, object_value +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._dashscope_video_support import deployed, json_reply, rate, settled_spend +from pydantic import JsonValue + +_API_KEY: Final = "synthetic-dashscope-key" +_MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +_SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +_OSS_PATH: Final = "/oss/" +_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +_PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(_PNG).decode()}" +_EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +_PROMPT: Final = "Turn the red circle into a blue square" +_TEXT: Final = "Hello from LiteLLM." +_BILLED_CHARACTERS: Final = len(_TEXT) +_WAV: Final = b"RIFF\x24\x00\x00\x00WAVEfmt " + uuid.uuid4().bytes +_MP3: Final = b"ID3\x04\x00\x00\x00\x00\x00\x00" + uuid.uuid4().bytes +_SIGNED_QUERY: Final = "Expires=1793316671&OSSAccessKeyId=LTAI5t&Signature=ab%2Bc%3D" +_QWEN_TTS: Final = "dashscope-qwen-tts" +_QWEN_AUDIO_TTS: Final = "dashscope-qwen-audio-tts" +_INPUT_TOKENS: Final = 8 +_OUTPUT_TOKENS: Final = 30 +_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) + + +def _edit_api(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", _MULTIMODAL_PATH), (request.method, request.target) + return json_reply( + { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": _EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, + } + ) + + +def _synthesized(request: Request, file_name: str, usage: dict[str, JsonValue]) -> Reply: + audio_url: Final = f"http://{request.headers['host']}{_OSS_PATH}{file_name}?{_SIGNED_QUERY}" + return json_reply({"output": {"audio": {"url": audio_url, "data": "", "id": "audio-1"}}, "usage": usage}) + + +def _speech_api(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.wav"): + return Reply(body=_WAV, content_type="audio/wav") + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.mp3"): + return Reply(body=_MP3, content_type="audio/mpeg") + if (request.method, request.target) == ("POST", _MULTIMODAL_PATH): + return _synthesized(request, "speech.wav", {"characters": _BILLED_CHARACTERS}) + assert (request.method, request.target) == ("POST", _SYNTHESIZER_PATH), (request.method, request.target) + return _synthesized( + request, + "speech.mp3", + {"characters": _BILLED_CHARACTERS, "input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, + ) + + +def _pooled_database_url(url: str) -> str: + parts: Final = urlsplit(url) + query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part) + return urlunsplit(parts._replace(query=query)) + + +def _tts_deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": _API_KEY}} + + +def _error_message(body: bytes) -> str: + return str(object_value(JSON_OBJECT.validate_json(body)["error"])["message"]) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def speech_wire() -> Iterator[Wire]: + with wire_server(_speech_api) as served: + yield served + + +@pytest.fixture(scope="module") +def speech_proxy(rig: Gateway, speech_wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("dashscope-speech") + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path: Final = directory / "dashscope-speech.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + _tts_deployment(_QWEN_TTS, "dashscope/qwen3-tts-flash", speech_wire.url), + _tts_deployment(_QWEN_AUDIO_TTS, "dashscope/qwen-audio-3.1-tts-flash", speech_wire.url), + ], + "general_settings": { + **object_value(config["general_settings"]), + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + with owned_proxy( + rig, directory, {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}, config=path + ) as owned: + yield owned + + +def test_a_multipart_edit_reaches_dashscope_as_inline_images_and_bills_the_edited_image(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + key: Final = scenario.key(models=[model]) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT, "size": "1024x1024", "negative_prompt": "blurry"}, + files=[("image[]", ("a.png", _PNG, "image/png")), ("image[]", ("b.png", _PNG, "image/png"))], + headers={"Authorization": f"Bearer {key}"}, + ) + + assert response.status_code == 200, response.text + data: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + assert [object_value(item)["url"] for item in data] == [_EDITED_URL] + (request,) = wire.drain() + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(request.body) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [{"image": _PNG_DATA_URI}, {"image": _PNG_DATA_URI}, {"text": _PROMPT}], + } + ] + }, + "parameters": {"size": "1024*1024", "negative_prompt": "blurry"}, + } + settled_spend(key, rate("dashscope", "output_cost_per_image", model="qwen-image-2.0")) + + +def test_a_fourth_image_is_refused_with_a_400_before_dashscope_is_called(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT}, + files=[("image[]", (f"{index}.png", _PNG, "image/png")) for index in range(4)], + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + + assert response.status_code == 400, response.text + assert "at most 3 input images, got 4" in _error_message(response.content) + assert wire.drain() == () + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen3_tts_returns_the_downloaded_wav_and_bills_each_input_character( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_TTS]) + response: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry"}, key=key + ) + + assert response.status_code == 200, response.text + assert response.content == _WAV + assert response.headers["content-type"] == "audio/wav" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _MULTIMODAL_PATH) + assert synthesis.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(synthesis.body) == {"model": "qwen3-tts-flash", "input": {"text": _TEXT, "voice": "Cherry"}} + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.wav?{_SIGNED_QUERY}") + assert "authorization" not in download.headers + settled_spend(key, _BILLED_CHARACTERS * rate("dashscope", "input_cost_per_character", "qwen3-tts-flash")) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen_audio_tts_sends_format_and_instruction_returns_the_mp3_and_bills_reported_tokens( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + model: Final = "qwen-audio-3.1-tts-flash" + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_AUDIO_TTS]) + response: Final = speech_proxy.request( + "POST", + "/v1/audio/speech", + { + "model": _QWEN_AUDIO_TTS, + "input": _TEXT, + "voice": "longanhuan_v3.1", + "response_format": "mp3", + "instructions": "Speak slowly.", + }, + key=key, + ) + + assert response.status_code == 200, response.text + assert response.content == _MP3 + assert response.headers["content-type"] == "audio/mpeg" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _SYNTHESIZER_PATH) + assert json.loads(synthesis.body) == { + "model": model, + "input": {"text": _TEXT, "voice": "longanhuan_v3.1", "format": "mp3", "instruction": "Speak slowly."}, + } + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.mp3?{_SIGNED_QUERY}") + settled_spend( + key, + _INPUT_TOKENS * rate("dashscope", "input_cost_per_token", model) + + _OUTPUT_TOKENS * rate("dashscope", "output_cost_per_token", model), + ) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_format_qwen3_tts_cannot_produce_answers_400_without_reaching_dashscope( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + refused: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "mp3"} + ) + served: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "wav"} + ) + + assert refused.status_code == 400, refused.text + assert "response_format='mp3'" in _error_message(refused.content) + assert served.status_code == 200, served.text + assert [(request.method, request.target.split("?")[0]) for request in speech_wire.drain()] == [ + ("POST", _MULTIMODAL_PATH), + ("GET", f"{_OSS_PATH}speech.wav"), + ] diff --git a/tests/unit/images/test_image_edit.py b/tests/unit/images/test_image_edit.py index 4f858050e3e..06372a29d92 100644 --- a/tests/unit/images/test_image_edit.py +++ b/tests/unit/images/test_image_edit.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, override import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import FileTypes, ImageResponse _PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" @@ -34,6 +35,26 @@ _EDIT_RESPONSE: Final = { }, } +_DASHSCOPE_IMAGE_URL: Final = ( + "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +) +_DASHSCOPE_EDIT_RESPONSE: Final = { + "output": { + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": [{"image": "https://example.com/edited.png"}]}, + } + ] + }, + "usage": {"image_count": 1}, +} + +_FAL_EDIT_URL: Final = "https://fal.run/fal-ai/flux-2/edit" +_FAL_EDIT_RESPONSE: Final = { + "images": [{"url": "https://example.com/edited.png", "width": 1024, "height": 768, "content_type": "image/png"}] +} + class _LoggedImageEdit(TypedDict): model: ReadOnly[str] @@ -229,3 +250,101 @@ async def test_router_image_edit_returns_the_provider_image( assert isinstance(result, ImageResponse) assert result.data is not None and result.data[0].b64_json == _EDITED_IMAGE_B64 assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_price", "size", "expected_cost"), + ( + ({"input_cost_per_image": 0.0123}, None, 0.0123), + ({"input_cost_per_pixel": 1e-08}, "2048x2048", 2048 * 2048 * 1e-08), + ), +) +async def test_an_edit_bills_the_deployment_price_at_its_size_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, + deployment_price: dict[str, float], + size: str | None, + expected_cost: float, +) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + respx_mock.post(_DASHSCOPE_IMAGE_URL).mock(return_value=httpx.Response(200, json=_DASHSCOPE_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="qwen-image-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="edit-call", + function_id="edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qwen-image-edit", + "litellm_params": { + "model": "dashscope/qwen-image-2.0", + "api_key": "fake-key", + **deployment_price, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="qwen-image-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + size=size, + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_a_fal_edit_bills_the_deployment_price_per_image_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("FAL_AI_API_BASE", raising=False) + respx_mock.post(_FAL_EDIT_URL).mock(return_value=httpx.Response(200, json=_FAL_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="flux-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="fal-edit-call", + function_id="fal-edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "flux-edit", + "litellm_params": { + "model": "fal_ai/fal-ai/flux-2/edit", + "api_key": "fake-key", + "output_cost_per_image": 0.0456, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="flux-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="fal-edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(0.0456) diff --git a/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py new file mode 100644 index 00000000000..c3d186fdd8a --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py @@ -0,0 +1,358 @@ +import base64 +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.dashscope.image_edit.transformation import DashScopeImageEditConfig +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.utils import ModelInfo + +PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(PNG).decode()}" +INTL_ENDPOINT: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +EDITED: Final = { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, +} + + +def _recording( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] +) -> httpx.MockTransport: # mutable-ok: test records the requests the transport receives + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _edited(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=EDITED) + + +def _sync_client( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] = _edited +) -> HTTPHandler: # mutable-ok: shared request log + return HTTPHandler(client=httpx.Client(transport=_recording(seen, reply))) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_an_edit_posts_every_image_inline_with_provider_params_at_the_top_of_parameters( + asynchronous: bool, +) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + arguments: Final = { + "model": "dashscope/qwen-image-2.0", + "image": [PNG, PNG], + "prompt": "Turn the circle into a square", + "size": "1024x1024", + "n": 1, + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + "api_key": "sk-dashscope-test", + } + response: Final = ( + await litellm.aimage_edit(**arguments, client=AsyncHTTPHandler(transport=_recording(seen, _edited))) + if asynchronous + else litellm.image_edit(**arguments, client=_sync_client(seen)) + ) + + assert [image.url for image in response.data or []] == [EDITED_URL] + (request,) = seen + assert str(request.url) == INTL_ENDPOINT + assert request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(request.content) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [ + {"image": PNG_DATA_URI}, + {"image": PNG_DATA_URI}, + {"text": "Turn the circle into a square"}, + ], + } + ] + }, + "parameters": { + "n": 1, + "size": "1024*1024", + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + }, + } + + +@pytest.mark.parametrize(("size", "dashscope_size"), (("1792x1024", "1792*1024"), ("1328x1328", "1328*1328"))) +def test_an_openai_size_reaches_dashscope_in_its_star_format(size: str, dashscope_size: str) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", size=size, api_key="k", client=_sync_client(seen) + ) + + assert json.loads(seen[0].content)["parameters"] == {"size": dashscope_size} + + +CHINA_ENDPOINT: Final = "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + + +@pytest.mark.parametrize( + ("api_base", "env_base", "image_base", "endpoint"), + ( + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", None, None, INTL_ENDPOINT), + ("https://dashscope.aliyuncs.com", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/apps/anthropic", None, None, INTL_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/apps/anthropic", None, CHINA_ENDPOINT), + ( + "https://gw.example.com/compatible-mode/v1", + None, + None, + "https://gw.example.com/api/v1/services/aigc/multimodal-generation/generation", + ), + (None, "https://dashscope.aliyuncs.com/compatible-mode/v1", None, CHINA_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/api/v1", None, CHINA_ENDPOINT), + (None, None, None, INTL_ENDPOINT), + ( + "https://images.example.com/edit", + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "https://images.example.com/edit", + ), + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, "https://images.example.com/edit", CHINA_ENDPOINT), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "https://images.example.com/edit", + "https://images.example.com/edit", + ), + ), +) +def test_an_edit_goes_to_the_image_endpoint_on_the_host_its_dashscope_base_names( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, env_base: str | None, image_base: str | None, endpoint: str +) -> None: + if env_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", env_base) + if image_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE_IMAGE", image_base) + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + api_base=api_base, + client=_sync_client(seen), + ) + + assert str(seen[0].url) == endpoint + + +@pytest.mark.parametrize("api_base", ("https://dashscope.aliyuncs.com/compatible-mode/v1", None)) +def test_an_edit_and_a_speech_call_with_the_same_dashscope_base_reach_the_same_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None +) -> None: + monkeypatch.setenv("DASHSCOPE_API_BASE", "https://gw.example.com/compatible-mode/v1") + edit_url: Final = DashScopeImageEditConfig().get_complete_url( + model="qwen-image-2.0", api_base=api_base, litellm_params={} + ) + speech_url: Final = DashScopeTextToSpeechConfig().get_complete_url( + model="qwen3-tts-flash", api_base=api_base, litellm_params={} + ) + + assert httpx.URL(edit_url).host == httpx.URL(speech_url).host + + +def test_a_fourth_image_is_refused_with_a_400_before_any_request_even_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + with pytest.raises(litellm.BadRequestError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=[PNG, PNG, PNG, PNG], + prompt="p", + api_key="k", + drop_params=True, + client=_sync_client(seen), + ) + + assert refused.value.status_code == 400 + assert "at most 3 input images, got 4" in str(refused.value) + assert seen == [] + + +def test_three_images_are_accepted() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=[PNG, PNG, PNG], prompt="p", api_key="k", client=_sync_client(seen) + ) + + content: Final = json.loads(seen[0].content)["input"]["messages"][0]["content"] + assert content == [{"image": PNG_DATA_URI}] * 3 + [{"text": "p"}] + + +def test_an_unsupported_openai_param_answers_400_and_is_dropped_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + client: Final = _sync_client(seen) + + with pytest.raises(litellm.UnsupportedParamsError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", quality="high", api_key="k", client=client + ) + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + quality="high", + api_key="k", + drop_params=True, + client=client, + ) + + assert refused.value.status_code == 400 + assert "quality" in str(refused.value) + (request,) = seen + assert json.loads(request.content)["parameters"] == {} + + +@pytest.mark.parametrize( + ("reply", "error", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "Image is too small"}), + litellm.APIError, + "Image is too small", + ), + ( + httpx.Response(400, json={"code": "InvalidParameter", "message": "Model not exist."}), + litellm.BadRequestError, + "Model not exist.", + ), + ), +) +def test_a_dashscope_rejection_reaches_the_caller_with_its_message( + reply: httpx.Response, error: type[Exception], message: str +) -> None: + with pytest.raises(error, match=message): + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + client=_sync_client([], lambda _: reply), + ) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_each_edited_image_is_billed_at_the_model_output_image_price(model: str) -> None: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}, {"url": EDITED_URL}]) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=f"dashscope/{model}", + call_type="aimage_edit", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(2 * litellm.model_cost[f"dashscope/{model}"]["output_cost_per_image"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_the_input_images_dashscope_reports_are_billed_on_top_of_the_edited_images() -> None: + def reply(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={**EDITED, "usage": {"input_image_count": 2, "output_image_count": 1}}) + + response: Final = litellm.image_edit( + model="dashscope/qwen-image-3.0", image=[PNG, PNG], prompt="p", api_key="k", client=_sync_client([], reply) + ) + + assert isinstance(response, litellm.ImageResponse) + assert CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=response, + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) == pytest.approx(2 * 0.004 + 0.04) + + +def _edited_images(count: int, input_images: int) -> litellm.ImageResponse: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}] * count) + response.hidden_params = {"billable_input_image_count": input_images} + return response + + +@pytest.mark.parametrize( + ("model", "deployment_price", "size", "expected_cost"), + ( + ("qwen-image-2.0", ModelInfo(input_cost_per_image=0.02), None, 2 * 0.02), + ("qwen-image-3.0-pro", ModelInfo(input_cost_per_pixel=1e-08), "1024x1024", 2 * 1024 * 1024 * 1e-08), + ), +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_image_price_bills_the_edit_the_way_it_bills_any_image_call( + model: str, deployment_price: ModelInfo, size: str | None, expected_cost: float +) -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + size=size, + call_type="aimage_edit", + model_info=deployment_price, + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_that_copies_both_image_prices_bills_edited_and_input_images_at_its_own_prices() -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) + + assert cost == pytest.approx(2 * 0.04 + 3 * 0.004) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_token_price_keeps_the_shared_per_image_billing() -> None: + prices: Final = litellm.model_cost["dashscope/qwen-image-3.0"] + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=1, input_images=2), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_token=1e-06), + ) + + assert cost == pytest.approx(2 * prices.get("input_cost_per_image", 0.0) + prices["output_cost_per_image"]) diff --git a/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py new file mode 100644 index 00000000000..d47c66be3cd --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py @@ -0,0 +1,415 @@ +import datetime +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.dashscope.common_utils import DashScopeError +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +INTL: Final = "https://dashscope-intl.aliyuncs.com" +MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +AUDIO_URL: Final = ( + "http://93.184.216.34/prod/qwen-tts/a1.mp3?Expires=1793316671&OSSAccessKeyId=LTAI&Signature=ab%2Bc%3D" +) +AUDIO: Final = b"ID3\x04\x00synthetic-mp3-frames" +SYNTHESIZED: Final = { + "output": {"audio": {"url": AUDIO_URL, "data": "", "id": "audio_1"}}, + "usage": {"characters": 6, "input_tokens": 8, "output_tokens": 30, "total_tokens": 38}, +} + +API_REQUEST: Final = httpx.Request("POST", f"{INTL}{MULTIMODAL_PATH}") + +Reply = Callable[[httpx.Request], httpx.Response] + + +def _api(reply: dict[str, object]) -> Reply: # mutable-ok: JSON body fixture + return lambda _: httpx.Response(200, json=reply) + + +def _audio(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=AUDIO, headers={"content-type": "audio/mpeg"}) + + +def _recording(seen: list[httpx.Request], reply: Reply) -> httpx.MockTransport: # mutable-ok: request log + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _refusing_blocking_download(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"blocking client used for {request.url} on the event loop") + + +def _logging() -> Logging: + return Logging( + model="dashscope-tts", + messages=[], + stream=False, + call_type="aspeech", + start_time=datetime.datetime(2026, 10, 28, tzinfo=datetime.timezone.utc), + litellm_call_id="call-tts", + function_id="fn-tts", + ) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize( + ("model", "optional_params", "path", "provider_input"), + ( + ( + "qwen3-tts-flash", + {"response_format": "wav"}, + MULTIMODAL_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ( + "qwen-audio-3.1-tts-flash", + {"response_format": "mp3", "instructions": "Speak slowly."}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry", "format": "mp3", "instruction": "Speak slowly."}, + ), + ( + "qwen-audio-3.0-tts-plus", + {}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ), +) +async def test_each_family_posts_to_its_endpoint_and_the_async_handler_returns_the_downloaded_audio( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + path: str, + provider_input: dict[str, object], # mutable-ok: parametrized fixture +) -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_refusing_blocking_download)) + ), + async_client=lambda: AsyncHTTPHandler(transport=_recording(download_seen, _audio)), + ) + voice, mapped = config.map_openai_params(model=model, optional_params=optional_params, voice="Cherry") + + speech: Final = await BaseLLMHTTPHandler().async_text_to_speech_handler( + model=model, + input="Hello.", + voice=voice, + text_to_speech_provider_config=config, + text_to_speech_optional_params=mapped, + custom_llm_provider="dashscope", + litellm_params={"api_key": "sk-dashscope-test", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(SYNTHESIZED))), + ) + + assert speech.content == AUDIO + (api_request,) = api_seen + assert str(api_request.url) == f"{INTL}{path}" + assert api_request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(api_request.content) == {"model": model, "input": provider_input} + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert "authorization" not in download.headers + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +def test_the_sync_handler_downloads_through_the_injected_blocking_client_within_the_call_timeout() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler(client=httpx.Client(transport=_recording(api_seen, _api(SYNTHESIZED)))), + ) + + assert isinstance(speech, HttpxBinaryResponseContent) + assert speech.content == AUDIO + (api_request,) = api_seen + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +@pytest.mark.parametrize( + ("audio_url", "download_url"), + ( + ( + "http://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + ), + ( + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + ), + ("http://audio.notaliyuncs.com/a1.wav", "http://audio.notaliyuncs.com/a1.wav"), + (AUDIO_URL, AUDIO_URL), + ), +) +def test_an_alibaba_cloud_audio_url_is_downloaded_over_https( + monkeypatch: pytest.MonkeyPatch, audio_url: str, download_url: str +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler( + client=httpx.Client( + transport=_recording([], _api({**SYNTHESIZED, "output": {"audio": {"url": audio_url}}})) + ) + ), + ) + + assert speech.content == AUDIO + (download,) = download_seen + assert str(download.url) == download_url + + +@pytest.mark.parametrize( + ("api_base", "environment", "model", "url"), + ( + (None, None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + (None, None, "qwen-audio-3.1-tts-plus", f"{INTL}{SYNTHESIZER_PATH}"), + ( + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "qwen-audio-3.1-tts-plus", + f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}", + ), + (f"{INTL}{SYNTHESIZER_PATH}", None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + ( + "https://dashscope.aliyuncs.com/apps/anthropic", + None, + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ( + "https://relay.example.com/dashscope/", + None, + "qwen3-tts-flash", + f"https://relay.example.com/dashscope{MULTIMODAL_PATH}", + ), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ), +) +def test_the_endpoint_follows_the_model_family_on_the_configured_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, environment: str | None, model: str, url: str +) -> None: + if environment is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", environment) + + assert DashScopeTextToSpeechConfig().get_complete_url(model=model, api_base=api_base, litellm_params={}) == url + + +@pytest.mark.parametrize( + ("model", "optional_params", "voice", "refused"), + ( + ("qwen3-tts-flash", {"response_format": "mp3"}, "Cherry", "response_format='mp3'"), + ("qwen3-tts-flash", {"instructions": "Whisper."}, "Cherry", "instructions='Whisper.'"), + ("qwen-audio-3.1-tts-flash", {"response_format": "aac"}, "Cherry", "response_format='aac'"), + ("qwen-audio-3.1-tts-flash", {"speed": 1.5}, "Cherry", "speed=1.5"), + ("qwen-audio-3.1-tts-flash", {}, {"voice_id": "Cherry"}, "voice as an object"), + ), +) +def test_an_unsupported_param_answers_400_and_is_dropped_under_drop_params( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + voice: str | dict[str, object], # mutable-ok: parametrized fixture + refused: str, +) -> None: + config: Final = DashScopeTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError) as rejected: + config.map_openai_params(model=model, optional_params=optional_params, voice=voice) + dropped_voice, dropped = config.map_openai_params( + model=model, optional_params=optional_params, voice=voice, kwargs={"drop_params": True} + ) + + assert rejected.value.status_code == 400 + assert refused in str(rejected.value) + assert dropped == {} + assert dropped_voice == (voice if isinstance(voice, str) else None) + + +async def test_aspeech_uses_the_native_endpoint_on_the_caller_api_base_and_refuses_a_private_audio_url() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + private_audio: Final = {"output": {"audio": {"url": "http://169.254.169.254/latest/meta-data"}}} + + with pytest.raises(litellm.BadGatewayError) as refused: + await litellm.aspeech( + model="dashscope/qwen-audio-3.1-tts-flash", + input="Hello.", + voice="longanhuan_v3.1", + response_format="opus", + api_key="sk-dashscope-test", + api_base="https://dashscope.aliyuncs.com/compatible-mode/v1", + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(private_audio))), + ) + + assert refused.value.status_code == 502 + assert "audio URL that was refused" in str(refused.value) + (api_request,) = api_seen + assert str(api_request.url) == f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}" + assert json.loads(api_request.content) == { + "model": "qwen-audio-3.1-tts-flash", + "input": {"text": "Hello.", "voice": "longanhuan_v3.1", "format": "opus"}, + } + + +@pytest.mark.parametrize( + ("api_reply", "audio_reply", "status", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "The voice property is required."}), + _audio, + 400, + "The voice property is required.", + ), + ( + httpx.Response(200, content=b"gateway"), + _audio, + 502, + "Unexpected DashScope text-to-speech response", + ), + ( + httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + lambda _: httpx.Response(403), + 502, + "audio download failed with HTTP 403", + ), + ), +) +async def test_a_failed_synthesis_or_download_raises_a_dashscope_error( + api_reply: httpx.Response, audio_reply: Reply, status: int, message: str +) -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(audio_reply)) + ) + + with pytest.raises(DashScopeError) as failed: + await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", raw_response=api_reply, logging_obj=_logging() + ) + + assert failed.value.status_code == status + assert message in failed.value.message + + +async def test_provider_token_usage_rides_on_the_audio_so_token_priced_speech_is_billed() -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen-audio-3.1-tts-flash", + raw_response=httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + logging_obj=_logging(), + ) + + assert speech.usage == Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_character_priced_speech_bills_the_characters_dashscope_reports() -> None: + text: Final = "你好 世界。" + reported: Final = {**SYNTHESIZED, "usage": {"characters": 10}} + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", + raw_response=httpx.Response(200, json=reported, request=API_REQUEST), + logging_obj=_logging(), + ) + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen3-tts-flash", + call_type="aspeech", + prompt=text, + custom_llm_provider="dashscope", + ) + + assert litellm.utils.count_characters(text) != 10 + assert cost == pytest.approx(10 * litellm.model_cost["dashscope/qwen3-tts-flash"]["input_cost_per_character"]) + + +@pytest.mark.parametrize( + "model", ("qwen3-tts-flash", "qwen-audio-3.1-tts-plus", "qwen-audio-3.0-tts-plus", "qwen-audio-3.0-tts-flash") +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_character_priced_speech_bills_each_input_character_at_the_model_rate(model: str) -> None: + cost: Final = litellm.completion_cost( + completion_response=HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)), + model=f"dashscope/{model}", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(6 * litellm.model_cost[f"dashscope/{model}"]["input_cost_per_character"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_token_priced_speech_bills_the_provider_reported_input_and_output_tokens() -> None: + speech: Final = HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)) + speech.usage = Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + row: Final = litellm.model_cost["dashscope/qwen-audio-3.1-tts-flash"] + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen-audio-3.1-tts-flash", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(8 * row["input_cost_per_token"] + 30 * row["output_cost_per_token"]) + assert row["output_cost_per_token"] > row["input_cost_per_token"] > 0 diff --git a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py index 2e4d68a02da..981ac3e4855 100644 --- a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py +++ b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py @@ -2,13 +2,51 @@ Test RunwayML text-to-speech transformation """ +from collections.abc import Iterator +from typing import Final +import httpx +import pytest +import respx +import litellm from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, ) +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_async_speech_polls_the_runwayml_task_with_the_caller_key_and_returns_its_audio( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + respx_mock.post("https://api.dev.runwayml.com/v1/text_to_speech").mock( + return_value=httpx.Response(200, json={"id": "task_1", "status": "PENDING"}) + ) + poll: Final = respx_mock.get("https://api.dev.runwayml.com/v1/tasks/task_1").mock( + return_value=httpx.Response( + 200, json={"id": "task_1", "status": "SUCCEEDED", "output": ["https://storage.example.com/a1.mp3"]} + ) + ) + respx_mock.get("https://storage.example.com/a1.mp3").mock( + return_value=httpx.Response(200, content=b"ID3runway-audio") + ) + + speech: Final = await litellm.aspeech( + model="runwayml/eleven_multilingual_v2", input="Hello.", voice="alloy", api_key="rw-key" + ) + + assert speech.content == b"ID3runway-audio" + assert poll.calls.last.request.headers["authorization"] == "Bearer rw-key" + + def test_openai_voice_mapping_to_runwayml(): """ Test that OpenAI voice names are correctly mapped to RunwayML preset IDs diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py index 43a74e56028..e35e21c67f9 100644 --- a/tests/unit/llms/test_get_complete_url_trailing_slash.py +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -137,6 +137,7 @@ API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( "brave.search.transformation.BraveSearchConfig", "cohere.chat.v2_transformation.CohereV2ChatConfig", "cohere.embed.transformation.CohereEmbeddingConfig", + "dashscope.image_edit.transformation.DashScopeImageEditConfig", "dashscope.image_generation.transformation.DashScopeImageGenerationConfig", "dataforseo.search.transformation.DataForSEOSearchConfig", "datarobot.chat.transformation.DataRobotConfig", diff --git a/tests/unit/test_dashscope_image_generation.py b/tests/unit/test_dashscope_image_generation.py index 9c65399350c..0f35136bac3 100644 --- a/tests/unit/test_dashscope_image_generation.py +++ b/tests/unit/test_dashscope_image_generation.py @@ -14,6 +14,7 @@ import respx from pydantic import ValidationError import litellm +from litellm.cost_calculator import default_image_cost_calculator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.dashscope.common_utils import ( DASHSCOPE_DEFAULT_ORIGIN, @@ -23,7 +24,8 @@ from litellm.llms.dashscope.image_generation.transformation import ( DEFAULT_API_BASE, DashScopeImageGenerationConfig, ) -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.types.utils import ImageResponse, ModelInfo from litellm.utils import get_llm_provider # --------------------------------------------------------------------------- @@ -614,6 +616,16 @@ def test_transform_response_without_image_content_has_no_images( assert _transform_response(payload).data == [] +@pytest.mark.parametrize("usage", [None, "n/a", ["input_image_count", 2]]) +def test_transform_response_returns_the_images_whatever_shape_usage_has(usage: object): + response = _transform_response( + {"output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, "usage": usage} + ) + + assert [image.url for image in response.data] == ["https://a.example/1.png"] + assert "billable_input_image_count" not in response.hidden_params + + def test_transform_response_skips_content_items_without_an_image(): response = _transform_response( { @@ -672,3 +684,50 @@ def test_transform_response_rejects_malformed_payloads_without_echoing_them( _transform_response(payload) assert "input_value" not in str(exc_info.value) + + +@pytest.mark.parametrize( + ("model", "usage"), + ( + ("qwen-image-2.0", {"width": 2048, "height": 1536, "image_count": 1}), + ( + "qwen-image-3.0", + {"output_width": 2048, "output_height": 1536, "output_image_count": 1, "input_image_count": 0}, + ), + ), +) +def test_a_per_pixel_deployment_bills_the_size_dashscope_rendered_when_the_caller_sent_none( + model: str, usage: dict[str, int] +): + response = _transform_response( + { + "output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, + "usage": usage, + } + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=response, + custom_llm_provider="dashscope", + call_type="image_generation", + model_info=ModelInfo(input_cost_per_pixel=1e-08), + ) + + assert cost == pytest.approx(2048 * 1536 * 1e-08) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_shared_row_bills_a_generated_image_the_same_on_the_default_calculator_released_proxies_use(model: str): + dashscope_cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=ImageResponse(data=[{"url": "https://a.example/1.png"}]), + custom_llm_provider="dashscope", + call_type="image_generation", + ) + + assert dashscope_cost > 0 + assert default_image_cost_calculator(model=f"dashscope/{model}", custom_llm_provider="dashscope") == pytest.approx( + dashscope_cost + ) From 60601ca99505da4be5cd27fb80524b14d204f340 Mon Sep 17 00:00:00 2001 From: "Juan V." Date: Fri, 30 Oct 2026 01:52:43 +0100 Subject: [PATCH 05/25] feat(pass-through): add eu/jev decision API pass-through with spend tracking (#46875) * feat(pass-through): add eu/jev decision API pass-through with spend tracking Adds a /eu_jev/{endpoint:path} pass-through route for eu/jev, an EU-hosted decision API that speaks the same POST /v1/systemone contract as TypeSafe. It reads EU_JEV_API_KEY and EU_JEV_API_BASE (defaults to https://jev.bevel.software), books spend through the shared decision logging handler, and prices eu_jev/eu-jeff-1.0.0 and eu_jev/eu-jeff-latest from the cost map * fix(pass-through): default eu_jev to its canonical eu-jev.com origin * fix(pass-through): reject custom_body on the eu_jev route A key limited to other models could send an eu/jev request through custom_body, because auth checks the outer body while the shared pass-through forwards custom_body upstream. The route now answers 400 before anything is forwarded, the same way validate_oss_request guards the laya and bespoke routes * fix(pass-through): give the eu_jev route request and 200 response schemas The OpenAPI docs gate flagged POST /eu_jev/{endpoint} for OA001 and OA003. The route now documents the eu/jev System One request body and its 200 answer, usage and meta shape, taken from the eu/jev OpenAPI document. The lazy OpenAPI snapshot and the dashboard API types are regenerated with python -m litellm.proxy._lazy_openapi_snapshot and npm run gen:api * fix(pass-through): reject streaming on the eu_jev route A body with stream=true switched the route to streaming, which skips the decision spend tracking. Reject it with a 400, as validate_oss_request does for laya and bespoke. * fix(pass-through): type the eu_jev request body like the other decision routes * fix(pass-through): check eu_jev decisions against the model eu/jev runs Serve only POST /eu_jev/v1/systemone, eu/jev's one endpoint, resolve an omitted model and the legacy names to eu-jeff-latest for key model checks and pricing, check scopes under eu_jev/ like Laya and Bespoke, and answer 503 when EU_JEV_API_KEY is unset * fix(pass-through): apply eu_jev key limits, register eu_jev, price unlisted releases Pre-call hooks and guardrails now see the body model as eu_jev/, so model_rpm_limit and model_tpm_limit set on eu_jev/eu-jeff-latest apply, and upstream still gets the client's exact body. eu_jev is a registered decisions provider, so access groups, config deployments, /v1/systemone and /v1/decisions accept it, and the Admin UI can add it. A release without its own cost map row is priced as eu-jeff-latest instead of $0 * fix(eu_jev): use the pass-through deployment's api_base and accept provider-prefixed models * fix(eu_jev): declare /v1/systemone on the eu/jev cost map entries so the generated endpoint table keeps it * fix(eu_jev): take pass-through credentials only from proxy-wide deployments The eu/jev route now ignores team-scoped deployments when it picks the upstream key and base, so one team's deployment can no longer redirect another team's calls, and a base ending in /v1 is not doubled * fix(eu_jev): let a configured pass-through at /eu_jev/v1/systemone keep serving it Scope the mapped pass-through and gateway allowlist entries to the decision route instead of the whole /eu_jev prefix, so custom routes under /eu_jev keep their auth, metering, and gateway behavior from before this PR * fix(eu_jev): treat /eu_jev/v1/systemone as a provider route only while the built-in route serves it Drop the decision path from the mapped pass-through list and classify it with one check that asks the pass-through registry, so an operator's own pass-through at that path keeps its auth, metering, LLM-route status and litellm_user_api_key handling from before this PR. The built-in route now yields on a per-request root path and on any configured method, and the gateway keeps only the exact decision path * fix(eu_jev): find a configured route at the decision path with or without a trailing slash, and stop a deleted one forwarding * test(eu_jev): annotate new test variables with Final * fix(proxy): keep eu/jev pass-through routing stable during updates and method mismatches Swap an updated pass-through's routes with no await between removal and re-registration, so the built-in eu/jev route never sees the path as unconfigured mid-update. The built-in eu/jev handler no longer takes a configured route's target, and the built-in route also yields on a method-only match, so GET and PUT on the no-slash form of a slash route redirect again. * test(pass-through): annotate the route-swap test's locals as Final * fix(pass-through): keep an updated auth route's LLM API route checks on the updating worker * fix(eu_jev): leave a null model out of the upstream decision request A JSON model of null is limited and counted as eu-jeff-latest, so send the request upstream without a model field, the way an omitted model goes, since eu/jev answers a null model with 400 Unknown model * fix(pass-through): keep route checks on a path another auth endpoint still serves after an update Forward a non-string eu/jev model unchanged so eu/jev rejects it, and drop only a null or missing one * test(eu_jev): keep the non-string model test in the eu/jev route test file so a CI shard runs it * refactor(eu_jev): build the scoped pass-through body as a new dict instead of updating it in place --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- README.md | 1 + gateway/routes/allowlist.py | 1 + litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + litellm/llms/eu_jev/__init__.py | 0 litellm/llms/eu_jev/common_utils.py | 34 ++ .../llms/eu_jev/decisions/transformation.py | 9 + .../pass_through/typesafe_logging_handler.py | 13 +- ...odel_prices_and_context_window_backup.json | 22 + .../provider_endpoints_support_backup.json | 26 + litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 521 ++++++++++++++++++ litellm/proxy/auth/auth_utils.py | 9 +- litellm/proxy/auth/route_checks.py | 15 +- .../billable_request_metrics_middleware.py | 3 + .../llm_passthrough_endpoints.py | 82 +++ .../pass_through_endpoints.py | 108 +++- .../passthrough_endpoint_router.py | 68 ++- .../pass_through_endpoints/success_handler.py | 2 +- .../provider_create_fields.json | 28 + litellm/types/passthrough_endpoints/eu_jev.py | 100 ++++ litellm/types/utils.py | 1 + litellm/utils.py | 3 +- model_prices_and_context_window.json | 22 + provider_endpoints_support.json | 26 + tests/unit/decisions/test_main.py | 1 + tests/unit/proxy/auth/test_auth_utils.py | 34 ++ tests/unit/proxy/auth/test_route_checks.py | 35 ++ tests/unit/proxy/conftest.py | 40 ++ ...est_billable_request_metrics_middleware.py | 8 + ...st_typesafe_passthrough_logging_handler.py | 91 +++ .../test_llm_pass_through_endpoints.py | 385 ++++++++++++- .../test_pass_through_endpoints.py | 183 ++++++ .../test_passthrough_auth_default.py | 13 + .../test_passthrough_endpoint_router.py | 44 ++ .../public_endpoints/test_public_endpoints.py | 13 +- tests/unit/proxy/test_component_allowlists.py | 16 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 208 +++++++ 38 files changed, 2136 insertions(+), 35 deletions(-) create mode 100644 litellm/llms/eu_jev/__init__.py create mode 100644 litellm/llms/eu_jev/common_utils.py create mode 100644 litellm/llms/eu_jev/decisions/transformation.py create mode 100644 litellm/types/passthrough_endpoints/eu_jev.py diff --git a/README.md b/README.md index 5a796257654..531a50e1aca 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | | | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | | +| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | | | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | | | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cc87a2ed16b..c2740ac1d41 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( "/test", "/debug/memory/summary", "/api/event_logging/batch", + "/eu_jev/v1/systemone", } ) diff --git a/litellm/__init__.py b/litellm/__init__.py index 3e8cddac12c..084a4c516ce 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1783,6 +1783,9 @@ if TYPE_CHECKING: from .llms.bespoke.decisions.transformation import ( BespokeDecisionsConfig as BespokeDecisionsConfig, ) + from .llms.eu_jev.decisions.transformation import ( + EuJevDecisionsConfig as EuJevDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 599234bab1a..7e81602fe77 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = ( "OllamaDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", + "EuJevDecisionsConfig", "OpenAIDecisionsConfig", "AzureDecisionsConfig", "AzureAIDecisionsConfig", @@ -753,6 +754,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), diff --git a/litellm/llms/eu_jev/__init__.py b/litellm/llms/eu_jev/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/eu_jev/common_utils.py b/litellm/llms/eu_jev/common_utils.py new file mode 100644 index 00000000000..18385ef8e5d --- /dev/null +++ b/litellm/llms/eu_jev/common_utils.py @@ -0,0 +1,34 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest" +_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0")) +_EU_JEV_PREFIX: Final = "eu_jev/" + + +def eu_jev_registry_model(requested_model: str | None) -> str: + release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None + if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES: + return EU_JEV_LATEST_MODEL + return release + + +def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str: + requested_model: Final = decision_body.get("model") + return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}" + + +def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]: + hook_model: Final = hook_body.get("model") + without_model: Final = {key: value for key, value in hook_body.items() if key != "model"} + if hook_model == eu_jev_scoped_model(client_body): + client_model: Final = client_body.get("model") + if client_model is None: + return MappingProxyType(without_model) + if isinstance(client_model, str): + return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)}) + return MappingProxyType({**without_model, "model": client_model}) + if isinstance(hook_model, str): + return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)}) + return hook_body diff --git a/litellm/llms/eu_jev/decisions/transformation.py b/litellm/llms/eu_jev/decisions/transformation.py new file mode 100644 index 00000000000..18ef391b040 --- /dev/null +++ b/litellm/llms/eu_jev/decisions/transformation.py @@ -0,0 +1,9 @@ +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class EuJevDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("EU_JEV_API_KEY",) + api_base_env = ("EU_JEV_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://eu-jev.com" diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 397ec408ae9..9eae57774c2 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.litellm_logging import ( get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature ) +from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model from litellm.llms.laya.common_utils import laya_response_model from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage @@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler: response_model: Final = ( laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model ) - logged_model: Final = response_model or request_model or "unknown" + registry_request_model: Final = ( + eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model + ) + logged_model: Final = response_model or registry_request_model or "unknown" model_name: Final = f"{custom_llm_provider}/{logged_model}" usage: Final = response.usage or _TypeSafeUsage() input_tokens: Final = usage.input_tokens output_tokens: Final = usage.output_tokens + candidate_models: Final = ( + (response_model, registry_request_model, EU_JEV_LATEST_MODEL) + if custom_llm_provider == "eu_jev" + else (response_model, registry_request_model) + ) candidate_model_keys: Final = tuple( - f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None + f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) response_cost: Final = ( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50cfc6007ca..b5d09d6ff4f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72383,6 +72383,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 069ecfb82b2..18edb2cd7dc 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2241,6 +2241,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index db01475154b..dc6575f1e1b 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/tinyfish/", "/transcribe", "/typesafe/", + "/eu_jev/", "/laya/", "/bespoke/", "/openrouter/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index a9694634de7..bc64a4c7ef0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -23170,6 +23170,39 @@ "title": "CachedTokensDetails", "type": "object" }, + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, "Click": { "additionalProperties": true, "description": "A click action.", @@ -23934,6 +23967,112 @@ "title": "ErrorResponse", "type": "object" }, + "EuJevDecisionResponse": { + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "description": "One answer per question, under the same names", + "title": "Answers", + "type": "object" + }, + "meta": { + "$ref": "#/components/schemas/EuJevMeta" + }, + "model": { + "description": "The release that answered, such as eu-jeff-1.0.0", + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/EuJevUsage" + } + }, + "required": [ + "model", + "answers", + "usage", + "meta" + ], + "title": "EuJevDecisionResponse", + "type": "object" + }, + "EuJevMeta": { + "properties": { + "cost_eur": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The exact charge in euros, on authenticated requests", + "title": "Cost Eur" + }, + "latency_ms": { + "description": "Server-side time for token quoting, credit reservation and inference", + "title": "Latency Ms", + "type": "integer" + }, + "mode": { + "const": "live", + "title": "Mode", + "type": "string" + }, + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id", + "mode", + "latency_ms" + ], + "title": "EuJevMeta", + "type": "object" + }, + "EuJevUsage": { + "properties": { + "input_tokens": { + "minimum": 0.0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "minimum": 0.0, + "title": "Output Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens" + ], + "title": "EuJevUsage", + "type": "object" + }, "FileSearchTool": { "additionalProperties": true, "description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).", @@ -25594,6 +25733,26 @@ "title": "NamespaceTool", "type": "object" }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, "OperationCreateFile": { "additionalProperties": true, "description": "Instruction describing how to create a file via the apply_patch tool.", @@ -28866,6 +29025,59 @@ "title": "Result", "type": "object" }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + }, "Screenshot": { "additionalProperties": true, "description": "A screenshot action.", @@ -31758,6 +31970,315 @@ ] } }, + "/eu_jev/v1/systemone": { + "post": { + "description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)", + "operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "eu-jeff-latest", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "team": { + "criteria": { + "billing": "Payments, invoices, and refunds", + "support": "Technical issues and bugs" + }, + "instructions": "Which team should handle this?", + "type": "choice" + } + }, + "state": "I was charged twice. Can I get a refund?" + }, + "schema": { + "properties": { + "model": { + "default": "eu-jeff-latest", + "description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "description": "Two to 64 options, option name to description. A null description uses the option name", + "maxProperties": 64, + "minProperties": 2, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevChoiceQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what true and false mean", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "EuJevNoulQuestion", + "type": "object" + }, + { + "properties": { + "criteria": { + "description": "Two to 10 level descriptions, ordered from low to high", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference" + }, + "maxItems": 10, + "minItems": 2, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "criteria" + ], + "title": "EuJevScoreQuestion", + "type": "object" + } + ] + }, + "description": "One to eight questions, keyed by names of up to 80 characters", + "maxProperties": 8, + "minProperties": 1, + "propertyNames": { + "maxLength": 80, + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "maxLength": 100000, + "minLength": 1, + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array", + "title": "State" + } + }, + "required": [ + "state", + "questions" + ], + "title": "EuJevDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EuJevDecisionResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Eu Jev Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/fal_ai/{endpoint}": { "delete": { "operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 33bec02e6bc..9350c9c935c 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import ( from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path from litellm.proxy._types import * +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, @@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str): public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"] ``` """ - from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.proxy_server import general_settings, premium_user try: @@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc return _dedupe_model_candidates((f"{provider}/{decision_model}",)) + if RouteChecks.is_builtin_eu_jev_decision_route(route=route): + from litellm.llms.eu_jev.common_utils import eu_jev_registry_model + + requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model") + if requested_model is not None and not isinstance(requested_model, str): + raise HTTPException(status_code=400, detail="eu_jev model must be a string") + return [f"eu_jev/{eu_jev_registry_model(requested_model)}"] if route == "/cost/predict-cache": prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload return _dedupe_model_candidates(prediction_models) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 74e4c4c6d11..56be44600fd 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -73,6 +73,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset( _PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES: Final = ("/regenerate", "/reset_spend") _LLM_API_ROUTE_GROUPS: Final = frozenset(("llm_api_routes", "llm_api_routes_without_mcp")) +_EU_JEV_DECISION_ROUTE: Final = "/eu_jev/v1/systemone" _AUTH_ENFORCED_PASS_THROUGH_ROUTE_GROUPS: Final = frozenset(("openai_routes", *_LLM_API_ROUTE_GROUPS)) # Writes that only touch the caller's own credential row (BYOK connections and @@ -503,7 +504,19 @@ class RouteChecks: for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value: if route == _llm_passthrough_route or route.startswith(_llm_passthrough_route + "/"): return True - return False + return RouteChecks.is_builtin_eu_jev_decision_route(route=route) + + @staticmethod + def is_builtin_eu_jev_decision_route(route: str) -> bool: + if route.rstrip("/") != _EU_JEV_DECISION_ROUTE: + return False + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass_through_endpoints imports the auth stack + InitPassThroughEndpointHelpers, + ) + + return not InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=_EU_JEV_DECISION_ROUTE + ) @staticmethod def _is_get_mcp_server_discovery_route(route: str, request: Request | None) -> bool: diff --git a/litellm/proxy/middleware/billable_request_metrics_middleware.py b/litellm/proxy/middleware/billable_request_metrics_middleware.py index 96c3276efac..fd78b6b58da 100644 --- a/litellm/proxy/middleware/billable_request_metrics_middleware.py +++ b/litellm/proxy/middleware/billable_request_metrics_middleware.py @@ -25,6 +25,7 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send from litellm._logging import verbose_proxy_logger from litellm.proxy._types import LiteLLMRoutes +from litellm.proxy.auth.route_checks import RouteChecks class BillableCategory(str, Enum): @@ -116,6 +117,8 @@ def _classify_llm_route(path: str) -> str | None: suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None) if suffix_match is not None: return suffix_match + if RouteChecks.is_builtin_eu_jev_decision_route(route=path): + return path # Deep passthrough paths only: the bare prefix itself is not an inference call. return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 0a09743217c..21078dfc848 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -91,6 +91,7 @@ from litellm.proxy.common_request_processing import ( open_sse_before_first_byte, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -119,6 +120,7 @@ from litellm.proxy.pass_through_endpoints.injected_stream_usage import ( ) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( HttpPassThroughEndpointHelpers, + YieldsToConfiguredPassThroughRoute, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -132,6 +134,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( is_allowed_to_call_vector_store_endpoint, ) from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str, str_to_bool +from litellm.types.passthrough_endpoints.eu_jev import EuJevDecisionRequest, EuJevDecisionResponse from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, @@ -714,6 +717,85 @@ def typesafe_allowlist_model_names() -> tuple[str, ...]: return ("typesafe/*", *jev_models_typesafe_serves) +_EU_JEV_DECISION_REQUEST_BODY: Final = inline_request_body( + EuJevDecisionRequest, + { + "model": "eu-jeff-latest", + "state": "I was charged twice. Can I get a refund?", + "questions": { + "team": { + "type": "choice", + "instructions": "Which team should handle this?", + "criteria": {"billing": "Payments, invoices, and refunds", "support": "Technical issues and bugs"}, + }, + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) + + +eu_jev_router: Final = APIRouter(route_class=YieldsToConfiguredPassThroughRoute) + + +@eu_jev_router.post( + "/eu_jev/v1/systemone", + tags=["eu/jev Pass-through", "pass-through"], + responses={200: {"model": EuJevDecisionResponse}}, + openapi_extra={"requestBody": _EU_JEV_DECISION_REQUEST_BODY}, +) +async def eu_jev_proxy_route( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + """ + Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + + `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. + + [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + """ + body: Final = TypeAdapter(dict[str, object]).validate_python(await read_request_body(request)) + if "custom_body" in body: + raise HTTPException(status_code=400, detail="custom_body is not supported for eu_jev requests") + if body.get("stream"): + raise HTTPException(status_code=400, detail="Streaming is not supported for eu_jev requests") + eu_jev_credentials: Final = passthrough_endpoint_router.get_proxy_wide_credentials(custom_llm_provider="eu_jev") + if eu_jev_credentials is None: + raise HTTPException(status_code=503, detail="eu_jev is not configured, set EU_JEV_API_KEY on the proxy") + base_url: Final = httpx.URL( + (eu_jev_credentials.api_base or get_secret_str("EU_JEV_API_BASE") or "https://eu-jev.com") + .rstrip("/") + .removesuffix("/v1") + ) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, "/v1/systemone"), + ) + endpoint_func: Final = create_pass_through_route( + endpoint="v1/systemone", + target=str(updated_url), + custom_headers={ + "Authorization": f"Bearer {eu_jev_credentials.api_key}", + "Content-Type": "application/json", + }, + custom_llm_provider="eu_jev", + is_streaming_request=False, + ) + return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python( + await endpoint_func(request, fastapi_response, user_api_key_dict) + ) + + +router.include_router(eu_jev_router) + + @router.post( "/laya/v1/systemone", tags=["Laya Pass-through", "pass-through"], diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 23dffb4a2bb..bda2bfb9e04 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -26,9 +26,11 @@ from fastapi import ( status, ) from fastapi.responses import StreamingResponse +from fastapi.routing import APIRoute from pydantic import TypeAdapter from starlette.datastructures import UploadFile as StarletteUploadFile -from starlette.routing import BaseRoute, Route +from starlette.routing import BaseRoute, Match, Route +from starlette.types import Scope from starlette.websockets import WebSocketState from typing_extensions import ReadOnly from websockets.asyncio.client import connect @@ -73,6 +75,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body from litellm.llms.oss_decision import validate_oss_request from litellm.passthrough import BasePassthroughUtils from litellm.proxy._lazy_features import lazy_owned_routes @@ -90,6 +93,7 @@ from litellm.proxy.auth.auth_utils import ( get_request_route, request_dispatched_to_pass_through_endpoint, ) +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, @@ -433,9 +437,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): parsed_url: Final = urlparse(url) if custom_llm_provider == "chatgpt": return EndpointType.OPENAI - if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( - "/v1/systemone" - ): + if custom_llm_provider in ("typesafe", "eu_jev", "laya", "bespoke") and parsed_url.path.removesuffix( + "/" + ).endswith("/v1/systemone"): return EndpointType.DECISIONS if ( ("generateContent") in url @@ -1137,6 +1141,16 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: + return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None + + +def _eu_jev_scoped_model_override(eu_jev_client_body: Mapping[str, object] | None) -> Mapping[str, str]: + if eu_jev_client_body is None: + return MappingProxyType({}) + return MappingProxyType({"model": eu_jev_scoped_model(eu_jev_client_body)}) + + async def pass_through_request( request: Request, target: str, @@ -1255,7 +1269,12 @@ async def pass_through_request( strip_client_guardrail_information(client_body) if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(client_body) - _parsed_body = client_body + eu_jev_client_body: Final = _eu_jev_client_body(custom_llm_provider, client_body) + scoped_client_body: Final = { + **client_body, + **_eu_jev_scoped_model_override(eu_jev_client_body), + } + _parsed_body = scoped_client_body if custom_llm_provider in ("laya", "bespoke"): decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) @@ -1339,6 +1358,12 @@ async def pass_through_request( ) _ = validate_oss_request(custom_llm_provider, decision_body) _parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body) + if eu_jev_client_body is not None: + _parsed_body = TypeAdapter(dict[str, object]).validate_python( # rebind-ok: body is rebound per stage + eu_jev_upstream_body( + TypeAdapter(Mapping[str, object]).validate_python(_parsed_body), eu_jev_client_body + ) + ) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) async_client_obj: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.PassThroughEndpoint, @@ -2240,17 +2265,23 @@ def create_pass_through_route( stream, ) = await _parse_request_data_by_content_type(request) - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path): + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=path) or ( + custom_llm_provider is None and RouteChecks.is_builtin_eu_jev_decision_route(route=path) + ): raise HTTPException( status_code=404, detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", ) - passthrough_params: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=path, method=request.method + serves_configured_route: Final = custom_llm_provider != "eu_jev" + passthrough_params: Final = ( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path, method=request.method) + if serves_configured_route + else None ) if ( - passthrough_params is None + serves_configured_route + and passthrough_params is None and InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=path) is not None ): raise HTTPException( @@ -3392,6 +3423,37 @@ class InitPassThroughEndpointHelpers: del _registered_pass_through_routes[key] verbose_proxy_logger.debug("Removed pass-through route from registry: %s", key) + @staticmethod + def _llm_api_routes_of_route(route_info: Mapping[str, object]) -> tuple[str, ...]: + path: Final = route_info.get("path") + if not isinstance(path, str): + return () + if route_info.get("type") == "subpath": + return (path, path.rstrip("/") + "/*") + return (path,) + + @staticmethod + def llm_api_routes_of_endpoint(endpoint_id: str) -> frozenset[str]: + route_infos: Final = ( + route_info + for route_info in _registered_pass_through_routes.values() + if route_info["endpoint_id"] == endpoint_id + ) + return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos))) + + @staticmethod + def auth_enforced_llm_api_routes() -> frozenset[str]: + route_infos: Final = ( + route_info for route_info in _registered_pass_through_routes.values() if route_info.get("auth") is True + ) + return frozenset(chain.from_iterable(map(InitPassThroughEndpointHelpers._llm_api_routes_of_route, route_infos))) + + @staticmethod + def drop_llm_api_routes(routes: frozenset[str]) -> None: + openai_routes: Final = LiteLLMRoutes.openai_routes.value + for route in routes.intersection(openai_routes): + openai_routes.remove(route) + @staticmethod def clear_all_pass_through_routes(): """Clear all pass-through routes from the registry""" @@ -3436,6 +3498,8 @@ class InitPassThroughEndpointHelpers: return True comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route) + if RouteChecks.is_builtin_eu_jev_decision_route(route=comparison_route): + return True # Fast path: check if any registered route key contains this path # Keys are in format: "{endpoint_id}:exact:{path}:{methods}" or "{endpoint_id}:subpath:{path}:{methods}" @@ -3512,6 +3576,25 @@ class InitPassThroughEndpointHelpers: return None + @staticmethod + def has_configured_route_ignoring_trailing_slash(route: str) -> bool: + bare_route: Final = route.rstrip("/") + return any( + InitPassThroughEndpointHelpers.get_registered_pass_through_route(route=form) is not None + for form in (bare_route, f"{bare_route}/") + ) + + +class YieldsToConfiguredPassThroughRoute(APIRoute): + def matches(self, scope: Scope) -> tuple[Match, Scope]: + match, child_scope = super().matches(scope) + if match is Match.NONE: + return match, child_scope + configured: Final = InitPassThroughEndpointHelpers.has_configured_route_ignoring_trailing_slash( + route=get_request_route(Request(scope)) + ) + return (Match.NONE, {}) if configured else (match, child_scope) + def _get_combined_pass_through_endpoints( pass_through_endpoints: list[dict] | list[PassThroughGenericEndpoint], @@ -3971,9 +4054,7 @@ async def update_pass_through_endpoints( # Update the list pass_through_endpoint_data[endpoint_index] = endpoint_dict - - # Remove old routes from registry before they get re-registered - InitPassThroughEndpointHelpers.remove_endpoint_routes(endpoint_id) + replaced_llm_api_routes: Final = InitPassThroughEndpointHelpers.llm_api_routes_of_endpoint(endpoint_id) ## Update db updated_data: Final = ConfigFieldUpdate( @@ -4023,6 +4104,9 @@ async def update_pass_through_endpoints( auth=updated_endpoint.auth, timeout=updated_endpoint.timeout, ) + InitPassThroughEndpointHelpers.drop_llm_api_routes( + replaced_llm_api_routes - InitPassThroughEndpointHelpers.auth_enforced_llm_api_routes() + ) return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else []) diff --git a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py index 7092055bf6f..3b49ca1450a 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import json import os -from collections.abc import Callable +from collections.abc import Callable, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Final from fastapi import HTTPException @@ -11,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i LiteLLM_ManagedVectorStore, ) from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.secret_managers.main import ( GOOGLE_OIDC_SECRET_PREFIX, UnresolvableSecretReference, @@ -25,12 +27,23 @@ if TYPE_CHECKING: from litellm.router import Router +@dataclass(frozen=True, slots=True) +class _PassThroughCredentials: + api_key: str + api_base: str | None + + def _get_proxy_llm_router() -> "Router | None": from litellm.proxy.proxy_server import llm_router return llm_router +def _is_team_deployment(deployment: Mapping[str, object]) -> bool: + model_info: Final = deployment.get("model_info") + return is_str_keyed_mapping(model_info) and bool(model_info.get("team_id")) + + def _get_str_value(values: dict[str, object] | None, key: str) -> str | None: value: Final = values.get(key) if values is not None else None return value if isinstance(value, str) else None @@ -84,12 +97,13 @@ class PassthroughEndpointRouter: custom_llm_provider: str, region_name: str | None, ) -> str | None: - deployment_api_key: Final = self._get_deployment_api_key( + deployment_credentials: Final = self._get_deployment_credentials( custom_llm_provider=custom_llm_provider, region_name=region_name, + include_team_deployments=True, ) - if deployment_api_key is not None: - return _resolve_oidc_reference(deployment_api_key) + if deployment_credentials is not None: + return _resolve_oidc_reference(deployment_credentials.api_key) verbose_router_logger.debug( "No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider ) @@ -98,29 +112,56 @@ class PassthroughEndpointRouter: ) return _resolve_oidc_reference(get_secret_str(_env_variable_name)) + def get_proxy_wide_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None: + deployment_credentials: Final = self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, + region_name=None, + include_team_deployments=False, + ) + api_key: Final = _resolve_oidc_reference( + deployment_credentials.api_key + if deployment_credentials is not None + else get_secret_str( + self._get_default_env_variable_name_passthrough_endpoint(custom_llm_provider=custom_llm_provider) + ) + ) + if not api_key: + return None + return _PassThroughCredentials( + api_key=api_key, + api_base=deployment_credentials.api_base if deployment_credentials is not None else None, + ) + def has_credentials(self, custom_llm_provider: str) -> bool: - if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None: + if ( + self._get_deployment_credentials( + custom_llm_provider=custom_llm_provider, region_name=None, include_team_deployments=True + ) + is not None + ): return True env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint( custom_llm_provider=custom_llm_provider, ) return bool(os.environ.get(env_variable_name)) - def _get_deployment_api_key( + def _get_deployment_credentials( self, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + include_team_deployments: bool, + ) -> _PassThroughCredentials | None: llm_router: Final = self.llm_router_getter() if llm_router is None: return None deployments: Final = llm_router.registry().deployments() or () return next( ( - api_key + credentials for deployment in deployments - if ( - api_key := self._resolve_matching_deployment_api_key( + if (include_team_deployments or not _is_team_deployment(deployment)) + and ( + credentials := self._resolve_matching_deployment_credentials( litellm_params=deployment["litellm_params"], custom_llm_provider=custom_llm_provider, region_name=region_name, @@ -131,12 +172,12 @@ class PassthroughEndpointRouter: None, ) - def _resolve_matching_deployment_api_key( + def _resolve_matching_deployment_credentials( self, litellm_params: LiteLLMParamsTypedDict, custom_llm_provider: str, region_name: str | None, - ) -> str | None: + ) -> _PassThroughCredentials | None: if litellm_params.get("use_in_pass_through") is not True: return None if self._get_deployment_provider(litellm_params) != custom_llm_provider: @@ -152,7 +193,8 @@ class PassthroughEndpointRouter: ) if deployment_region != region_name: return None - return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + api_key: Final = _get_str_value(credential_values, "api_key") or litellm_params.get("api_key") + return _PassThroughCredentials(api_key=api_key, api_base=api_base) if api_key is not None else None def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None: model: Final = litellm_params.get("model") diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index bafdd9e197d..1f4e7b6df95 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -375,7 +375,7 @@ class PassThroughEndpointLogging: kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract elif ( self.is_typesafe_route(custom_llm_provider) - or custom_llm_provider in ("laya", "bespoke") + or custom_llm_provider in ("eu_jev", "laya", "bespoke") or self.is_openrouter_decisions_route(url_route, custom_llm_provider) ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 04e6c5d8ba9..d8ff32c3819 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1553,6 +1553,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "EU_JEV", + "provider_display_name": "eu/jev", + "litellm_provider": "eu_jev", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://eu-jev.com", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "eu_jev/eu-jeff-latest" + }, { "provider": "FalAI", "provider_display_name": "Fal AI", diff --git a/litellm/types/passthrough_endpoints/eu_jev.py b/litellm/types/passthrough_endpoints/eu_jev.py new file mode 100644 index 00000000000..3b81d68deea --- /dev/null +++ b/litellm/types/passthrough_endpoints/eu_jev.py @@ -0,0 +1,100 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Literal, TypeAlias + +from pydantic import ConfigDict, Field + +from litellm.types.decisions import ChoiceAnswer, DecisionsJSON, NoulAnswer, NoulCriteria, ScoreAnswer +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +EuJevEntry: TypeAlias = Annotated[ + DecisionsJSON | None, + Field(description="Free text, or a JSON object, array or null that eu/jev serializes to text before inference"), +] + + +class EuJevChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + instructions: EuJevEntry + criteria: Annotated[ + Mapping[str, EuJevEntry], + Field( + min_length=2, + max_length=64, + description="Two to 64 options, option name to description. A null description uses the option name", + ), + ] + + model_config = ConfigDict(frozen=True) + + +class EuJevNoulQuestion(LiteLLMPydanticObjectBase): + type: Literal["noul"] + instructions: EuJevEntry + criteria: Annotated[NoulCriteria | None, Field(description="Optional descriptions of what true and false mean")] = ( + None + ) + + model_config = ConfigDict(frozen=True) + + +class EuJevScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + instructions: EuJevEntry + criteria: Annotated[ + Sequence[EuJevEntry], + Field(min_length=2, max_length=10, description="Two to 10 level descriptions, ordered from low to high"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevQuestion: TypeAlias = Annotated[ + EuJevChoiceQuestion | EuJevNoulQuestion | EuJevScoreQuestion, + Field(discriminator="type"), +] + + +class EuJevDecisionRequest(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it")] = ( + "eu-jeff-latest" + ) + state: Annotated[ + Annotated[str, Field(min_length=1, max_length=100_000)] | Mapping[str, object] | Sequence[object], + Field(description="The content to evaluate, a string of up to 100,000 characters or a JSON object or array"), + ] + questions: Annotated[ + Mapping[Annotated[str, Field(min_length=1, max_length=80)], EuJevQuestion], + Field(min_length=1, max_length=8, description="One to eight questions, keyed by names of up to 80 characters"), + ] + + model_config = ConfigDict(frozen=True) + + +EuJevAnswer: TypeAlias = Annotated[ChoiceAnswer | NoulAnswer | ScoreAnswer, Field(discriminator="type")] + + +class EuJevUsage(LiteLLMPydanticObjectBase): + input_tokens: Annotated[int, Field(ge=0)] + output_tokens: Annotated[int, Field(ge=0)] + + model_config = ConfigDict(frozen=True) + + +class EuJevMeta(LiteLLMPydanticObjectBase): + request_id: str + mode: Literal["live"] + latency_ms: Annotated[ + int, Field(description="Server-side time for token quoting, credit reservation and inference") + ] + cost_eur: Annotated[str | None, Field(description="The exact charge in euros, on authenticated requests")] = None + + model_config = ConfigDict(frozen=True) + + +class EuJevDecisionResponse(LiteLLMPydanticObjectBase): + model: Annotated[str, Field(description="The release that answered, such as eu-jeff-1.0.0")] + answers: Annotated[Mapping[str, EuJevAnswer], Field(description="One answer per question, under the same names")] + usage: EuJevUsage + meta: EuJevMeta + + model_config = ConfigDict(frozen=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1e59ec0c2cb..1055ffe7b2f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4489,6 +4489,7 @@ class LlmProviders(str, Enum): STRANDS_DECIDER = "strands_decider" LAYA = "laya" BESPOKE = "bespoke" + EU_JEV = "eu_jev" MISTRAL = "mistral" MILVUS = "milvus" GROQ = "groq" diff --git a/litellm/utils.py b/litellm/utils.py index 904d154f8df..549590e1cf5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9968,11 +9968,12 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI): + if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV): configs: Final = { LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, + LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, } return configs[provider]() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50cfc6007ca..b5d09d6ff4f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72383,6 +72383,28 @@ "source": "https://docs.typesafe.ai/models", "supports_box_questions": true }, + "eu_jev/eu-jeff-1.0.0": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, + "eu_jev/eu-jeff-latest": { + "input_cost_per_token": 4.15e-08, + "litellm_provider": "eu_jev", + "max_input_tokens": 32768, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://eu-jev.com/docs", + "supported_endpoints": [ + "/v1/systemone" + ] + }, "wandb/deepseek-ai/DeepSeek-V4.1-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 2e-07, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 069ecfb82b2..18edb2cd7dc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2241,6 +2241,32 @@ "generateContent": false } }, + "eu_jev": { + "display_name": "eu/jev (`eu_jev`)", + "url": "https://docs.litellm.ai/docs/pass_through/eu_jev", + "endpoints": { + "chat_completions": false, + "responses": false, + "text_completion": false, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": false, + "interactions": false, + "systemone": true, + "a2a": false, + "generateContent": false + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e5501310556..0c357837d31 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -123,6 +123,7 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ("typesafe", "typesafe/jev-1.13", "https://api.typesafe.ai/v1/systemone", "jev-1.13"), ("nace", "nace/drex-v2.0", "https://console.nace.ai/v1/systemone", "drex-v2.0"), + ("eu_jev", "eu_jev/eu-jeff-latest", "https://eu-jev.com/v1/systemone", "eu-jeff-latest"), ( "openrouter", "openrouter/typesafe/jev-1.13", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index de37b710f36..8612351eda6 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -1028,6 +1028,40 @@ def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(pr assert denied.value.status_code == 400 +# eu-jev.com/docs (read 2026-10-21): an omitted model means eu-jeff-latest, and the legacy names +# jeff-latest and jeff-1.0.0 answer with the current release. +@pytest.mark.parametrize( + ("request_data", "permission_model"), + [ + ({}, "eu_jev/eu-jeff-latest"), + ({"model": None}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-1.0.0"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/eu-jeff-1.0.0"}, "eu_jev/eu-jeff-1.0.0"), + ({"model": "eu_jev/jeff-latest"}, "eu_jev/eu-jeff-latest"), + ], +) +@pytest.mark.parametrize("suffix", ["", "/"]) +def test_eu_jev_decision_is_checked_against_the_model_eu_jev_runs( + request_data: dict[str, object], permission_model: str, suffix: str +) -> None: + assert get_model_from_request(request_data=request_data, route=f"/eu_jev/v1/systemone{suffix}") == permission_model + + +@pytest.mark.parametrize("model", [["eu-jeff-latest"], 7, {"name": "eu-jeff-latest"}]) +def test_eu_jev_decision_rejects_a_model_that_is_not_a_string(model: object) -> None: + with pytest.raises(HTTPException) as denied: + get_model_from_request(request_data={"model": model}, route="/eu_jev/v1/systemone") + assert denied.value.status_code == 400 + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_keeps_the_requested_model() -> None: + assert get_model_from_request(request_data={"model": "jeff-latest"}, route="/eu_jev/v1/systemone") == "jeff-latest" + + def test_laya_model_normalization_does_not_change_other_provider_routes() -> None: assert get_model_from_request(request_data={"model": "jev-latest"}, route="/typesafe/v1/systemone") == "jev-latest" assert get_model_from_request(request_data={}, route="/laya/health") is None diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index b2c67b4e009..529f639999d 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -662,6 +662,41 @@ def test_virtual_key_llm_api_routes_allows_model_info(route): assert result is True +@pytest.mark.parametrize( + ("route", "is_llm_route"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone/extra", False), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/custom", False), + ], +) +def test_only_the_built_in_eu_jev_decision_path_is_an_llm_route(route: str, is_llm_route: bool) -> None: + assert RouteChecks.is_llm_api_route(route=route) is is_llm_route + + +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_an_llm_route() -> None: + assert RouteChecks.is_llm_api_route(route="/eu_jev/v1/systemone") is False + + +@pytest.mark.usefixtures("configured_eu_jev_route_with_trailing_slash") +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/"]) +def test_a_pass_through_configured_at_the_eu_jev_decision_path_with_a_trailing_slash_is_not_an_llm_route( + route: str, +) -> None: + assert RouteChecks.is_llm_api_route(route=route) is False + + +@pytest.mark.parametrize("allowed_route", ["llm_api_routes", "llm_api_routes_without_mcp"]) +def test_an_llm_api_key_can_call_the_built_in_eu_jev_decision_route(allowed_route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=[allowed_route]) + + assert RouteChecks.is_virtual_key_allowed_to_call_route( + route="/eu_jev/v1/systemone", valid_token=valid_token, request=_mock_request("POST") + ) + + @pytest.mark.parametrize("route", ["/model/info", "/v1/model/info", "/model_group/info"]) def test_model_info_not_classified_as_llm_api(route): """Membership in `llm_api_routes` must not promote /model/info to an diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index e13eddef43b..298d32def7d 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -8,17 +8,23 @@ import os import tempfile import warnings from collections.abc import Awaitable, Callable, Iterator +from contextlib import contextmanager from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest import yaml +from fastapi import FastAPI from fastapi.testclient import TestClient from prisma.errors import ClientNotConnectedError import litellm import litellm.proxy.proxy_server from litellm._logging import verbose_proxy_logger +from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, +) from litellm._service_logger import ServiceTypes from litellm.integrations.otel.model.payloads import ServiceSpanData from litellm.integrations.otel.model.spans import service_span_name @@ -239,6 +245,40 @@ def secret_vault_factory() -> type[FakeSecretVault]: return FakeSecretVault +@contextmanager +def _eu_jev_route_configured_at(path: str) -> Iterator[None]: + saved_registry: Final = dict(_registered_pass_through_routes) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path=path, + target="https://operator.example/decide", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + try: + yield + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + +@pytest.fixture +def configured_eu_jev_route() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone"): + yield + + +@pytest.fixture +def configured_eu_jev_route_with_trailing_slash() -> Iterator[None]: + with _eu_jev_route_configured_at("/eu_jev/v1/systemone/"): + yield + + @pytest.fixture def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py index 8c81a2c9b36..d2e961ad158 100644 --- a/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py +++ b/tests/unit/proxy/middleware/test_billable_request_metrics_middleware.py @@ -123,6 +123,7 @@ def test_is_pure_asgi_not_base_http_middleware(): ("/azure_speech/speechtotext/v3.2/transcriptions", (BillableCategory.LLM, "/azure_speech")), ("/transcribe", (BillableCategory.LLM, "/transcribe")), ("/transcribe/StartTranscriptionJob", (BillableCategory.LLM, "/transcribe")), + ("/eu_jev/v1/systemone", (BillableCategory.LLM, "/eu_jev/v1/systemone")), ("/mcp", (BillableCategory.MCP, "/mcp")), ("/mcp/", (BillableCategory.MCP, "/mcp")), ("/mcp/tools/list", (BillableCategory.MCP, "/mcp")), @@ -165,6 +166,8 @@ def test_classify_billable(path: str, expected: Tuple[BillableCategory, str]): "/langfuse/api/public/ingestion", # a bare provider prefix is not an inference call "/bedrock", + "/eu_jev/v1/systemone/extra", + "/eu_jev/v1/systemone2", "/v1/mcp", "/v1/mcp/tools", "/v1/mcp/server", @@ -184,6 +187,11 @@ def test_classify_non_billable_returns_none(path: str): assert classify_billable_request(path) is None +@pytest.mark.usefixtures("configured_eu_jev_route") +def test_a_pass_through_configured_at_the_eu_jev_decision_path_is_not_billed() -> None: + assert classify_billable_request("/eu_jev/v1/systemone") is None + + @pytest.mark.parametrize( "path", [ diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 79209a32f28..4011bc0294a 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -366,3 +366,94 @@ def test_success_handler_skips_typesafe_pricing_for_non_decisions_openrouter_rou assert normalized["standard_logging_response_object"] is None assert "combined_usage_object" not in normalized["kwargs"] assert normalized["kwargs"].get("model") != "openrouter/typesafe/jev-1.13-20260917" + + +def test_eu_jev_decision_is_priced_from_its_registry_row(): + model_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"] + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-latest"}, + custom_llm_provider="eu_jev", + ) + + assert response["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" + assert response["kwargs"]["custom_llm_provider"] == "eu_jev" + assert response["kwargs"]["response_cost"] == pytest.approx(64 * model_cost["input_cost_per_token"]) + assert response["kwargs"]["response_cost"] > 0 + assert response["kwargs"]["combined_usage_object"].prompt_tokens == 64 + assert response["kwargs"]["combined_usage_object"].completion_tokens == 0 + + +# eu-jev.com/docs (read 2026-10-21): an omitted model and the legacy names answer with the current release, +# which a future eu/jev release reports under a version the registry does not list yet, and a caller may pin it. +@pytest.mark.parametrize( + "request_body", [{}, {"model": "jeff-latest"}, {"model": "jeff-1.0.0"}, {"model": "eu-jeff-99.0.0"}] +) +def test_eu_jev_release_without_its_own_row_is_priced_as_the_latest_alias(request_body: dict[str, object]) -> None: + unlisted_release: Final = "eu-jeff-99.0.0" + assert f"eu_jev/{unlisted_release}" not in litellm.model_cost + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": unlisted_release, "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body=request_body, + custom_llm_provider="eu_jev", + ) + + latest_cost: Final = litellm.model_cost["eu_jev/eu-jeff-latest"]["input_cost_per_token"] + assert response["kwargs"]["model"] == f"eu_jev/{unlisted_release}" + assert latest_cost > 0 + assert response["kwargs"]["response_cost"] == pytest.approx(64 * latest_cost) + + +def test_eu_jev_pinned_release_is_not_priced_as_the_latest_alias(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "eu_jev/eu-jeff-latest", + {**litellm.model_cost["eu_jev/eu-jeff-latest"], "input_cost_per_token": 1.0}, + ) + response: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result='{"answers": {}}', + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"model": "eu-jeff-1.0.0"}, + custom_llm_provider="eu_jev", + ) + + pinned_cost: Final = litellm.model_cost["eu_jev/eu-jeff-1.0.0"]["input_cost_per_token"] + assert response["kwargs"]["response_cost"] == pytest.approx(64 * pinned_cost) + + +def test_success_handler_dispatches_eu_jev_to_the_shared_handler(): + normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( + httpx_response=_response(), + response_body={"model": "eu-jeff-1.0.0", "usage": {"input_tokens": 64, "output_tokens": 0}}, + request_body={"model": "eu-jeff-latest"}, + logging_obj=_logging_obj(), + url_route="https://eu-jev.com/v1/systemone", + result="{}", + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + custom_llm_provider="eu_jev", + ) + + assert normalized["kwargs"]["custom_llm_provider"] == "eu_jev" + assert normalized["kwargs"]["model"] == "eu_jev/eu-jeff-1.0.0" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 8d330cbd749..806d88b2467 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -15,7 +15,7 @@ from urllib.parse import parse_qs import httpx import pytest import respx -from fastapi import HTTPException, Request, Response +from fastapi import FastAPI, HTTPException, Request, Response from fastapi.routing import APIRoute from fastapi.responses import StreamingResponse from fastapi.testclient import TestClient @@ -52,6 +52,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( bedrock_proxy_route, create_pass_through_route, cursor_proxy_route, + eu_jev_proxy_route, fal_ai_proxy_route, get_azure_ai_search_index_from_endpoint, get_vertex_base_url, @@ -74,6 +75,7 @@ from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials +from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body def _assert_bedrock_processing_data_classification(data: dict[str, object], is_streaming: bool) -> None: @@ -8601,6 +8603,387 @@ class TestTypeSafePassthroughRoute: ) +class _RecordsPreCallModel(CustomLogger): + def __init__(self, switch_to: str | None = None) -> None: + super().__init__() + self.switch_to: Final = switch_to + self.models: Final[list[object]] = [] + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict, call_type: str + ) -> dict | None: + self.models.append(data.get("model")) + return None if self.switch_to is None else {**data, "model": self.switch_to} + + +class TestEuJevPassthroughRoute: + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + def test_forwards_a_decision_upstream_with_the_eu_jev_key(self, client: TestClient) -> None: + body: Final = {"model": "eu-jeff-latest", "state": "x"} + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").mock( + return_value=httpx.Response(200, json={"id": "upstream_123"}) + ) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert (response.status_code, response.json()) == (200, {"id": "upstream_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer eu-jev-test-key" + assert json.loads(sent.content) == body + + @pytest.mark.parametrize( + ("body", "hook_model"), + [ + ({"state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "jeff-latest", "state": "x"}, "eu_jev/eu-jeff-latest"), + ({"model": "eu-jeff-1.0.0", "state": "x"}, "eu_jev/eu-jeff-1.0.0"), + ], + ) + def test_pre_call_hooks_see_the_model_key_limits_name_while_upstream_gets_the_client_body( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, body: dict[str, str], hook_model: str + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json=body) + + assert response.status_code == 200 + assert recorder.models == [hook_model] + assert json.loads(route.calls.last.request.content) == body + + @pytest.mark.parametrize("model", [7, ["eu-jeff-latest"], False]) + def test_a_non_string_client_model_reaches_upstream_unchanged(self, model: int | list[str] | bool) -> None: + client_body: Final = {"model": model, "state": "x"} + hook_body: Final = {"model": eu_jev_scoped_model(client_body), "state": "x"} + + assert eu_jev_upstream_body(hook_body, client_body) == client_body + + def test_a_null_model_is_limited_as_the_latest_release_and_left_out_upstream( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": None, "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-latest"] + assert json.loads(route.calls.last.request.content) == {"state": "x"} + + def test_forwards_the_release_a_pre_call_hook_switched_to( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "callbacks", [_RecordsPreCallModel(switch_to="eu_jev/eu-jeff-1.0.0")]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "jeff-latest", "state": "x"}) + + assert response.status_code == 200 + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_provider_prefixed_model_is_limited_once_and_forwarded_without_the_prefix( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + recorder: Final = _RecordsPreCallModel() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu_jev/eu-jeff-1.0.0", "state": "x"}) + + assert response.status_code == 200 + assert recorder.models == ["eu_jev/eu-jeff-1.0.0"] + assert json.loads(route.calls.last.request.content) == {"model": "eu-jeff-1.0.0", "state": "x"} + + def test_a_pass_through_deployment_key_goes_to_that_deployment_base( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "eu-jev-friendly", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "eu-jev-deployment-key", + "api_base": "https://eu-jev-deployment.example/v2", + "use_in_pass_through": True, + }, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", deployment_router) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev-deployment.example/v2/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.calls.last.request.headers["authorization"] == "Bearer eu-jev-deployment-key" + + def test_a_team_deployment_never_redirects_another_team_s_calls( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy import proxy_server + + team_router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-a-eu-jev", + "litellm_params": { + "model": "eu_jev/eu-jeff-latest", + "api_key": "team-a-own-key", + "api_base": "https://team-a-server.example", + "use_in_pass_through": True, + }, + "model_info": {"team_id": "team-a"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", team_router) + with respx.mock(assert_all_called=False) as upstream: + team_server: Final = upstream.route(host="team-a-server.example").respond(200, json={"answers": {}}) + eu_jev: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "team b data"}) + + assert response.status_code == 200 + assert not team_server.called + assert eu_jev.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + + def test_a_base_ending_in_v1_is_not_doubled(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base/v1/") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_forwards_query_params_to_the_decision_endpoint(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone", params={"trace": "yes"}).respond( + 200, json={"answers": {}} + ) + response: Final = client.post("/eu_jev/v1/systemone?trace=yes", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_uses_the_hosted_api_when_base_is_unset(self, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("EU_JEV_API_BASE") + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://eu-jev.com/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 200 + assert route.called + + def test_answers_503_without_calling_upstream_when_the_key_is_unset( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("EU_JEV_API_KEY") + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 503 + assert "EU_JEV_API_KEY" in response.text + assert not route.called + + @pytest.mark.parametrize( + ("method", "path"), + [("GET", "/eu_jev/api/account/keys"), ("DELETE", "/eu_jev/v1/keys/abc"), ("GET", "/eu_jev/v1/systemone")], + ) + def test_forwards_nothing_but_a_decision(self, client: TestClient, method: str, path: str) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(host="eu-jev.example").respond(200, json={}) + response: Final = client.request(method, path) + + assert response.status_code in (404, 405) + assert not route.called + + def test_rejects_custom_body_that_would_replace_the_authorized_request(self, client: TestClient) -> None: + smuggled: Final = {"model": "eu-jeff-latest", "state": "refund", "questions": {"q": {"type": "noul"}}} + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "gpt-4o", "custom_body": smuggled}) + + assert response.status_code == 400 + assert "custom_body" in response.text + assert not route.called + + def test_rejects_streaming_that_would_skip_spend_tracking(self, client: TestClient) -> None: + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post( + "/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x", "stream": True} + ) + + assert response.status_code == 400 + assert "Streaming" in response.text + assert not route.called + + @pytest.fixture + def operator_app(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[list[str], str], FastAPI]]: + from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, + ) + + monkeypatch.delenv("EU_JEV_API_KEY", raising=False) + monkeypatch.delenv("LITELLM_DISABLE_LAZY_ROUTES", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + features: Final = tuple(feature for feature in LAZY_FEATURES if feature.name == "llm_passthrough") + + def build(methods: list[str], path: str) -> FastAPI: + app: Final = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + monkeypatch.setitem( + app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual") + ) + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path=path, + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=methods, + ) + return app + + saved_registry: Final = dict(_registered_pass_through_routes) + try: + yield build + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + + @pytest.mark.parametrize("root_path", ["", "/tenant-a"]) + def test_a_pass_through_endpoint_configured_at_the_same_path_keeps_serving_it( + self, operator_app: Callable[[list[str], str], FastAPI], root_path: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone"), root_path=root_path) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post(f"{root_path}/eu_jev/v1/systemone", json={"state": "x", "stream": True}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_post_to_a_get_only_pass_through_at_the_same_path_is_still_refused( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with TestClient(operator_app(["GET"], "/eu_jev/v1/systemone")) as client: + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert response.status_code == 405 + + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_the_path_without_one( + self, operator_app: Callable[[list[str], str], FastAPI] + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["POST"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.post("https://operator.example/decide").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + @pytest.mark.parametrize("method", ["GET", "PUT"]) + def test_a_pass_through_configured_with_a_trailing_slash_still_gets_other_methods_without_one( + self, operator_app: Callable[[list[str], str], FastAPI], method: str + ) -> None: + with ( + respx.mock(assert_all_called=True) as upstream, + TestClient(operator_app(["GET", "POST", "PUT"], "/eu_jev/v1/systemone/")) as client, + ): + route: Final = upstream.route(method=method, url="https://operator.example/decide").respond( + 200, json={"operator": True} + ) + response: Final = client.request(method, "/eu_jev/v1/systemone", json={"state": "x"}) + + assert (response.status_code, response.json()) == (200, {"operator": True}) + assert route.calls.last.request.headers["x-operator"] == "on" + + def test_a_request_the_built_in_route_took_is_not_forwarded_with_a_pass_through_added_meanwhile( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("EU_JEV_API_KEY", "eu-jev-test-key") + monkeypatch.setenv("EU_JEV_API_BASE", "https://eu-jev.example/base") + app: Final = operator_app(["POST"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + + def add_the_pass_through_after_routing() -> UserAPIKeyAuth: + InitPassThroughEndpointHelpers.add_exact_path_route( + app=app, + path="/eu_jev/v1/systemone", + target="https://operator.example/decide", + custom_headers={"x-operator": "on"}, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-eu-jev", + methods=["POST"], + ) + return UserAPIKeyAuth(api_key="sk-virtual") + + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, add_the_pass_through_after_routing) + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + built_in: Final = upstream.post("https://eu-jev.example/base/v1/systemone").respond(200, json={"answers": {}}) + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + response: Final = client.post("/eu_jev/v1/systemone", json={"model": "eu-jeff-latest", "state": "x"}) + + assert (response.status_code, response.json()) == (200, {"answers": {}}) + assert built_in.calls.last.request.headers["authorization"] == "Bearer eu-jev-test-key" + assert not operator.called + + def test_a_deleted_pass_through_at_the_same_path_stops_forwarding_when_built_in_routes_come_first( + self, operator_app: Callable[[list[str], str], FastAPI], monkeypatch: pytest.MonkeyPatch + ) -> None: + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + + monkeypatch.setenv("LITELLM_BUILTIN_PASS_THROUGH_ROUTES_FIRST", "true") + app: Final = operator_app(["GET", "PUT"], "/eu_jev/v1/systemone") + InitPassThroughEndpointHelpers.remove_endpoint_routes("operator-eu-jev") + with respx.mock(assert_all_called=False) as upstream, TestClient(app) as client: + operator: Final = upstream.route(host="operator.example").respond(200, json={"operator": True}) + responses: Final = (client.get("/eu_jev/v1/systemone"), client.put("/eu_jev/v1/systemone", json={})) + + assert [response.status_code for response in responses] == [404, 404] + assert not operator.called + + @pytest.mark.parametrize("provider", ["laya", "bespoke"]) class TestOssDecisionPassthroughRoute: @pytest.fixture diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 72e5d13661f..7b62853b0e9 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -82,6 +82,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES: Final = MappingProxyType( "/tinyfish/{endpoint:path}": frozenset({"GET", "POST"}), "/laya/v1/systemone": frozenset({"POST"}), "/bespoke/v1/systemone": frozenset({"POST"}), + "/eu_jev/v1/systemone": frozenset({"POST"}), } ) @@ -2977,6 +2978,56 @@ async def test_update_pass_through_endpoint_preserves_auth_false(): assert RouteChecks.is_auth_enforced_pass_through_route(route="/public-passthrough", method="POST") is False +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_serving_the_old_route_until_the_new_one_is_registered(): + from litellm.proxy._types import ConfigFieldInfo, PassThroughGenericEndpoint, UserAPIKeyAuth + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + update_pass_through_endpoints, + ) + + def registered_target() -> object: + registered: Final = InitPassThroughEndpointHelpers.get_registered_pass_through_route(route="/operator") or {} + return registered.get("passthrough_params", {}).get("target") + + targets_during_db_write: Final[list[object]] = [] + + async def record_registered_target(**_: object) -> None: + targets_during_db_write.append(registered_target()) + + with ( + patch("litellm.proxy.proxy_server.get_config_general_settings") as mock_get_config, + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=record_registered_target), + patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints._registered_pass_through_routes", {}), + ): + InitPassThroughEndpointHelpers.add_exact_path_route( + app=FastAPI(), + path="/operator", + target="http://old.example/api", + custom_headers=None, + forward_headers=None, + merge_query_params=None, + dependencies=None, + cost_per_request=None, + endpoint_id="operator-route", + methods=["POST"], + ) + mock_get_config.return_value = ConfigFieldInfo( + field_name="pass_through_endpoints", + field_value=[ + {"id": "operator-route", "path": "/operator", "target": "http://old.example/api", "methods": ["POST"]} + ], + ) + await update_pass_through_endpoints( + endpoint_id="operator-route", + data=PassThroughGenericEndpoint(path="/operator", target="http://new.example/api", methods=["POST"]), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (targets_during_db_write, registered_target()) == (["http://old.example/api"], "http://new.example/api") + + @pytest.mark.asyncio async def test_update_pass_through_endpoint_not_found(): """ @@ -10548,6 +10599,138 @@ async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_bod assert await read_request_body(request) == {"model": "qa-model"} +@pytest.mark.parametrize( + ("update", "expected"), + [ + ({"path": "/moved-to"}, (True, True, False, False)), + ({"path": "/moved-from"}, (True, True, True, True)), + ({"path": "/moved-from", "auth": False}, (False, False, False, False)), + ], +) +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_llm_api_route_checks_on_the_updated_path( + monkeypatch: pytest.MonkeyPatch, + update: dict[str, str | bool], + expected: tuple[bool, bool, bool, bool], +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.route_checks import RouteChecks + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import update_pass_through_endpoints + + async def serve_stored_endpoints(data: ConfigFieldUpdate, **_: object) -> None: + await initialize_pass_through_endpoints(data.field_value) + + stored_endpoint: Final = { + "id": "moved-route", + "path": "/moved-from", + "target": "http://example.com/api", + "auth": True, + "include_subpath": True, + } + saved_registry: Final = dict(_registered_pass_through_routes) + saved_openai_routes: Final = tuple(LiteLLMRoutes.openai_routes.value) + try: + _registered_pass_through_routes.clear() + monkeypatch.setattr(proxy_server, "app", FastAPI()) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "config_passthrough_endpoints", None) + await initialize_pass_through_endpoints([stored_endpoint]) + with ( + patch( + "litellm.proxy.proxy_server.get_config_general_settings", + return_value=ConfigFieldInfo(field_name="pass_through_endpoints", field_value=[stored_endpoint]), + ), + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=serve_stored_endpoints), + ): + await update_pass_through_endpoints( + endpoint_id="moved-route", + data=PassThroughGenericEndpoint.model_validate({"target": "http://example.com/api", **update}), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + updated_path: Final = str(update["path"]) + assert ( + RouteChecks.is_llm_api_route(route=updated_path), + f"{updated_path}/*" in LiteLLMRoutes.openai_routes.value, + RouteChecks.is_llm_api_route(route="/moved-from"), + "/moved-from/*" in LiteLLMRoutes.openai_routes.value, + ) == expected + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + LiteLLMRoutes.openai_routes.value[:] = saved_openai_routes + + +@pytest.mark.asyncio +async def test_update_pass_through_endpoint_keeps_llm_api_route_checks_on_a_path_another_endpoint_shares( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy._types import ( + ConfigFieldInfo, + ConfigFieldUpdate, + LiteLLMRoutes, + PassThroughGenericEndpoint, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.route_checks import RouteChecks + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import update_pass_through_endpoints + + async def serve_stored_endpoints(data: ConfigFieldUpdate, **_: object) -> None: + await initialize_pass_through_endpoints(data.field_value) + + stored_endpoints: Final = [ + { + "id": f"shared-{method.lower()}", + "path": "/shared", + "target": "http://example.com/api", + "auth": True, + "methods": [method], + } + for method in ("GET", "POST") + ] + saved_registry: Final = dict(_registered_pass_through_routes) + saved_openai_routes: Final = tuple(LiteLLMRoutes.openai_routes.value) + try: + _registered_pass_through_routes.clear() + monkeypatch.setattr(proxy_server, "app", FastAPI()) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "config_passthrough_endpoints", None) + await initialize_pass_through_endpoints(stored_endpoints) + with ( + patch( + "litellm.proxy.proxy_server.get_config_general_settings", + return_value=ConfigFieldInfo(field_name="pass_through_endpoints", field_value=stored_endpoints), + ), + patch("litellm.proxy.proxy_server.update_config_general_settings", side_effect=serve_stored_endpoints), + ): + await update_pass_through_endpoints( + endpoint_id="shared-get", + data=PassThroughGenericEndpoint.model_validate( + {"target": "http://example.com/api", "path": "/moved", "auth": True, "methods": ["GET"]} + ), + request=MagicMock(spec=Request), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + ) + + assert (RouteChecks.is_llm_api_route(route="/shared"), RouteChecks.is_llm_api_route(route="/moved")) == ( + True, + True, + ) + finally: + _registered_pass_through_routes.clear() + _registered_pass_through_routes.update(saved_registry) + LiteLLMRoutes.openai_routes.value[:] = saved_openai_routes + + @pytest.mark.asyncio @pytest.mark.parametrize( ("flat_cost", "upstream_cost", "expected"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py index 0e337195bb8..7771ff0274e 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_auth_default.py @@ -26,6 +26,7 @@ import pytest from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from starlette.requests import Request from litellm.proxy._types import PassThroughGenericEndpoint, ProxyException @@ -137,3 +138,15 @@ async def test_runtime_check_explicit_auth_false_still_skips_validation(): ) assert isinstance(result, UserAPIKeyAuth) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/eu_jev/v1/systemone", "/eu_jev/v1/systemone/extra", "/eu_jev/v1/systemone2"]) +async def test_no_eu_jev_route_reads_the_litellm_user_api_key_header(route: str) -> None: + request: Final = Request({"type": "http", "headers": [(b"litellm_user_api_key", b"sk-from-header")]}) + + result: Final = await check_api_key_for_custom_headers_or_pass_through_endpoints( + request=request, route=route, pass_through_endpoints=None, api_key="sk-from-bearer" + ) + + assert result == "sk-from-bearer" diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py index ab4f3f7a675..dd48a3e55ef 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_endpoint_router.py @@ -1,6 +1,7 @@ import asyncio import os import unittest +from typing import Final from unittest.mock import patch import pytest @@ -60,6 +61,49 @@ def test_credential_loaded_after_deployment_registration_still_resolves(): ) +def test_proxy_wide_credentials_pair_a_deployment_key_with_that_deployment_base() -> None: + llm_router: Final = litellm.Router( + model_list=[ + _flagged_deployment("openai/gpt-4o", api_base="https://keyless.example/v1"), + _flagged_deployment("openai/gpt-4o-mini", api_key="sk-deployment", api_base="https://keyed.example/v1"), + ] + ) + credentials: Final = _passthrough_router(llm_router).get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-deployment", "https://keyed.example/v1") + + +def test_proxy_wide_credentials_never_come_from_a_team_deployment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-proxy-env") + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a", api_base="https://team-a.example/v1"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + credentials: Final = passthrough_router.get_proxy_wide_credentials(custom_llm_provider="openai") + + assert credentials is not None + assert (credentials.api_key, credentials.api_base) == ("sk-proxy-env", None) + + +def test_credentials_for_a_provider_route_still_include_a_team_deployment_key() -> None: + team_deployment: Final = { + **_flagged_deployment("openai/gpt-4o", api_key="sk-team-a"), + "model_info": {"team_id": "team-a"}, + } + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[team_deployment])) + + assert passthrough_router.get_credentials(custom_llm_provider="openai", region_name=None) == "sk-team-a" + + +def test_proxy_wide_credentials_are_missing_without_a_deployment_or_env_key() -> None: + passthrough_router: Final = _passthrough_router(litellm.Router(model_list=[])) + + assert passthrough_router.get_proxy_wide_credentials(custom_llm_provider="anthropic") is None + + def test_credential_rotation_is_reflected_without_deployment_update(): CredentialAccessor.upsert_credentials([_credential("cred_openai", "sk-before-rotation")]) llm_router = litellm.Router( diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index da608dbca0e..48672c7a305 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -481,9 +481,16 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: return entry -@pytest.mark.parametrize("display_name,provider", (("TypeSafe", LlmProviders.TYPESAFE), ("Nace", LlmProviders.NACE))) -def test_decisions_provider_fields(display_name: str, provider: LlmProviders) -> None: - entry: Final = _decisions_provider_entry(display_name) +@pytest.mark.parametrize( + ("provider_key", "display_name", "provider"), + ( + ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), + ("Nace", "Nace", LlmProviders.NACE), + ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), + ), +) +def test_decisions_provider_fields(provider_key: str, display_name: str, provider: LlmProviders) -> None: + entry: Final = _decisions_provider_entry(provider_key) assert entry.provider_display_name == display_name assert entry.litellm_provider == provider.value diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index eb058d01770..4c0b3ad913b 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -397,6 +397,22 @@ def test_gateway_keeps_memory_summary_and_trims_the_other_debug_routes(): assert not _is_gateway_route(debug_memory_routes[path]), f"{path} must not be served by the gateway" +@pytest.mark.parametrize( + ("path", "served"), + [ + ("/eu_jev/v1/systemone", True), + ("/eu_jev/v1/systemone2", False), + ("/eu_jev/v1/systemone/{subpath:path}", False), + ("/eu_jev/custom", False), + ], +) +def test_gateway_serves_the_eu_jev_decision_route_and_no_other_eu_jev_path(path: str, served: bool) -> None: + async def endpoint() -> None: + return None + + assert _is_gateway_route(Route(path, endpoint, methods=["POST"])) is served + + def test_every_app_mount_is_assigned_to_a_component(): """Every Mount on the proxy app must be consciously assigned to a component. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6c4c5ba3b11..d542880f868 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5362,6 +5362,32 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/eu_jev/v1/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Eu Jev Proxy Route + * @description Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`. + * + * `POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`) + * and returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and + * `stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it. + * + * [Docs](https://docs.litellm.ai/docs/pass_through/eu_jev) + */ + post: operations["eu_jev_proxy_route_eu_jev_v1_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/extraction": { parameters: { query?: never; @@ -32287,6 +32313,50 @@ export interface components { [key: string]: unknown; }; }; + /** EuJevDecisionResponse */ + EuJevDecisionResponse: { + /** + * Answers + * @description One answer per question, under the same names + */ + answers: { + [key: string]: components["schemas"]["ChoiceAnswer"] | components["schemas"]["NoulAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + meta: components["schemas"]["EuJevMeta"]; + /** + * Model + * @description The release that answered, such as eu-jeff-1.0.0 + */ + model: string; + usage: components["schemas"]["EuJevUsage"]; + }; + /** EuJevMeta */ + EuJevMeta: { + /** + * Cost Eur + * @description The exact charge in euros, on authenticated requests + */ + cost_eur?: string | null; + /** + * Latency Ms + * @description Server-side time for token quoting, credit reservation and inference + */ + latency_ms: number; + /** + * Mode + * @constant + */ + mode: "live"; + /** Request Id */ + request_id: string; + }; + /** EuJevUsage */ + EuJevUsage: { + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + }; /** * Eval * @description Represents an evaluation from the OpenAI Evals API @@ -59476,6 +59546,144 @@ export interface operations { }; }; }; + eu_jev_proxy_route_eu_jev_v1_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "eu-jeff-latest", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "team": { + * "criteria": { + * "billing": "Payments, invoices, and refunds", + * "support": "Technical issues and bugs" + * }, + * "instructions": "Which team should handle this?", + * "type": "choice" + * } + * }, + * "state": "I was charged twice. Can I get a refund?" + * } + */ + "application/json": { + /** + * Model + * @description eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it + * @default eu-jeff-latest + */ + model?: string; + /** + * Questions + * @description One to eight questions, keyed by names of up to 80 characters + */ + questions: { + [key: string]: { + /** + * Criteria + * @description Two to 64 options, option name to description. A null description uses the option name + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Criteria + * @description Optional descriptions of what true and false mean + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } | { + /** + * Criteria + * @description Two to 10 level descriptions, ordered from low to high + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[] | null)[]; + /** + * Instructions + * @description Free text, or a JSON object, array or null that eu/jev serializes to text before inference + */ + instructions: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + }; + }; + /** + * State + * @description The content to evaluate, a string of up to 100,000 characters or a JSON object or array + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EuJevDecisionResponse"]; + }; + }; + }; + }; extraction_extraction_post: { parameters: { query?: never; From 24afa12253ce5493a27acfd4799b6d143615f88b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 18:02:31 -0700 Subject: [PATCH 06/25] chore(caching)!: remove the disk cache backend (#48114) * chore(caching)!: remove the disk cache backend Drop the disk cache type and its diskcache dependency, along with the caching install extra that existed only to pull it in. diskcache has not had a release since August 2023 Cache now raises a ValueError listing the supported types when it gets a type it does not know. Before, an unknown type built a Cache with no backend, which only failed on first use BREAKING CHANGE: cache_params type "disk" and the litellm[caching] extra are gone. A proxy configured with type disk now refuses to start. Use local or redis instead * test(proxy): expect the unsupported cache type error from _init_cache The test pinned the old fallthrough, where an unknown type left the cache without a backend and the proxy hit an AttributeError reading it * chore(deps): drop the diskcache scan ignore main extended diskcache is no longer in the lock, so the ignore matches nothing * fix(caching): ignore a leftover disk_cache_dir instead of crashing redis and s3 Dropping the disk_cache_dir parameter sent the key into **kwargs, which the redis and s3 branches hand to their clients, so a config that moved from the disk cache to redis or s3 and kept the key failed with a TypeError. Accept the key again, ignore it, and log a warning The unsupported type test now checks the whole supported list, and two proxy docstrings no longer name disk as a cache backend * fix(caching): warn about a leftover disk_cache_dir only once the cache is built The warning ran before the type check, so a config with type disk logged that the key was ignored and then failed on the type anyway. It now logs only after a supported backend is built. The test also checks that a cache without the key logs nothing --- .github/workflows/test-unit.yml | 8 +- license_cache.json | 1 - litellm/caching/Readme.md | 8 +- litellm/caching/__init__.py | 1 - litellm/caching/caching.py | 19 +-- litellm/caching/caching_handler.py | 2 +- litellm/caching/disk_cache.py | 92 ---------- litellm/proxy/proxy_server.py | 4 +- litellm/router.py | 2 +- litellm/types/caching.py | 1 - osv-scanner.toml | 5 - pyproject.toml | 1 - tests/unit/caching/test_caching.py | 25 +++ tests/unit/caching/test_disk_cache.py | 158 ------------------ .../proxy/proxy_server/test_proxy_config.py | 2 +- uv.lock | 15 +- 16 files changed, 47 insertions(+), 297 deletions(-) delete mode 100644 litellm/caching/disk_cache.py delete mode 100644 tests/unit/caching/test_disk_cache.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 509271cbf54..a5013155eae 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1003,9 +1003,9 @@ jobs: run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" @@ -1451,9 +1451,9 @@ jobs: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} run: | if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/license_cache.json b/license_cache.json index dc061b48f4f..cd8a54d72a0 100644 --- a/license_cache.json +++ b/license_cache.json @@ -11,7 +11,6 @@ "azure-keyvault-secrets:4.10.0": "MIT License", "google-cloud-kms:2.24.2": "Apache 2.0", "numpydoc:1.8.0": "Copyright (C) 2008-2023 Stefan van der Walt , Pauli Virtanen Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ", - "diskcache:5.6.3": "Apache 2.0", "mlflow:3.9.0": "Copyright 2018 Databricks, Inc. All rights reserved.\n \n \t\t\t\tApache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n \n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n \n 1. Definitions.\n \n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n \n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n \n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n \n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n \n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n \n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n \n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n \n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n \n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n \n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n \n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n \n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n \n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n \n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n \n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n \n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n \n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n \n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n \n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n \n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n \n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n \n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n \n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n \n END OF TERMS AND CONDITIONS\n APPENDIX: How to apply the Apache License to your work.\n \n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n \n Copyright [yyyy] [name of copyright owner]\n \n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n \n http://www.apache.org/licenses/LICENSE-2.0\n \n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n ", "diff-cover:9.7.2": "Apache-2.0", "flake8:7.3.0": "MIT", diff --git a/litellm/caching/Readme.md b/litellm/caching/Readme.md index 1d920219830..e91a5fec888 100644 --- a/litellm/caching/Readme.md +++ b/litellm/caching/Readme.md @@ -8,10 +8,9 @@ The following caching mechanisms are supported: 2. **RedisSemanticCache** 3. **QdrantSemanticCache** 4. **InMemoryCache** -5. **DiskCache** -6. **S3Cache** -7. **AzureBlobCache** -8. **DualCache** (updates both Redis and an in-memory cache simultaneously) +5. **S3Cache** +6. **AzureBlobCache** +7. **DualCache** (updates both Redis and an in-memory cache simultaneously) ## Folder Structure @@ -20,7 +19,6 @@ litellm/caching/ ├── base_cache.py ├── caching.py ├── caching_handler.py -├── disk_cache.py ├── dual_cache.py ├── in_memory_cache.py ├── qdrant_semantic_cache.py diff --git a/litellm/caching/__init__.py b/litellm/caching/__init__.py index 87f4f7a7c63..db79a9cafee 100644 --- a/litellm/caching/__init__.py +++ b/litellm/caching/__init__.py @@ -1,6 +1,5 @@ from .azure_blob_cache import AzureBlobCache from .caching import Cache, LiteLLMCacheType -from .disk_cache import DiskCache from .dual_cache import DualCache from .gcs_cache import GCSCache from .in_memory_cache import InMemoryCache diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 1ced13445de..d875ac09d40 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg from .azure_blob_cache import AzureBlobCache from .base_cache import BaseCache -from .disk_cache import DiskCache from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching from .gcs_cache import GCSCache from .in_memory_cache import InMemoryCache @@ -153,7 +152,7 @@ class Cache: Initializes the cache based on the given type. Args: - type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local". + type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local". # Redis Cache Args host (str, optional): The host address for the Redis cache. Required if type is "redis". @@ -174,9 +173,6 @@ class Cache: semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key". max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4. - # Disk Cache Args - disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None. - # S3 Cache Args s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None. s3_region_name (str, optional): The region name for the s3 cache. Defaults to None. @@ -306,8 +302,11 @@ class Cache: account_url=azure_account_url, container=azure_blob_container, ) - elif type == LiteLLMCacheType.DISK: - self.cache = DiskCache(disk_cache_dir=disk_cache_dir) + else: + supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType) + raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}") + if disk_cache_dir is not None: + verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed") if "cache" not in litellm.input_callback: litellm.input_callback.append("cache") if "cache" not in litellm.success_callback: @@ -1044,7 +1043,7 @@ def enable_cache( Enable cache with the specified configuration. Args: - type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local". + type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local". host (Optional[str]): The host address of the cache server. Defaults to None. port (Optional[str]): The port number of the cache server. Defaults to None. password (Optional[str]): The password for the cache server. Defaults to None. @@ -1056,7 +1055,7 @@ def enable_cache( None Raises: - None + ValueError: If no cache is set yet and the cache type is unsupported. """ print_verbose("LiteLLM: Enabling Cache") if "cache" not in litellm.input_callback: @@ -1091,7 +1090,7 @@ def update_cache( Update the cache for LiteLLM. Args: - type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local". + type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local". host (Optional[str]): The host of the cache. Defaults to None. port (Optional[str]): The port of the cache. Defaults to None. password (Optional[str]): The password for the cache. Defaults to None. diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 364ce9450cd..1d80d6952a4 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -9,7 +9,7 @@ This file is a wrapper around caching.py This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc) -It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup +It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup In each method it will call the appropriate method from caching.py """ diff --git a/litellm/caching/disk_cache.py b/litellm/caching/disk_cache.py deleted file mode 100644 index 9470f23570a..00000000000 --- a/litellm/caching/disk_cache.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -from typing import TYPE_CHECKING, Any, Final - -from pydantic import TypeAdapter - -from .base_cache import BaseCache - -if TYPE_CHECKING: - from opentelemetry.trace import Span as _Span - - Span = _Span -else: - Span = Any - -_DECODED_DISK_VALUE: Final = TypeAdapter(object) - - -class DiskCache(BaseCache): - def __init__(self, disk_cache_dir: str | None = None): - try: - import diskcache as dc - except ModuleNotFoundError as e: - raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e - - # if users don't provider one, use the default litellm cache - if disk_cache_dir is None: - self.disk_cache = dc.Cache(".litellm_cache") - else: - self.disk_cache = dc.Cache(disk_cache_dir) - - def set_cache(self, key, value, **kwargs): - if "ttl" in kwargs: - self.disk_cache.set(key, value, expire=kwargs["ttl"]) - else: - self.disk_cache.set(key, value) - - async def async_set_cache(self, key, value, **kwargs): - self.set_cache(key=key, value=value, **kwargs) - - async def async_set_cache_pipeline(self, cache_list, **kwargs): - for cache_key, cache_value in cache_list: - if "ttl" in kwargs: - self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"]) - else: - self.set_cache(key=cache_key, value=cache_value) - - def get_cache(self, key, **kwargs): - original_cached_response: Final = self.disk_cache.get(key) - if original_cached_response: - try: - cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response)) - except Exception: - cached_response = original_cached_response - return cached_response - return None - - def batch_get_cache(self, keys: list, **kwargs): - return_val: Final = [] - for k in keys: - val = self.get_cache(key=k, **kwargs) - return_val.append(val) - return return_val - - def increment_cache(self, key, value: int, **kwargs) -> int: - with self.disk_cache.transact(): - cached_value: Final = self.get_cache(key=key) - init_value: Final = cached_value if isinstance(cached_value, int) else 0 - new_value: Final = init_value + value - self.set_cache(key, new_value, **kwargs) - return new_value - - async def async_get_cache(self, key, **kwargs): - return self.get_cache(key=key, **kwargs) - - async def async_batch_get_cache(self, keys: list, **kwargs): - return_val: Final = [] - for k in keys: - val = self.get_cache(key=k, **kwargs) - return_val.append(val) - return return_val - - async def async_increment(self, key, value: int, **kwargs) -> int: - return self.increment_cache(key=key, value=value, **kwargs) - - def flush_cache(self): - self.disk_cache.clear() - - async def disconnect(self): - pass - - def delete_cache(self, key): - self.disk_cache.pop(key) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e6d157a7f81..6d92e8733a3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5190,7 +5190,7 @@ def _build_redis_usage_cache_from_environment() -> RedisCache | None: Lets the proxy's coordination Redis (cross-pod tpm/rpm rate limits, spend tracking, pod lock manager) run when the response-cache backend is not a - plain Redis KV cache (e.g. a semantic cache, disk, or s3). + plain Redis KV cache (e.g. a semantic cache or s3). Returns None when the environment carries no connection target (host, url, cluster nodes, or sentinel nodes). @@ -10974,7 +10974,7 @@ class ProxyStartupEvent: """ Builds a standalone Redis cache from REDIS_* environment variables so use_redis_transaction_buffer can run when the proxy cache backend is not - Redis (e.g. disk, s3). + Redis (e.g. s3). Returns None when the buffer is disabled, or when no Redis host or url is set in the environment. diff --git a/litellm/router.py b/litellm/router.py index 77958686957..f2e68424c8d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1617,7 +1617,7 @@ class Router(DeprecatedRouterMembers): self._core_deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 self._core_deployment_latency_map = {} ### CACHING ### - cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache + cache_type: Literal["local", "redis", "redis-semantic", "s3"] = "local" # default to an in-memory cache redis_cache = None cache_config: Final[dict[str, Any]] = {} diff --git a/litellm/types/caching.py b/litellm/types/caching.py index ced2065a674..fb5ab3b4cea 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -13,7 +13,6 @@ class LiteLLMCacheType(str, Enum): REDIS_SEMANTIC = "redis-semantic" VALKEY_SEMANTIC = "valkey-semantic" S3 = "s3" - DISK = "disk" QDRANT_SEMANTIC = "qdrant-semantic" AZURE_BLOB = "azure-blob" GCS = "gcs" diff --git a/osv-scanner.toml b/osv-scanner.toml index 32a4f65fdf9..45fb7635088 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -78,11 +78,6 @@ id = "GHSA-r4j5-9xpp-3985" ignoreUntil = 2026-11-27 reason = "tornado 6.6 (published 2026-10-21) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 6.5.10 while 3.11+ locks 6.6; litellm starts no tornado server or client, since semantic-router and aurelio-sdk only declare it and tenacity and sentry-sdk import it only to wrap tornado code when it is present, so tornado's chunked body reading is never reached; remove this entry once 3.10 is dropped" -[[IgnoredVulns]] -id = "GHSA-w8v5-vhqr-4h9v" -ignoreUntil = 2026-11-27 -reason = "diskcache still has no fixed release published (checked 2026-10-28), and it is reached only through the optional caching extra; remove this entry once one exists" - [[IgnoredVulns]] id = "GHSA-45c3-73f7-pv4m" ignoreUntil = 2026-11-27 diff --git a/pyproject.toml b/pyproject.toml index 6e4732b4f7a..bff00a2dbf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,7 +130,6 @@ utils = [ # Not in Docker or PyPI proxy extra. "numpydoc>=1.8.0,<2.0", ] -caching = ["diskcache>=5.6.3,<6.0"] mcp = ["mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3"] # Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API. # The floor is 4.9 because that is the release AsyncMongoClient landed in. diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 91083921130..c4727cbeff0 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -250,6 +250,15 @@ def test_semantic_cache_key_reads_tenant_identity_from_litellm_metadata(): assert key_a == key_a_in_litellm_params +def test_cache_rejects_unsupported_type() -> None: + cache_params: Final = {"type": "not-a-cache"} + with pytest.raises( + ValueError, match=re.escape("Unsupported cache type 'not-a-cache'. Supported types: ") + ) as raised: + Cache(**cache_params) + assert str(raised.value).split("Supported types: ")[1].split(", ") == [member.value for member in LiteLLMCacheType] + + def test_semantic_cache_scope_rejects_unknown_value(): with pytest.raises(ValueError, match="'team' is not a valid SemanticCacheScope"): _semantic_cache(semantic_cache_scope="team") @@ -1230,6 +1239,22 @@ def test_custom_redis_cache_params(redis_client_without_network: None) -> None: assert cache.cache.redis_client.connection_pool.connection_kwargs["db"] == 4 +def test_redis_cache_ignores_a_leftover_disk_cache_dir( + redis_client_without_network: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + Cache(type=LiteLLMCacheType.REDIS, host="127.0.0.1", port="6379") + assert "disk_cache_dir" not in caplog.text + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + cache: Final = Cache( + type=LiteLLMCacheType.REDIS, host="127.0.0.1", port="6379", disk_cache_dir="/tmp/litellm-cache" + ) + + assert isinstance(cache.cache, RedisCache) + assert "disk_cache_dir" in caplog.text + + def test_custom_redis_cache_with_key(monkeypatch: pytest.MonkeyPatch) -> None: cache: Final = Cache(type=LiteLLMCacheType.LOCAL) monkeypatch.setattr(litellm, "cache", cache) diff --git a/tests/unit/caching/test_disk_cache.py b/tests/unit/caching/test_disk_cache.py deleted file mode 100644 index 243f194cc7a..00000000000 --- a/tests/unit/caching/test_disk_cache.py +++ /dev/null @@ -1,158 +0,0 @@ -import threading -import time -from concurrent.futures import ThreadPoolExecutor - -import pytest - -pytest.importorskip("diskcache") - -from litellm.caching.disk_cache import DiskCache -from tests._wait_helpers import await_until - - -class _SlowInt(int): - def __add__(self, value: int) -> "_SlowInt": - time.sleep(0.05) - return _SlowInt(int(self) + value) - - -@pytest.fixture -def cache(tmp_path): - return DiskCache(disk_cache_dir=str(tmp_path)) - - -def test_increment_cache_starts_from_zero_when_key_missing(cache): - assert cache.increment_cache("counter", 3) == 3 - assert cache.get_cache("counter") == 3 - - -def test_increment_cache_adds_to_existing_int(cache): - cache.set_cache("counter", 7) - assert cache.increment_cache("counter", 5) == 12 - assert cache.get_cache("counter") == 12 - - -def test_increment_cache_treats_non_int_cached_value_as_zero(cache): - cache.set_cache("counter", "not-a-number") - assert cache.increment_cache("counter", 4) == 4 - assert cache.get_cache("counter") == 4 - - -def test_increment_cache_is_atomic_under_thread_concurrency(cache): - seed = 1000 - cache.set_cache("counter", _SlowInt(seed)) - thread_count = 8 - barrier = threading.Barrier(thread_count) - - def increment(_: int) -> int: - barrier.wait() - return cache.increment_cache("counter", 1) - - with ThreadPoolExecutor(max_workers=thread_count) as executor: - tuple(executor.map(increment, range(thread_count))) - - assert cache.get_cache("counter") == seed + thread_count - - -async def test_async_increment_starts_from_zero_when_key_missing(cache): - assert await cache.async_increment("counter", 2) == 2 - - -async def test_async_increment_adds_to_existing_int(cache): - await cache.async_set_cache("counter", 10) - assert await cache.async_increment("counter", 5) == 15 - - -async def test_async_increment_treats_non_int_cached_value_as_zero(cache): - await cache.async_set_cache("counter", "corrupt") - assert await cache.async_increment("counter", 9) == 9 - - -@pytest.fixture -def disk_cache(tmp_path): - import litellm - from litellm.caching import Cache - - litellm.cache = Cache(type="disk", disk_cache_dir=str(tmp_path)) - yield litellm.cache - litellm.cache = None - - -COMPLETION_MESSAGES = [{"role": "user", "content": "write a one sentence poem about: disk-cache-unit-key"}] - - -def test_completion_is_served_from_disk_cache_until_params_change(disk_cache): - import litellm - - first = litellm.completion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="first" - ) - repeat = litellm.completion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="second" - ) - changed = litellm.completion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, temperature=0.5, mock_response="third" - ) - - assert (repeat.choices[0].message.content, repeat.id, repeat.created) == ( - first.choices[0].message.content, - first.id, - first.created, - ) - assert changed.choices[0].message.content == "third" - - -async def test_async_completion_is_served_from_disk_cache_until_params_change(disk_cache): - import litellm - - first = await litellm.acompletion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="first" - ) - await await_until(lambda: len(disk_cache.cache.disk_cache) == 1, message="the first completion is cached") - repeat = await litellm.acompletion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, max_tokens=20, mock_response="second" - ) - changed = await litellm.acompletion( - "gpt-3.5-turbo", messages=COMPLETION_MESSAGES, caching=True, temperature=0.5, mock_response="third" - ) - - assert (repeat.choices[0].message.content, repeat.id, repeat.created) == ( - first.choices[0].message.content, - first.id, - first.created, - ) - assert changed.choices[0].message.content == "third" - - -async def test_embedding_is_served_from_disk_cache_until_params_change(disk_cache, respx_mock, monkeypatch): - import httpx - - import litellm - - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - input_text = ["hello disk-embedding-unit-key"] - embeddings_route = respx_mock.post("https://api.openai.com/v1/embeddings").mock( - return_value=httpx.Response( - 200, - json={ - "object": "list", - "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], - "model": "text-embedding-ada-002", - "usage": {"prompt_tokens": 2, "total_tokens": 2}, - }, - ) - ) - - await litellm.aembedding("openai/text-embedding-ada-002", input=input_text, caching=True, api_key="sk-unit") - await await_until(lambda: len(disk_cache.cache.disk_cache) == 1, message="the first embedding is cached") - cached = await litellm.aembedding( - "openai/text-embedding-ada-002", input=input_text, caching=True, api_key="sk-unit" - ) - assert cached._hidden_params["cache_hit"] is True - assert cached.data[0]["embedding"] == [0.1, 0.2, 0.3] - assert embeddings_route.call_count == 1 - - await litellm.aembedding( - "openai/text-embedding-ada-002", input=input_text, caching=True, user="charlie", api_key="sk-unit" - ) - assert embeddings_route.call_count == 2 diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 6bd1468cfc1..49f9e99cf48 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -1659,7 +1659,7 @@ def test_ProxyConfig__init_cache_sets_litellm_cache(monkeypatch): def test_ProxyConfig__init_cache_invalid_params_raises(): pc = ProxyConfig() - with pytest.raises(AttributeError): + with pytest.raises(ValueError, match="Unsupported cache type 'this-cache-type-does-not-exist'"): pc._init_cache(cache_params={"type": "this-cache-type-does-not-exist"}) diff --git a/uv.lock b/uv.lock index b3e8ddc492c..f36fd3495a9 100644 --- a/uv.lock +++ b/uv.lock @@ -1955,15 +1955,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] -[[package]] -name = "diskcache" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, -] - [[package]] name = "distro" version = "1.9.0" @@ -4995,9 +4986,6 @@ dependencies = [ bedrock-realtime = [ { name = "aws-sdk-bedrock-runtime", extra = ["awscrt"], marker = "python_full_version >= '3.12'" }, ] -caching = [ - { name = "diskcache" }, -] cli = [ { name = "inquirerpy" }, { name = "keyring" }, @@ -5250,7 +5238,6 @@ requires-dist = [ { name = "cryptography", marker = "extra == 'proxy'", specifier = ">=49.0.0,<51.0" }, { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = ">=4.8.2,<5.0" }, { name = "detect-secrets", marker = "extra == 'proxy-runtime'", specifier = ">=1.5.0,<2.0" }, - { name = "diskcache", marker = "extra == 'caching'", specifier = ">=5.6.3,<6.0" }, { name = "expression", marker = "extra == 'proxy'", specifier = ">=5.6.0,<6.0" }, { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, @@ -5343,7 +5330,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] -provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] +provides-extras = ["proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }] From d76db470f9e1b3bf3916f95fe94008abf08a2cf0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 18:06:28 -0700 Subject: [PATCH 07/25] test: update integration tests left stale by recent product changes (#49143) * test: update integration tests left stale by recent product changes * test: expect proxy Messages calls to stay on Python with Rust enabled --- ...uto_router_databricks_classifier_writes.py | 23 ++++- .../integration/mcp/test_mcp_access_matrix.py | 31 ++++-- tests/integration/mcp/test_mcp_credentials.py | 96 ++++++++++++------- .../integration/mcp/test_mcp_llm_endpoints.py | 35 ++++--- tests/integration/mcp/test_mcp_oauth_flows.py | 56 +++++++---- tests/integration/mcp/test_mcp_resilience.py | 66 +++++++++---- tests/integration/mcp/test_mcp_toolsets.py | 23 ++++- .../observability/test_guardrail_effects.py | 9 +- .../test_databricks_decisions_wire.py | 40 ++++++-- .../test_decisions_openai_format_wire.py | 13 ++- ...test_idle_upstream_keepalive_edges_wire.py | 5 +- ...t_idle_upstream_keepalive_override_wire.py | 2 +- ...e_upstream_reconnect_rust_messages_wire.py | 56 ----------- .../providers/test_inferenceapis_wire.py | 10 +- ...nses_bridge_tool_search_undeferred_wire.py | 33 +++++-- .../test_responses_bridge_tool_search_wire.py | 8 +- ...st_rust_anthropic_safeguards_beta_chaos.py | 2 +- ...est_rust_anthropic_safeguards_beta_wire.py | 18 ++-- .../test_vertex_credentials_source_wire.py | 4 +- .../golden/daily_activity_team_paginated.json | 29 ------ .../golden/daily_activity_user_paginated.json | 29 ------ .../integration/spend/test_shutdown_flush.py | 16 +++- .../chat_completions/bases/openai.py | 7 ++ .../chat_completions/cross_model/openai.py | 63 ++++++++++-- 24 files changed, 415 insertions(+), 259 deletions(-) delete mode 100644 tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py diff --git a/tests/integration/management/test_auto_router_databricks_classifier_writes.py b/tests/integration/management/test_auto_router_databricks_classifier_writes.py index af486d90a7e..d17ce947671 100644 --- a/tests/integration/management/test_auto_router_databricks_classifier_writes.py +++ b/tests/integration/management/test_auto_router_databricks_classifier_writes.py @@ -3,6 +3,7 @@ import json import threading import uuid from collections.abc import Callable, Mapping, Sequence +from pathlib import Path from queue import SimpleQueue from typing import Final @@ -64,6 +65,21 @@ _ROWS_QUERY: Final = ( ) +def _number(value: JsonValue) -> float: + assert isinstance(value, (int, float)) and not isinstance(value, bool), value + return float(value) + + +def _endpoint_cost() -> float: + prices: Final = object_value( + json.loads(Path("model_prices_and_context_window.json").read_text())[f"databricks/{_ENDPOINT}"] + ) + usage: Final = object_value(_ENDPOINT_ANSWER["usage"]) + return _number(usage["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( + usage["output_tokens"] + ) * _number(prices["output_cost_per_token"]) + + def _classifier(request: Request) -> Reply: answer: Final = _ENDPOINT_ANSWER if request.target.endswith("/invocations") else _AI_DECIDE_ANSWER return Reply(body=json.dumps(answer).encode()) @@ -221,11 +237,14 @@ def test_validate_names_each_databricks_classifier_misconfiguration(gateway: Gat @pytest.mark.parametrize( ("model", "target", "cost"), - [(_MODEL, _ROUTE, {}), (_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": 0.0})], + [ + (_MODEL, _ROUTE, {}), + (_ENDPOINT, f"/{_ENDPOINT}/invocations", {"classifier_cost": pytest.approx(_endpoint_cost())}), + ], ids=["ai_decide", "serving_endpoint"], ) def test_test_routing_classifies_through_databricks_without_calling_the_tier_it_picked( - gateway: Gateway, model: str, target: str, cost: dict[str, JsonValue] + gateway: Gateway, model: str, target: str, cost: dict[str, object] ) -> None: prompt: Final = f"design a distributed cache {uuid.uuid4().hex}" with ( diff --git a/tests/integration/mcp/test_mcp_access_matrix.py b/tests/integration/mcp/test_mcp_access_matrix.py index e59f3e52a11..762fbee384d 100644 --- a/tests/integration/mcp/test_mcp_access_matrix.py +++ b/tests/integration/mcp/test_mcp_access_matrix.py @@ -181,7 +181,6 @@ def test_same_tool_name_on_two_servers_routes_by_prefix(gateway: Gateway) -> Non _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -217,13 +216,26 @@ def _echoed_description(outcome: Outcome) -> str: return found.group(1) +def _add_described_anew_per_fetch(described: str) -> ScriptedTool: + return ScriptedTool("add", lambda _: text_result("3"), description=lambda _: f"{described} {uuid.uuid4().hex}") + + +def _fetched(described: str) -> re.Pattern[str]: + return re.compile(rf"{re.escape(described)} [0-9a-f]{{32}}") + + +def _listed_description(listing: Outcome, described: str) -> str: + found: Final = _fetched(described).search(listing.raw) + assert found is not None, listing.raw + return found.group(0) + + @pytest.mark.parametrize("subject", SUBJECTS) def test_each_subjects_call_is_evaluated_only_against_the_catalog_its_own_listing_served( echo_rig: Gateway, subject: Subject ) -> None: described: Final = "Adds under grant " + uuid.uuid4().hex[:8] - tool: Final = ScriptedTool("add", lambda _: text_result("3"), description=described) - with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch(described)) as peer, echo_rig.scenario() as scenario: group: Final = "grp" + uuid.uuid4().hex[:8] alias: Final = "cat" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias, mcp_access_groups=[group]) @@ -235,8 +247,10 @@ def test_each_subjects_call_is_evaluated_only_against_the_catalog_its_own_listin cold: Final = _echoed_description(reach.call(f"{alias}-add", {"probe": _PROBE})) listed: Final = reach.list_tools() assert listed.ok and f"{alias}-add" in listed.tools, listed.raw + served: Final = _listed_description(listed, described) warm: Final = _echoed_description(reach.call(f"{alias}-add", {"probe": _PROBE})) - assert (cold, warm) == (_UNLISTED, described), (cold, warm) + assert _fetched(described).fullmatch(cold) and cold != served, (cold, served) + assert warm == served, (cold, served, warm) assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" @@ -244,8 +258,7 @@ def test_end_users_of_one_key_share_its_catalog_slot_because_the_identity_exclud echo_rig: Gateway, ) -> None: described: Final = "Adds for end users " + uuid.uuid4().hex[:8] - tool: Final = ScriptedTool("add", lambda _: text_result("3"), description=described) - with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch(described)) as peer, echo_rig.scenario() as scenario: alias: Final = "eu" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias) granted: Final = grant(scenario, "end_user", (identity,), (identity,)) @@ -254,10 +267,12 @@ def test_end_users_of_one_key_share_its_catalog_slot_because_the_identity_exclud echo_rig, granted.key, "mcp", alias, {"x-litellm-end-user-id": "integration-" + uuid.uuid4().hex[:10]} ) assert second.initialize().ok - assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == _UNLISTED + cold: Final = _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) listed: Final = first.list_tools() assert listed.ok and f"{alias}-add" in listed.tools, listed.raw - assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == described, ( + served: Final = _listed_description(listed, described) + assert _fetched(described).fullmatch(cold) and cold != served, (cold, served) + assert _echoed_description(second.call(f"{alias}-add", {"probe": _PROBE})) == served, ( "the end-user header is intentionally not part of the catalog identity: one key, one slot" ) assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/mcp/test_mcp_credentials.py b/tests/integration/mcp/test_mcp_credentials.py index 63f1b0024c0..08dd5234b68 100644 --- a/tests/integration/mcp/test_mcp_credentials.py +++ b/tests/integration/mcp/test_mcp_credentials.py @@ -448,7 +448,6 @@ def test_deprecated_string_x_mcp_auth_lists_a_byok_server_for_a_key_without_a_us _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -488,6 +487,28 @@ _PROBE_ARGUMENTS: Final = {"probe": _PROBE} _HEADERS: Final = TypeAdapter(dict[str, str]) +def _add_described_anew_per_fetch_for(header: str) -> ScriptedTool: + return ScriptedTool( + "add", + lambda _: text_result("3"), + description=lambda headers: f"Adds for {headers.get(header, 'nobody')} at {uuid.uuid4().hex}", + ) + + +def _fetched_for(credential: str) -> re.Pattern[str]: + return re.compile(rf"Adds for {re.escape(credential)} at [0-9a-f]{{32}}") + + +def _listed_description(listing: Outcome, credential: str) -> str: + found: Final = _fetched_for(credential).search(listing.raw) + assert found is not None, listing.raw + return found.group(0) + + +def _sent_header(request: dict[str, object], name: str) -> str | None: + return _HEADERS.validate_python(request["headers"]).get(name) + + def _store_byok_credential(scenario: Scenario, identity: str, key: str, secret: str) -> None: stored: Final = scenario.gateway.client.post( f"/v1/mcp/server/{identity}/user-credential", json={"credential": secret}, headers={"x-litellm-api-key": key} @@ -499,7 +520,7 @@ def _store_byok_credential(scenario: Scenario, identity: str, key: str, secret: def test_rotating_the_credential_drops_the_callers_listing_until_it_lists_again(echo_rig: Gateway) -> None: - with mcp_peer() as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch_for("authorization")) as peer, echo_rig.scenario() as scenario: first: Final = "cred-" + uuid.uuid4().hex second: Final = "cred-" + uuid.uuid4().hex alias: Final = "rot" + uuid.uuid4().hex[:8] @@ -508,53 +529,62 @@ def test_rotating_the_credential_drops_the_callers_listing_until_it_lists_again( ) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) caller: Final = McpCaller(echo_rig, key, "mcp", alias) - assert caller.list_tools().ok - assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers" + listed: Final = caller.list_tools() + assert listed.ok, listed.raw + served: Final = _listed_description(listed, f"Bearer {first}") + assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served rotated: Final = echo_rig.request( "PUT", "/v1/mcp/server", {"server_id": identity, "credentials": {"auth_value": second}} ) assert rotated.status_code == 202, rotated.text eventually( - lambda: _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)), lambda seen: seen == _UNLISTED + lambda: _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)), + lambda seen: _fetched_for(f"Bearer {second}").fullmatch(seen) is not None, ) peer.drain() - assert caller.list_tools().ok - assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers" + relisting: Final = caller.list_tools() + assert relisting.ok, relisting.raw + served_again: Final = _listed_description(relisting, f"Bearer {second}") + assert _echoed_description(caller.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served_again relisted: Final = _listings(peer) assert len(relisted) == 1, relisted - assert _header(relisted[0], b"authorization") == f"Bearer {second}".encode(), relisted[0]["headers"] + assert _sent_header(relisted[0], "authorization") == f"Bearer {second}", relisted[0]["headers"] def test_byok_callers_are_evaluated_against_their_own_listing_and_the_stored_secret_never_keys_the_slot( echo_rig: Gateway, ) -> None: - with mcp_peer() as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch_for("x-api-key")) as peer, echo_rig.scenario() as scenario: alias: Final = "byok" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias, auth_type="api_key", is_byok=True) owner_key: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]}) stranger_key: Final = scenario.key(user_id=scenario.user(), object_permission={"mcp_servers": [identity]}) owner_secret: Final = "byok-" + uuid.uuid4().hex + stranger_secret: Final = "byok-" + uuid.uuid4().hex replacement: Final = "byok-" + uuid.uuid4().hex _store_byok_credential(scenario, identity, owner_key, owner_secret) - _store_byok_credential(scenario, identity, stranger_key, "byok-" + uuid.uuid4().hex) + _store_byok_credential(scenario, identity, stranger_key, stranger_secret) owner: Final = McpCaller(echo_rig, owner_key, "mcp", alias) stranger: Final = McpCaller(echo_rig, stranger_key, "mcp", alias) peer.drain() - assert owner.list_tools().ok + listed: Final = owner.list_tools() + assert listed.ok, listed.raw listings: Final = _listings(peer) - assert [_header(item, b"x-api-key") for item in listings] == [owner_secret.encode()], listings + assert [_sent_header(item, "x-api-key") for item in listings] == [owner_secret], listings + served: Final = _listed_description(listed, owner_secret) own: Final = _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) other: Final = _echoed_description(stranger.call(f"{alias}-add", _PROBE_ARGUMENTS)) - assert (own, other) == ("Add two integers", _UNLISTED), (own, other) + assert own == served, (own, served) + assert _fetched_for(stranger_secret).fullmatch(other), (other, served) _store_byok_credential(scenario, identity, owner_key, replacement) - assert _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) == "Add two integers", ( - "the slot is keyed by the client-supplied header, never by the stored credential" - ) sent: Final = eventually( lambda: (owner.call(f"{alias}-add", ADD).ok, tool_calls(peer.drain())), - lambda value: any(_header(call, b"x-api-key") == replacement.encode() for call in value[1]), + lambda value: any(_sent_header(call, "x-api-key") == replacement for call in value[1]), ) assert sent[0], sent + assert _echoed_description(owner.call(f"{alias}-add", _PROBE_ARGUMENTS)) == served, ( + "the slot is keyed by the client-supplied header, never by the stored credential" + ) def test_callers_with_different_server_scoped_auth_headers_are_evaluated_against_their_own_listings( @@ -585,12 +615,7 @@ def test_callers_with_different_server_scoped_auth_headers_are_evaluated_against def test_deprecated_string_x_mcp_auth_callers_on_a_user_less_key_own_separate_listings(echo_rig: Gateway) -> None: - tool: Final = ScriptedTool( - "add", - lambda _: text_result("3"), - description=lambda headers: "Adds for " + headers.get("authorization", "nobody"), - ) - with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: + with scripted_peer(_add_described_anew_per_fetch_for("authorization")) as peer, echo_rig.scenario() as scenario: alias: Final = "legacy" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias, auth_type="bearer_token") key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) @@ -598,28 +623,33 @@ def test_deprecated_string_x_mcp_auth_callers_on_a_user_less_key_own_separate_li second_token: Final = "second-" + uuid.uuid4().hex first: Final = McpCaller(echo_rig, key, "mcp", alias, {"x-mcp-auth": f"Bearer {first_token}"}) second: Final = McpCaller(echo_rig, key, "mcp", alias, {"x-mcp-auth": f"Bearer {second_token}"}) - assert _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)) == _UNLISTED + first_cold: Final = _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)) + assert _fetched_for(f"Bearer {first_token}").fullmatch(first_cold), first_cold peer.drain() - assert first.list_tools().ok + first_listing: Final = first.list_tools() + assert first_listing.ok, first_listing.raw listings: Final = _listings(peer) assert len(listings) == 1, listings - listed_with: Final = _HEADERS.validate_python(listings[0]["headers"]) - assert listed_with.get("authorization") == f"Bearer {first_token}", listed_with + assert _sent_header(listings[0], "authorization") == f"Bearer {first_token}", listings[0]["headers"] + first_served: Final = _listed_description(first_listing, f"Bearer {first_token}") warm: Final = eventually( lambda: _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)), - lambda seen: seen != _UNLISTED, + lambda seen: seen == first_served, ) - assert warm == f"Adds for Bearer {first_token}", warm - assert _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)) == _UNLISTED - assert second.list_tools().ok + assert warm == first_served != first_cold, (first_cold, first_served, warm) + second_cold: Final = _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)) + assert _fetched_for(f"Bearer {second_token}").fullmatch(second_cold), (second_cold, first_served) + second_listing: Final = second.list_tools() + assert second_listing.ok, second_listing.raw + second_served: Final = _listed_description(second_listing, f"Bearer {second_token}") seen: Final = eventually( lambda: ( _echoed_description(first.call(f"{alias}-add", _PROBE_ARGUMENTS)), _echoed_description(second.call(f"{alias}-add", _PROBE_ARGUMENTS)), ), - lambda pair: _UNLISTED not in pair, + lambda pair: pair == (first_served, second_served), ) - assert seen == (f"Adds for Bearer {first_token}", f"Adds for Bearer {second_token}"), seen + assert seen == (first_served, second_served), seen assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index f751d1f8f4c..4b967802893 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -1,5 +1,6 @@ import asyncio import json +import re import uuid from collections.abc import Callable, Generator, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor @@ -1408,10 +1409,6 @@ REPORT: Final[tuple[str, dict[str, JsonValue]]] = ( "Write one report", {"type": "object", "properties": {"query": {"type": "string"}, "format": {}}}, ) -COLD: Final[tuple[str, dict[str, JsonValue]]] = ( - "", - {"type": "object", "properties": {}, "additionalProperties": False}, -) RELOAD_FAST: Final = {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "5"} CALL_ID: Final = "x-litellm-call-id" T = TypeVar("T") @@ -1577,12 +1574,14 @@ def _echoed(value: JsonValue) -> Echo: @contextmanager -def _bridge_rig(hooked: Hooked, bridge: Bridge) -> Generator[BridgeRig, None, None]: +def _bridge_rig( + hooked: Hooked, bridge: Bridge, lookup_description: str | Callable[[Mapping[str, str]], str] = LOOKUP[0] +) -> Generator[BridgeRig, None, None]: alias: Final = "brg" + uuid.uuid4().hex[:8] lookup: Final = ScriptedTool( "lookup", lambda params: text_result("found:" + json.dumps(JSON_OBJECT.validate_python(params)["arguments"])), - description=LOOKUP[0], + description=lookup_description, input_schema=LOOKUP[1], ) report: Final = ScriptedTool( @@ -1753,23 +1752,35 @@ def test_bridge_hook_sees_the_definition_of_the_tool_filtered_for_that_request( assert _spend_row(key, found.call_id)["status"] == "success" +def _fetched_lookup(description: JsonValue) -> bool: + return isinstance(description, str) and re.fullmatch(rf"{LOOKUP[0]} [0-9a-f]{{32}}", description) is not None + + @pytest.mark.parametrize("bridge", BRIDGES) def test_direct_call_after_bridge_only_discovery_stays_cold(hooked: Hooked, bridge: Bridge) -> None: - with _bridge_rig(hooked, bridge) as rig: + with _bridge_rig(hooked, bridge, lambda _: f"{LOOKUP[0]} {uuid.uuid4().hex}") as rig: key: Final = _bridge_key(rig) workers: Final = _both_workers(rig.hooked.proxy) bridged: Final = _ask(rig, key, HOOK_PROBE, [rig.mcp("lookup")], False, "sync") - assert _echoed(bridged.text) == _served(LOOKUP), bridged + discovered: Final = _echoed(bridged.text) + offered: Final = rig.upstream_by_prompt() + assert _fetched_lookup(discovered[0]) and discovered[1] == _served(LOOKUP)[1], bridged + assert set(offered) == {HOOK_PROBE} and all( + definitions == ((rig.tool("lookup"), *discovered),) for definitions in offered[HOOK_PROBE] + ), (discovered, offered) direct: Final = eventually( lambda: tuple( _on_worker(rig.hooked.proxy, lambda client: _direct_probe(rig, key, "lookup", client)) for _ in range(6) ), lambda seen: frozenset(pid for pid, _ in seen) == workers, ) - assert all(echo == COLD for _, echo in direct) and frozenset(pid for pid, _ in direct) == workers, ( - direct, - workers, - ) + assert ( + all( + _fetched_lookup(description) and description != discovered[0] and parameters == discovered[1] + for _, (description, parameters) in direct + ) + and frozenset(pid for pid, _ in direct) == workers + ), (direct, discovered, workers) assert rig.peer_calls() == () diff --git a/tests/integration/mcp/test_mcp_oauth_flows.py b/tests/integration/mcp/test_mcp_oauth_flows.py index 90a462472b2..0e174ac37aa 100644 --- a/tests/integration/mcp/test_mcp_oauth_flows.py +++ b/tests/integration/mcp/test_mcp_oauth_flows.py @@ -19,7 +19,7 @@ import jwt import pytest import yaml from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, generate_private_key -from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment from integration._support.database import read_rows from integration._support.mcp import ( ENTRY_POINTS, @@ -786,13 +786,20 @@ def test_dcr_bridge_relays_client_registration_and_advertises_gateway_endpoints( assert "S256" in issuer.json()["code_challenge_methods_supported"] -def _ui_session_cookie(gateway: Gateway, user_id: str) -> dict[str, str]: - claims: Final = {"user_id": user_id, "login_method": "username_password", "exp": int(time.time()) + 600} - return {"token": jwt.encode(claims, gateway.key, algorithm="HS256")} +def _ui_session_cookie(scenario: Scenario, user_id: str) -> dict[str, str]: + session_key: Final = scenario.key(user_id=user_id, duration="10m") + claims: Final = { + "user_id": user_id, + "key": session_key, + "login_method": "username_password", + "exp": int(time.time()) + 600, + } + return {"token": jwt.encode(claims, scenario.gateway.key, algorithm="HS256")} def _gateway_session_sign_in( gateway: Gateway, + scenario: Scenario, user_id: str, resource: str | None = None, *, @@ -806,7 +813,7 @@ def _gateway_session_sign_in( assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) - cookies: Final = _ui_session_cookie(gateway, user_id) + cookies: Final = _ui_session_cookie(scenario, user_id) started: Final = gateway.client.get( authorization_endpoint, params={ @@ -843,8 +850,8 @@ def _gateway_session_sign_in( return _response_object(issued), client_id -def _gateway_session_bearer(gateway: Gateway, user_id: str, resource: str | None = None) -> str: - issued, _ = _gateway_session_sign_in(gateway, user_id, resource) +def _gateway_session_bearer(gateway: Gateway, scenario: Scenario, user_id: str, resource: str | None = None) -> str: + issued, _ = _gateway_session_sign_in(gateway, scenario, user_id, resource) return _issued_token(issued) @@ -887,7 +894,7 @@ def test_spec_client_discovers_the_aggregate_authorization_server_and_signs_in_t assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) - cookies: Final = _ui_session_cookie(gateway, member) + cookies: Final = _ui_session_cookie(scenario, member) started: Final = gateway.client.get( authorization_endpoint, params={ @@ -997,7 +1004,7 @@ def test_per_server_resource_scopes_the_session_bearer_to_the_named_server(gatew assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(48)) - cookies: Final = _ui_session_cookie(gateway, member) + cookies: Final = _ui_session_cookie(scenario, member) started: Final = gateway.client.get( "/authorize/mcp-session", params={ @@ -1110,7 +1117,7 @@ def test_rotated_session_refresh_token_cannot_be_replayed_after_revocation(gatew alias: Final = "rt6" + uuid.uuid4().hex[:10] identity: Final = register_mcp(scenario, peer, alias) member: Final = scenario.member(scenario.team(object_permission={"mcp_servers": [identity]})) - issued, client_id = _gateway_session_sign_in(gateway, member) + issued, client_id = _gateway_session_sign_in(gateway, scenario, member) r1: Final = str(issued["refresh_token"]) a1: Final = str(issued["access_token"]) assert r1.startswith("llm_srefresh_"), r1 @@ -1207,7 +1214,7 @@ def test_lite_logout_revokes_the_cli_refresh_token_sent_the_way_the_cli_sends_it assert registered.status_code in (200, 201), registered.text client_id: Final = str(_response_object(registered)["client_id"]) pkce: Final = _Pkce(secrets.token_urlsafe(64)) - cookies: Final = _ui_session_cookie(gateway, member) + cookies: Final = _ui_session_cookie(scenario, member) consent: Final = gateway.client.get( str(contract["authorization_endpoint"]), params={ @@ -1362,6 +1369,7 @@ def test_gateway_session_bearer_of_a_team_member_is_served_the_team_toolset_on_i token_endpoint: Final = str(authorization_body["token_endpoint"]) issued, client_id = _gateway_session_sign_in( gateway, + scenario, member, resource, registration_endpoint=str(authorization_body["registration_endpoint"]), @@ -1413,7 +1421,7 @@ def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_a inside_id: Final = create_toolset(scenario, ((inside_server, "add"),), toolset_name=inside_name) outside_id: Final = create_toolset(scenario, ((outside_server, "add"),), toolset_name=outside_name) member: Final = scenario.member(scenario.team(object_permission={"mcp_toolsets": [inside_id, outside_id]})) - bearer: Final = _gateway_session_bearer(gateway, member, resource=f"{_base(gateway)}/{inside}/mcp") + bearer: Final = _gateway_session_bearer(gateway, scenario, member, resource=f"{_base(gateway)}/{inside}/mcp") assert bearer.startswith("llm_session_"), bearer[:16] listed: Final = _toolset_rpc(gateway, bearer, inside_name, "tools/list", {}) assert listed.tools == (f"{inside}-add",), listed.raw @@ -1430,7 +1438,6 @@ def test_resource_scoped_session_bearer_opens_a_team_toolset_inside_its_server_a _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -1466,8 +1473,22 @@ def _echoed_description(outcome: Outcome) -> str: return found.group(1) +def _add_described_for_the_exchanged_subject(auth: AuthorizationServer) -> ScriptedTool: + def described(headers: Mapping[str, str]) -> str: + token: Final = headers.get("authorization", "").removeprefix("Bearer ") + with auth.lock: + grant: Final = auth.access_tokens.get(token, {}) + return "Add for " + grant.get("subject", "nobody") + + return ScriptedTool("add", lambda _: text_result("must not run"), description=described) + + def test_token_exchange_callers_with_different_subject_tokens_own_separate_listings(echo_rig: Gateway) -> None: - with mcp_peer() as peer, oauth_server() as auth, echo_rig.scenario() as scenario: + with ( + oauth_server() as auth, + scripted_peer(_add_described_for_the_exchanged_subject(auth)) as peer, + echo_rig.scenario() as scenario, + ): alias: Final = "te" + uuid.uuid4().hex[:8] identity: Final = register_mcp( scenario, @@ -1479,19 +1500,20 @@ def test_token_exchange_callers_with_different_subject_tokens_own_separate_listi ) key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) first_subject: Final = "subject-" + uuid.uuid4().hex + second_subject: Final = "subject-" + uuid.uuid4().hex first: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": f"Bearer {first_subject}"}) - second: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": "Bearer subject-" + uuid.uuid4().hex}) + second: Final = McpCaller(echo_rig, key, "mcp", alias, {"Authorization": f"Bearer {second_subject}"}) auth.drain() assert first.list_tools().ok assert [request["subject_token"] for request in auth.token_requests()] == [first_subject] probe: Final = {"probe": _PROBE} own: Final = _echoed_description(first.call(f"{alias}-add", probe)) other: Final = _echoed_description(second.call(f"{alias}-add", probe)) - assert (own, other) == ("Add two integers", _UNLISTED), ( + assert (own, other) == (f"Add for exchanged:{first_subject}", f"Add for exchanged:{second_subject}"), ( "the caller bearer is part of the identity on a token-exchange server: one subject, one slot" ) assert second.list_tools().ok - assert _echoed_description(second.call(f"{alias}-add", probe)) == "Add two integers" + assert _echoed_description(second.call(f"{alias}-add", probe)) == f"Add for exchanged:{second_subject}" assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/mcp/test_mcp_resilience.py b/tests/integration/mcp/test_mcp_resilience.py index c335012d787..145994f1b29 100644 --- a/tests/integration/mcp/test_mcp_resilience.py +++ b/tests/integration/mcp/test_mcp_resilience.py @@ -1,4 +1,5 @@ import itertools +import re import uuid from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor @@ -37,6 +38,7 @@ pytestmark: Final = pytest.mark.shares_worker_state _ECHO: Final = "catalog-echo:" _PROBE: Final = "catalog-probe" _DESCRIPTION: Final = "Look up one record" +_FETCHED: Final = re.compile(rf"{_DESCRIPTION} [0-9a-f]{{32}}") _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" ' texts = list(inputs.get("texts") or [])\n' @@ -69,6 +71,20 @@ class _RpcReply(BaseModel): result: _Result +class _ListedTool(BaseModel): + name: str + description: str | None = None + + +class _ListedTools(BaseModel): + tools: tuple[_ListedTool, ...] + + +class _ListReply(BaseModel): + id: int + result: _ListedTools + + class _SessionsReport(BaseModel): worker_pid: int @@ -207,9 +223,13 @@ def test_peer_restart_on_the_same_url_is_picked_up_without_gateway_restart(gatew assert len(tool_calls(replacement.drain())) >= 1 -def _rpc_reply(raw: str) -> _RpcReply: +def _rpc_payload(raw: str) -> str: data: Final = tuple(line[5:].strip() for line in raw.splitlines() if line.startswith("data:")) - return _RpcReply.model_validate_json(data[-1] if data else raw) + return data[-1] if data else raw + + +def _rpc_reply(raw: str) -> _RpcReply: + return _RpcReply.model_validate_json(_rpc_payload(raw)) def _call_params(call: Mapping[str, object]) -> Mapping[str, object]: @@ -220,9 +240,18 @@ def _call_nonce(call: Mapping[str, object]) -> str: return _STRINGS.validate_python(_call_params(call)["arguments"])["nonce"] -def _listed(caller: McpCaller, name: str) -> None: +def _listed(caller: McpCaller, name: str) -> Outcome: listing: Final = eventually(caller.list_tools, lambda outcome: name in outcome.tools, seconds=45) assert listing.error is None, (caller.gateway.client.base_url, listing.raw) + return listing + + +def _listed_description(caller: McpCaller, name: str) -> str: + listing: Final = _listed(caller, name) + tools: Final = _ListReply.model_validate_json(_rpc_payload(listing.raw)).result.tools + described: Final = next(tool.description for tool in tools if tool.name == name) + assert described is not None and _FETCHED.fullmatch(described), listing.raw + return described @dataclass(frozen=True, slots=True) @@ -251,15 +280,18 @@ def _probed_description(worker: _Worker, name: str) -> str | None: return _ECHOED.validate_json(carrier.split(_ECHO, 1)[1])["description"] -def _catalog_is_cold(worker: _Worker, name: str) -> bool: - return not _probed_description(worker, name) +def _catalog_is_cold(worker: _Worker, name: str, listings: tuple[str, ...]) -> bool: + probed: Final = _probed_description(worker, name) + return probed is not None and _FETCHED.fullmatch(probed) is not None and probed not in listings @pytest.mark.timeout(600) def test_worker_restart_cools_its_listed_catalog_while_the_sibling_worker_keeps_serving( gateway: Gateway, echo_config: Path, tmp_path: Path ) -> None: - tool: Final = ScriptedTool("lookup", lambda _: text_result("found"), description=_DESCRIPTION) + tool: Final = ScriptedTool( + "lookup", lambda _: text_result("found"), description=lambda _: f"{_DESCRIPTION} {uuid.uuid4().hex}" + ) with scripted_peer(tool) as peer, gateway.scenario() as scenario: alias: Final = "cold" + uuid.uuid4().hex[:8] identity: Final = register_mcp(scenario, peer, alias) @@ -272,21 +304,23 @@ def test_worker_restart_cools_its_listed_catalog_while_the_sibling_worker_keeps_ assert first.pid != sibling.pid _served(first, name, "first-unlisted") _served(sibling, name, "sibling-unlisted") - assert _catalog_is_cold(first, name) and _catalog_is_cold(sibling, name) - _listed(first.caller, name) - assert _probed_description(first, name) == _DESCRIPTION - assert _catalog_is_cold(sibling, name), "a listing on one worker warmed its sibling" - _listed(sibling.caller, name) - assert _probed_description(sibling, name) == _DESCRIPTION + assert _catalog_is_cold(first, name, ()) and _catalog_is_cold(sibling, name, ()) + first_listing: Final = _listed_description(first.caller, name) + assert _probed_description(first, name) == first_listing + assert _catalog_is_cold(sibling, name, (first_listing,)), "a listing on one worker warmed its sibling" + sibling_listing: Final = _listed_description(sibling.caller, name) + assert _probed_description(sibling, name) == sibling_listing _served(sibling, name, "sibling-alone") with owned_proxy_process(gateway, tmp_path / "restarted", {}, config=echo_config) as restarted_proxy: restarted: Final = _worker(restarted_proxy.gateway, key, alias) assert restarted.pid not in (first.pid, sibling.pid) - assert _catalog_is_cold(restarted, name), "a restarted worker kept the old process's catalog" - assert _probed_description(sibling, name) == _DESCRIPTION + assert _catalog_is_cold(restarted, name, (first_listing, sibling_listing)), ( + "a restarted worker kept the old process's catalog" + ) + assert _probed_description(sibling, name) == sibling_listing _served(restarted, name, "restarted-unlisted") - _listed(restarted.caller, name) - assert _probed_description(restarted, name) == _DESCRIPTION + restarted_listing: Final = _listed_description(restarted.caller, name) + assert _probed_description(restarted, name) == restarted_listing calls: Final = tool_calls(peer.drain()) assert [_call_nonce(call) for call in calls] == [ "first-unlisted", diff --git a/tests/integration/mcp/test_mcp_toolsets.py b/tests/integration/mcp/test_mcp_toolsets.py index f275bd0e427..fc040810970 100644 --- a/tests/integration/mcp/test_mcp_toolsets.py +++ b/tests/integration/mcp/test_mcp_toolsets.py @@ -538,7 +538,6 @@ def test_owned_proxy_loss_retains_exit_status_and_first_http_failure(gateway: Ga _PROBE: Final = "catalog-probe" _ECHO: Final = "catalog-echo" -_UNLISTED: Final = "" _GUARDRAIL_CODE: Final = ( "def apply_guardrail(inputs, request_data, input_type):\n" f' if "{_PROBE}" not in list(inputs.get("texts") or []):\n' @@ -574,9 +573,21 @@ def _echoed_description(outcome: Outcome) -> str: return found.group(1) +def _fetched(described: str) -> re.Pattern[str]: + return re.compile(rf"{re.escape(described)} [0-9a-f]{{32}}") + + +def _listed_description(listing: Outcome, described: str) -> str: + found: Final = _fetched(described).search(listing.raw) + assert found is not None, listing.raw + return found.group(0) + + def test_a_team_keys_toolset_route_listing_feeds_its_own_calls_but_not_a_team_mates(echo_rig: Gateway) -> None: described: Final = "Adds for the team " + uuid.uuid4().hex[:8] - tool: Final = ScriptedTool("add", lambda _: text_result("9"), description=described) + tool: Final = ScriptedTool( + "add", lambda _: text_result("9"), description=lambda _: f"{described} {uuid.uuid4().hex}" + ) with scripted_peer(tool) as peer, echo_rig.scenario() as scenario: alias: Final = "lit6029echo" + uuid.uuid4().hex[:6] server_id: Final = register_mcp(scenario, peer, alias) @@ -587,10 +598,14 @@ def test_a_team_keys_toolset_route_listing_feeds_its_own_calls_but_not_a_team_ma _assert_team_grants_only(echo_rig, team_id, key, granted_id) listed: Final = _toolset_rpc(echo_rig, _bearer(key), granted_name, "tools/list", {}) assert listed.ok and listed.tools == (f"{alias}-add",), listed.raw + served: Final = _listed_description(listed, described) probe: Final[dict[str, object]] = {"name": f"{alias}-add", "arguments": {"probe": _PROBE}} own: Final = _echoed_description(_toolset_rpc(echo_rig, _bearer(key), granted_name, "tools/call", probe)) mate: Final = _echoed_description(_toolset_rpc(echo_rig, _bearer(team_mate), granted_name, "tools/call", probe)) - assert (own, mate) == (described, _UNLISTED), ( - "the slot is keyed by the hashed key, so a team-mate that never listed is handed nothing" + assert own == served, (own, served) + assert _fetched(described).fullmatch(mate) and mate != served, ( + "the slot is keyed by the hashed key, so a team-mate that never listed is not handed the key's listing", + mate, + served, ) assert tool_calls(peer.drain()) == (), "a blocked probe reached the peer" diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 537eb2bbf69..4f5c04bddcc 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1085,7 +1085,8 @@ def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway: mcp_peer() as peer, candidate.scenario() as scenario, ): - identity = register_mcp(scenario, peer, "guardrail" + uuid.uuid4().hex) + server = "guardrail" + uuid.uuid4().hex + identity = register_mcp(scenario, peer, server) permission = {"mcp_servers": [identity], "mcp_tool_search_enabled": True} key = scenario.key(object_permission=permission) key_selected = scenario.key(object_permission=permission, guardrails=[guardrail]) @@ -1116,8 +1117,10 @@ def test_request_selected_mcp_guardrail_blocks_direct_and_virtual_calls(gateway: ) calls = tuple(item for item in peer.drain() if item["body"].get("method") == "tools/call") if expected is None: - assert response.status_code == 400, response.text - assert "integration resolved add denied" in response.text, response.text + assert response.status_code == 403, response.text + assert response.json() == { + "detail": {"error": f"Tool add is unavailable after MCP guardrail checks for server {server}."} + }, response.text assert calls == (), "pre-call denial must prevent upstream execution" else: assert response.status_code == 200, response.text diff --git a/tests/integration/providers/test_databricks_decisions_wire.py b/tests/integration/providers/test_databricks_decisions_wire.py index f105092e9a3..7b54895adaf 100644 --- a/tests/integration/providers/test_databricks_decisions_wire.py +++ b/tests/integration/providers/test_databricks_decisions_wire.py @@ -1,3 +1,4 @@ +import json import threading import time import uuid @@ -67,7 +68,12 @@ _NON_BARE_NAMES: Final[tuple[tuple[str, str], ...]] = ( ) _DATABRICKS_ERRORS: Final[tuple[tuple[str, str, int, dict[str, JsonValue]], ...]] = ( (_MODEL, _ROUTE, 400, {"error_code": "BAD_REQUEST", "message": "Invalid input: questions must not be empty"}), - (_MODEL, _ROUTE, 403, {"error_code": "PERMISSION_DENIED", "message": "ai_decide is not enabled for this workspace"}), + ( + _MODEL, + _ROUTE, + 403, + {"error_code": "PERMISSION_DENIED", "message": "ai_decide is not enabled for this workspace"}, + ), ( _ENDPOINT, f"/{_ENDPOINT}/invocations", @@ -89,6 +95,15 @@ def _number(value: JsonValue) -> float: return float(value) +def _endpoint_spend() -> float: + prices: Final = object_value( + json.loads(Path("model_prices_and_context_window.json").read_text())[f"databricks/{_ENDPOINT}"] + ) + return _number(_USAGE["input_tokens"]) * _number(prices["input_cost_per_token"]) + _number( + _USAGE["output_tokens"] + ) * _number(prices["output_cost_per_token"]) + + def _register(scenario: Scenario, body: dict[str, JsonValue], *, status: int = 200) -> ScenarioHandle: handle: Final = register_scenario( f"databricks-{uuid.uuid4().hex[:12]}", JsonResponse(content_type="application/json", body=body, status=status) @@ -300,9 +315,14 @@ def test_an_openai_format_request_at_v1_decisions_reaches_the_serving_endpoint_a row["model_group"], row["api_base"], _number(row["spend"]), - ) == ("success", "adecisions", "databricks", deployment, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), ( - row - ) + ) == ( + "success", + "adecisions", + "databricks", + deployment, + f"{handle.api_base()}/{_ENDPOINT}/invocations", + pytest.approx(_endpoint_spend(), rel=1e-9), + ), row def test_an_openai_format_request_at_v1_decisions_reaches_the_ai_decide_route_without_a_model_field( @@ -381,6 +401,7 @@ def test_a_noul_probability_reaches_the_caller_as_the_system_one_noul_answer(gat def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bills_each_once(gateway: Gateway) -> None: + expected_spend: Final = _endpoint_spend() with gateway.scenario() as scenario: handle: Final = _register(scenario, _ANSWER_BODY) prefix: Final = _wildcard_deployment(scenario, api_base=handle.api_base(), api_key=_API_KEY) @@ -408,9 +429,14 @@ def test_a_wildcard_deployment_sends_each_request_to_the_named_endpoint_and_bill row["model_group"], row["api_base"], _number(row["spend"]), - ) == ("success", "asystemone", "databricks", model, f"{handle.api_base()}/{_ENDPOINT}/invocations", 0.0), ( - row - ) + ) == ( + "success", + "asystemone", + "databricks", + model, + f"{handle.api_base()}/{_ENDPOINT}/invocations", + pytest.approx(expected_spend, rel=1e-9), + ), row def test_a_wildcard_deployment_sends_each_ai_decide_request_to_the_route_and_logs_each_once(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_decisions_openai_format_wire.py b/tests/integration/providers/test_decisions_openai_format_wire.py index 2f82561d1b1..0c40fe06199 100644 --- a/tests/integration/providers/test_decisions_openai_format_wire.py +++ b/tests/integration/providers/test_decisions_openai_format_wire.py @@ -187,7 +187,16 @@ _PROVIDERS: Final = ( "cloudflare/@cf/cloudflare/clef", takes_images=True, ), - _Provider("hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None), + _Provider( + "hosted_vllm", + "hosted_vllm/Qwen/Qwen3-0.6B", + "/v1/systemone", + "Qwen/Qwen3-0.6B", + None, + False, + None, + takes_images=True, + ), _Provider( "databricks", "databricks/ai_decide", @@ -205,7 +214,7 @@ _PROVIDERS: Final = ( "databricks-openjev-qwen35-4b", _API_KEY, False, - None, + "databricks/databricks-openjev-qwen35-4b", ), _Provider( "azure_ai", "azure_ai/decision-1", "/providers/microsoft/v1/systemone", "decision-1", _API_KEY, False, None diff --git a/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py b/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py index fc27f0f6e1e..a5628d7764c 100644 --- a/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py +++ b/tests/integration/providers/test_idle_upstream_keepalive_edges_wire.py @@ -33,15 +33,14 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + REUSE_ROWS: Final = ( Row("reuse-chat-openai-sdk-sync", openai_deployment, chat_sdk), - Row("reuse-messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True), + Row("reuse-messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk), ) QUIET_ROWS: Final = ( Row("quiet-chat-openai-sdk-async-stream", openai_deployment, chat_sdk_stream, respond=quiet_stream_answer), Row( - "quiet-messages-rust-anthropic-sdk-async-stream", + "quiet-messages-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream, - rust=True, respond=quiet_stream_answer, ), ) diff --git a/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py b/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py index 8b5a69d729e..ddcf1ed7741 100644 --- a/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py +++ b/tests/integration/providers/test_idle_upstream_keepalive_override_wire.py @@ -27,7 +27,7 @@ pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + ROWS: Final = ( Row("override-chat-openai-sdk-sync", openai_deployment, chat_sdk), - Row("override-messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True), + Row("override-messages-anthropic-sdk-sync", anthropic_deployment, messages_sdk), Row("override-ocr-mistral-rust-bridge", ocr_deployment, ocr, rust=True), ) diff --git a/tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py b/tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py deleted file mode 100644 index 8d92fb609e9..00000000000 --- a/tests/integration/providers/test_idle_upstream_reconnect_rust_messages_wire.py +++ /dev/null @@ -1,56 +0,0 @@ -from collections.abc import Iterator -from typing import Final - -import pytest -from integration._support.client import Gateway -from integration._support.idle_gateway import Drop -from integration._support.process import graceful_stop_seconds -from integration.providers._idle_upstream_support import ( - IDLE_SECONDS, - Row, - StalePools, - answered, - anthropic_deployment, - expected_answers, - landed, - landed_once_undropped, - messages_sdk, - messages_sdk_stream, - served_by_rust, - spend_statuses, - stale_pools, -) - -pytestmark: Final = pytest.mark.timeout(int(2 * graceful_stop_seconds() + 120 + IDLE_SECONDS)) - -ROWS: Final = ( - Row("messages-rust-anthropic-sdk-sync", anthropic_deployment, messages_sdk, rust=True), - Row("messages-rust-anthropic-sdk-async-stream", anthropic_deployment, messages_sdk_stream, rust=True), - Row( - "messages-rust-upstream-resets-stale-connection", - anthropic_deployment, - messages_sdk, - drop=Drop.RESET, - rust=True, - ), -) - - -@pytest.fixture(scope="module") -def pools(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[StalePools]: - with stale_pools( - session_gateway, tmp_path_factory.mktemp("idle-upstream-rust"), ROWS, {"LITELLM_RUST": "1"} - ) as value: - yield value - - -@pytest.mark.parametrize("name", [row.name for row in ROWS]) -def test_rust_messages_after_upstream_idle_past_keepalive_open_a_fresh_connection_on_each_worker( - pools: StalePools, name: str -) -> None: - row: Final = pools.rows[name] - sents, arrivals = pools.probe(name) - assert answered(sents) == expected_answers(row, sents), sents - assert landed(arrivals) == landed_once_undropped(sents), arrivals - assert served_by_rust(sents) == (row.rust,) * len(sents), sents - assert spend_statuses(sents) == ("success",) * len(sents), sents diff --git a/tests/integration/providers/test_inferenceapis_wire.py b/tests/integration/providers/test_inferenceapis_wire.py index 8288c59ad5d..5f400d05957 100644 --- a/tests/integration/providers/test_inferenceapis_wire.py +++ b/tests/integration/providers/test_inferenceapis_wire.py @@ -30,6 +30,7 @@ _WEATHER_TOOL: Final[JsonValue] = { }, } _COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_ENDPOINT_SUPPORT_PATH: Final = Path(__file__).resolve().parents[3] / "litellm/provider_endpoints_support_backup.json" _COST_MAP: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _JSON_LIST: Final = TypeAdapter(list[JsonValue]) @@ -512,6 +513,12 @@ def _lists_inferenceapis(endpoint: Mapping[str, JsonValue]) -> bool: return any(object_value(provider)["slug"] == "inferenceapis" for provider in providers) +def _bundled_inferenceapis_endpoints() -> frozenset[str]: + providers: Final = object_value(_JSON_OBJECT.validate_json(_ENDPOINT_SUPPORT_PATH.read_bytes())["providers"]) + flags: Final = object_value(object_value(providers["inferenceapis"])["endpoints"]) + return frozenset(key for key, supported in flags.items() if supported) + + def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway: Gateway) -> None: providers: Final = gateway.request("GET", "/public/providers") assert providers.status_code == 200, providers.text @@ -538,7 +545,8 @@ def test_public_routes_list_the_provider_with_its_endpoints_and_fields(gateway: for endpoint in _JSON_LIST.validate_python(_JSON_OBJECT.validate_json(endpoints_response.content)["endpoints"]) ) listed: Final = frozenset(str(endpoint["key"]) for endpoint in endpoints if _lists_inferenceapis(endpoint)) - assert listed == {"chat_completions", "messages", "responses"} + assert {"chat_completions", "messages", "responses"} <= listed, listed + assert listed == _bundled_inferenceapis_endpoints() def test_openai_and_anthropic_sdk_clients_reach_the_vendor(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py index a7fc2b379ea..f6037dd4835 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_undeferred_wire.py @@ -113,10 +113,8 @@ def _namespace_tool() -> dict[str, JsonValue]: } -def _search_tools(route: Route, search: Search) -> tuple[dict[str, JsonValue], ...]: - if search == "absent": - return () - return (_tool_search(execution="client"),) if route == "anthropic" else (_tool_search(),) +def _search_tools(search: Search) -> tuple[dict[str, JsonValue], ...]: + return () if search == "absent" else (_tool_search(),) def _gemini_frame(marker: str) -> dict[str, JsonValue]: @@ -406,7 +404,6 @@ _UNDEFERRING: Final[tuple[tuple[Route, Search], ...]] = ( _KEEPING: Final[tuple[tuple[Route, Search], ...]] = ( ("unlisted_claude", "absent"), ("anthropic", "absent"), - ("anthropic", "dropped"), ("azure_ai", "absent"), ) @@ -428,7 +425,7 @@ def test_without_a_search_tool_on_the_wire_every_tool_arrives_without_defer_load call: Final = _Call(route, stream, uuid.uuid4().hex) with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, route, wire.url) - tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool()) + tools: Final = (*_search_tools(search), _function_tool(), _namespace_tool()) response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream)) _assert_served(response, call.marker, stream) request, body = _only_request(wire) @@ -442,7 +439,7 @@ def test_a_claude_route_sent_no_server_search_tool_keeps_each_tools_own_defer_lo call: Final = _Call(route, stream, uuid.uuid4().hex) with wire_server(_vendor(route)) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, route, wire.url) - tools: Final = (*_search_tools(route, search), _function_tool(), _namespace_tool(), _loaded_tool()) + tools: Final = (*_search_tools(search), _function_tool(), _namespace_tool(), _loaded_tool()) response: Final = _raw(gateway, _responses_body(model, call.marker, tools, stream)) _assert_served(response, call.marker, stream) request, body = _only_request(wire) @@ -456,6 +453,26 @@ def test_a_claude_route_sent_no_server_search_tool_keeps_each_tools_own_defer_lo ], sent +@pytest.mark.parametrize("stream", [False, True], ids=["sync", "stream"]) +def test_a_client_search_tool_on_anthropic_goes_out_as_a_function_and_holds_back_the_deferred_tools( + gateway: Gateway, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_claude) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "anthropic", wire.url) + tools: Final = (_tool_search(execution="client"), _function_tool(), _namespace_tool(), _loaded_tool()) + response: Final = _raw(gateway, _responses_body(model, marker, tools, stream)) + _assert_served(response, marker, stream) + request, body = _only_request(wire) + _assert_target("anthropic", stream, request, body) + sent: Final = _ITEMS.validate_python(body["tools"]) + assert [(tool["name"], tool.get("defer_loading")) for tool in sent] == [ + ("tool_search", None), + ("send_email", None), + ], sent + assert "defer_loading" not in request.body.decode(), body + + def test_a_false_defer_loading_never_reaches_an_openai_compatible_wire_as_a_key(gateway: Gateway) -> None: call: Final = _Call("mistral", False, uuid.uuid4().hex) with wire_server(_mistral) as wire, gateway.scenario() as scenario: @@ -682,7 +699,7 @@ def test_a_streamed_answer_closes_its_message_item_with_every_streamed_delta(gat async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served: - tools: Final = (*_search_tools(call.route, "dropped"), _function_tool(), _namespace_tool()) + tools: Final = (*_search_tools("dropped"), _function_tool(), _namespace_tool()) body: Final = _responses_body(model, call.marker, tools, call.stream) async with client.stream( "POST", "/v1/responses", json=body, headers={"Authorization": f"Bearer {key}"} diff --git a/tests/integration/providers/test_responses_bridge_tool_search_wire.py b/tests/integration/providers/test_responses_bridge_tool_search_wire.py index 03cd4ce9dc4..57d9655b7d5 100644 --- a/tests/integration/providers/test_responses_bridge_tool_search_wire.py +++ b/tests/integration/providers/test_responses_bridge_tool_search_wire.py @@ -835,15 +835,17 @@ def test_a_claude_the_cost_map_does_not_list_gets_the_tool_dropped_on_the_anthro _assert_dropped_on_wire(body, _DEFERRED_NAMES) -def test_client_execution_drops_the_tool_search_tool(gateway: Gateway) -> None: +def test_client_execution_offers_tool_search_as_a_function_and_holds_back_the_deferred_tools(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_Vendor().respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, "anthropic", wire.url, gateway.upstream_url) tools: Final = (_tool_search(execution="client"), _function_tool(defer_loading=False), _namespace_tool()) answer: Final = _raw_answer(gateway, _responses_body(model, marker, tools, False)) assert _output_types(answer.payload) == ["message", "function_call"], answer - _, body = _only_request(wire) - _assert_dropped_on_wire(body, _DEFERRED_NAMES) + request, body = _only_request(wire) + assert request.target == "/v1/messages", request.target + _assert_dropped_on_wire(body, ("tool_search", "get_weather")) + assert "defer_loading" not in request.body.decode(), body @pytest.mark.parametrize("drop_params", [False, True], ids=["refused", "dropped"]) diff --git a/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py b/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py index f90d68c1a63..4ddfd16f931 100644 --- a/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py +++ b/tests/integration/providers/test_rust_anthropic_safeguards_beta_chaos.py @@ -45,7 +45,7 @@ def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client, def _post(client: httpx.Client, key: str, marker: str) -> int: body: Final = safeguarded(messages_body(ANTHROPIC_DEPLOYMENT, marker), list(SAFEGUARDS)) response: Final = client.post("/v1/messages", json=body, headers={"Authorization": f"Bearer {key}"}) - assert response.status_code != 200 or rust_routed(response.headers), dict(response.headers) + assert response.status_code != 200 or not rust_routed(response.headers), dict(response.headers) return response.status_code diff --git a/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py b/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py index c13d4da7dbb..2941f8003ee 100644 --- a/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py +++ b/tests/integration/providers/test_rust_anthropic_safeguards_beta_wire.py @@ -155,7 +155,7 @@ def test_sdk_messages_with_safeguards_go_upstream_with_the_dangerous_tool_use_be extra_body={"safeguards": list(SAFEGUARDS), "tools": [dict(BASH_TOOL)]}, ) message: Final = raw.parse() - assert rust_routed(raw.headers), dict(raw.headers) + assert not rust_routed(raw.headers), dict(raw.headers) received: Final = json_object(message.model_dump_json()) assert message.id == f"msg_{marker}", received assert received.get("safeguard_results") == list(SAFEGUARD_RESULTS), received @@ -192,7 +192,7 @@ async def test_sdk_streamed_messages_with_safeguards_go_upstream_with_the_beta( ) as stream: text: Final = "".join([piece async for piece in stream.text_stream]) final: Final = await stream.get_final_message() - assert rust_routed(stream.response.headers), dict(stream.response.headers) + assert not rust_routed(stream.response.headers), dict(stream.response.headers) assert (final.id, text) == (f"msg_{marker}", REPLY_TEXT), (final, text) sent: Final = one_delivery(rig.wire, marker) assert (sent.target, sent.beta, sent.safeguards) == ( @@ -209,7 +209,7 @@ def test_raw_messages_merge_the_safeguards_beta_into_the_client_betas(rig: _Rig, body: Final = safeguarded(messages_body(ANTHROPIC_DEPLOYMENT, marker, stream=stream), list(SAFEGUARDS)) response: Final = _post(rig.gateway, body, beta=CLIENT_BETAS) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text and "safeguard_results" in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert (sent.target, sent.beta) == (ANTHROPIC_TARGET, MERGED_BETAS), sent @@ -221,7 +221,7 @@ def test_null_or_absent_safeguards_add_no_beta(rig: _Rig, extra: Mapping[str, Js marker: Final = new_marker() response: Final = _post(rig.gateway, {**messages_body(ANTHROPIC_DEPLOYMENT, marker), **extra}) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text and "safeguard_results" not in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert (sent.beta, sent.safeguards) == (None, None), sent @@ -253,7 +253,7 @@ def test_malformed_safeguards_still_add_the_beta_and_relay_the_vendor_verdict( control: Final = new_marker() after: Final = _post(rig.gateway, messages_body(ANTHROPIC_DEPLOYMENT, control)) assert after.status_code == 200, after.text - assert rust_routed(after.headers), dict(after.headers) + assert not rust_routed(after.headers), dict(after.headers) assert one_delivery(rig.wire, control).beta is None assert rig.gateway.request("GET", "/health/liveliness").status_code == 200 @@ -276,7 +276,7 @@ def test_duplicated_safeguards_keys_resolve_to_the_last_value(rig: _Rig, last: s }, ) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert ("safeguard_results" in response.text) == (last == "list"), response.text sent: Final = one_delivery(rig.wire, marker) assert sent.beta == (DANGEROUS_TOOL_USE if last == "list" else None), sent @@ -304,7 +304,7 @@ def test_vertex_messages_with_safeguards_keep_the_beta_from_the_shared_helper( body: Final = safeguarded(messages_body(VERTEX_DEPLOYMENT, marker, stream=stream), list(SAFEGUARDS)) response: Final = _post(rig.gateway, {**body, "tools": tools}) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text and "safeguard_results" in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert sent.target == (VERTEX_STREAM_TARGET if stream else VERTEX_TARGET), sent @@ -326,7 +326,7 @@ def test_deepseek_messages_forward_safeguards_without_adding_a_beta(rig: _Rig, b beta=beta, ) assert response.status_code == 200, response.text - assert rust_routed(response.headers), dict(response.headers) + assert not rust_routed(response.headers), dict(response.headers) assert f"msg_{marker}" in response.text, response.text sent: Final = one_delivery(rig.wire, marker) assert (sent.target, sent.api_key, sent.safeguards) == (DEEPSEEK_TARGET, DEEPSEEK_KEY, list(SAFEGUARDS)), sent @@ -370,7 +370,7 @@ def _wave(pool: ThreadPoolExecutor, gateway: Gateway, markers: tuple[str, ...], def _assert_served(answer: _Answer) -> None: assert answer.status == 200, answer assert REPLY_TEXT in answer.text, answer - assert answer.rust == (answer.path == "/v1/messages"), answer + assert not answer.rust, answer if answer.path == "/v1/messages": assert f"msg_{answer.marker}" in answer.text and "safeguard_results" in answer.text, answer diff --git a/tests/integration/providers/test_vertex_credentials_source_wire.py b/tests/integration/providers/test_vertex_credentials_source_wire.py index c66bb10f9a9..f3bc86eca19 100644 --- a/tests/integration/providers/test_vertex_credentials_source_wire.py +++ b/tests/integration/providers/test_vertex_credentials_source_wire.py @@ -416,7 +416,7 @@ def vertex_claude_peer(request: Request) -> Reply: @pytest.mark.timeout(2 * graceful_stop_seconds() + 120) -def test_rust_messages_route_loads_a_claude_key_file_and_names_unloadable_credentials( +def test_rust_enabled_messages_route_loads_a_claude_key_file_and_names_unloadable_credentials( gateway: Gateway, tmp_path: Path ) -> None: token_url: Final = gateway.upstream_url.rstrip("/") @@ -452,7 +452,7 @@ def test_rust_messages_route_loads_a_claude_key_file_and_names_unloadable_creden candidate, wire, ask(str(tmp_path / "missing.json"), "rust refuses a missing file"), - "failed to open application credentials file", + FILE_MESSAGE, str(tmp_path), ) refused( diff --git a/tests/integration/spend/golden/daily_activity_team_paginated.json b/tests/integration/spend/golden/daily_activity_team_paginated.json index 4d91b0da209..18ee30a74e6 100644 --- a/tests/integration/spend/golden/daily_activity_team_paginated.json +++ b/tests/integration/spend/golden/daily_activity_team_paginated.json @@ -177,35 +177,6 @@ "endpoints": { "/v1/chat/completions": { "api_key_breakdown": { - "__ptu_flat_cost__": { - "metadata": { - "key_alias": null, - "key_exists": false, - "team_id": null, - "user_email": null, - "user_id": null - }, - "metrics": { - "api_requests": 0, - "autorouter_savings_spend": 0.0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "completion_tokens": 0, - "compression_saved_tokens": 0, - "compression_savings_spend": 0.0, - "failed_requests": 0, - "flat_cost": 0.0, - "ptu_hours": 0.0, - "gateway_injected_caching_savings_spend": 0.0, - "prompt_caching_savings_spend": 0.0, - "prompt_tokens": 0, - "spend": 1000.0, - "successful_requests": 0, - "timed_requests": 0, - "total_response_time_ms": 0, - "total_tokens": 0 - } - }, "key-a": { "metadata": { "key_alias": "alias-a", diff --git a/tests/integration/spend/golden/daily_activity_user_paginated.json b/tests/integration/spend/golden/daily_activity_user_paginated.json index 00389289ebe..4f7b62b3147 100644 --- a/tests/integration/spend/golden/daily_activity_user_paginated.json +++ b/tests/integration/spend/golden/daily_activity_user_paginated.json @@ -177,35 +177,6 @@ "endpoints": { "/v1/chat/completions": { "api_key_breakdown": { - "__ptu_flat_cost__": { - "metadata": { - "key_alias": null, - "key_exists": false, - "team_id": null, - "user_email": null, - "user_id": null - }, - "metrics": { - "api_requests": 0, - "autorouter_savings_spend": 0.0, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0, - "completion_tokens": 0, - "compression_saved_tokens": 0, - "compression_savings_spend": 0.0, - "failed_requests": 0, - "flat_cost": 0.0, - "ptu_hours": 0.0, - "gateway_injected_caching_savings_spend": 0.0, - "prompt_caching_savings_spend": 0.0, - "prompt_tokens": 0, - "spend": 1000.0, - "successful_requests": 0, - "timed_requests": 0, - "total_response_time_ms": 0, - "total_tokens": 0 - } - }, "key-a": { "metadata": { "key_alias": "alias-a", diff --git a/tests/integration/spend/test_shutdown_flush.py b/tests/integration/spend/test_shutdown_flush.py index 5c2e9cf98da..174403d1873 100644 --- a/tests/integration/spend/test_shutdown_flush.py +++ b/tests/integration/spend/test_shutdown_flush.py @@ -16,7 +16,7 @@ import httpx import psycopg import pytest import yaml -from integration._support.client import Gateway, delete_key_if_present, eventually, string_value +from integration._support.client import Gateway, delete_key_if_present, eventually, object_value, string_value from integration._support.database import read_rows from integration._support.process import OwnedProxy, owned_proxy_process from integration._support.wire import Reply, Request, wire_server @@ -25,7 +25,6 @@ from psycopg import sql REQUESTS_WHILE_BLOCKED: Final = 6 CANCEL_LOG_LINE: Final = "in-flight scheduled job(s) for shutdown" BATCH_DRAINED_LOG_LINE: Final = f"flushed {REQUESTS_WHILE_BLOCKED} daily spend update items from in-memory queue" -MODEL_INSERT_ARRIVED_LOG_LINE: Final = "path=/model/new" COMMIT_DELAY_SECONDS: Final = 15 BURST_REQUESTS: Final = 30 @@ -356,8 +355,15 @@ class _Shutdown: for outcome in pool.map(lambda _: self.chat(), range(requests)): assert outcome is None - def logged(self, line: str, times: int = 1) -> bool: - return self.owned.log.read_text(errors="replace").count(line) >= times + def logged(self, line: str) -> bool: + return line in self.owned.log.read_text(errors="replace") + + def other_requests_in_flight(self) -> int: + response: Final = self.owned.gateway.request("GET", "/health/backlog") + assert response.status_code == 200, response.text + in_flight: Final = object_value(response.json())["in_flight_requests"] + assert isinstance(in_flight, int) + return in_flight - 1 def chat_while_spend_update_is_blocked(self, blocker: psycopg.Connection, table: str) -> None: blocker.execute(f'LOCK TABLE "{table}" IN EXCLUSIVE MODE') @@ -463,7 +469,7 @@ def test_daily_spend_batch_cancelled_while_waiting_for_a_pool_connection_is_writ eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 1, seconds=30) shutdown.chat_while_spend_update_is_blocked(memberships, "LiteLLM_TeamMembership") second: Final = shutdown.start_blocked_model_insert() - eventually(lambda: shutdown.logged(MODEL_INSERT_ARRIVED_LOG_LINE, times=2), lambda seen: seen, seconds=30) + eventually(shutdown.other_requests_in_flight, lambda in_flight: in_flight == 2, seconds=30) memberships.rollback() eventually(lambda: _waiting_on("LiteLLM_ProxyModelTable"), lambda waiting: waiting == 2, seconds=30) shutdown.terminate_once(lambda: shutdown.logged(BATCH_DRAINED_LOG_LINE), models.rollback) diff --git a/tests/integration/translation/chat_completions/bases/openai.py b/tests/integration/translation/chat_completions/bases/openai.py index 21886e4c774..25d8121cf31 100644 --- a/tests/integration/translation/chat_completions/bases/openai.py +++ b/tests/integration/translation/chat_completions/bases/openai.py @@ -1,4 +1,5 @@ from typing import Final +from unittest.mock import ANY from integration._support.client import object_value from integration.translation.case import TranslationTestCase @@ -420,6 +421,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -630,6 +632,7 @@ GPT_5_4_MINI_TEST_CASE: Final = TranslationTestCase( "type": "function", }, ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -729,6 +732,7 @@ GPT_6_SOL_TEST_CASE: Final = TranslationTestCase( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -886,6 +890,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -1042,6 +1047,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], @@ -1201,6 +1207,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "summary": [], } ], + "provider_specific_fields": {"phase": "commentary", "openai_responses_replay": ANY}, }, } ], diff --git a/tests/integration/translation/chat_completions/cross_model/openai.py b/tests/integration/translation/chat_completions/cross_model/openai.py index 0793daa96b1..46e9277b7cb 100644 --- a/tests/integration/translation/chat_completions/cross_model/openai.py +++ b/tests/integration/translation/chat_completions/cross_model/openai.py @@ -1,5 +1,7 @@ +from collections.abc import Mapping from dataclasses import replace from typing import Final +from unittest.mock import ANY from integration._support.client import list_value, object_value from integration.translation.case import ( @@ -29,6 +31,30 @@ from integration.translation.templates.openai import ( OPENAI_RESPONSES_AGENT_REQUEST, OPENAI_RESPONSES_AGENT_RESPONSE, ) +from pydantic import JsonValue + + +def _without_replay_record(response: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + choices: Final = response["choices"] + assert isinstance(choices, list) and isinstance(choices[0], dict), choices + message: Final = choices[0]["message"] + assert isinstance(message, dict), message + fields: Final = message["provider_specific_fields"] + assert isinstance(fields, dict), fields + kept: Final = {key: value for key, value in fields.items() if key != "openai_responses_replay"} + return { + **response, + "choices": [ + { + **choices[0], + "message": { + **{key: value for key, value in message.items() if key != "provider_specific_fields"}, + **({"provider_specific_fields": kept} if kept else {}), + }, + } + ], + } + GPT_5_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_MINI_TEST_CASE, @@ -559,7 +585,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_4_MINI_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_4_MINI_TEST_CASE.litellm_request, GPT_5_4_MINI_TEST_CASE.expected_litellm_response + GPT_5_4_MINI_TEST_CASE.litellm_request, _without_replay_record(GPT_5_4_MINI_TEST_CASE.expected_litellm_response) ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -570,6 +596,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( { "type": "message", "role": "assistant", + "phase": "commentary", "content": [ { "type": "output_text", @@ -679,6 +706,7 @@ GPT_5_4_MINI_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "caller in `src/worker.py` (`remaining = retries_remaining(3, attempts)`), and " "`tests/test_retry.py` failed with `assert 4 == 2` for `retries_remaining(3, 1)`.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -1310,6 +1338,7 @@ GPT_5_6_LUNA_AGENT_TEST_CASE: Final = replace( } ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -1343,7 +1372,8 @@ GPT_5_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_6_LUNA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_6_LUNA_AGENT_TEST_CASE.litellm_request, GPT_5_6_LUNA_AGENT_TEST_CASE.expected_litellm_response + GPT_5_6_LUNA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_5_6_LUNA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -1456,6 +1486,7 @@ GPT_5_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "of 2 and potentially negative values are not clamped. It is called by " "`src/worker.py:12`, and `tests/test_retry.py::test_retries_remaining` failed.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -1819,6 +1850,7 @@ GPT_5_6_SOL_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -1852,7 +1884,8 @@ GPT_5_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_6_SOL_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_6_SOL_AGENT_TEST_CASE.litellm_request, GPT_5_6_SOL_AGENT_TEST_CASE.expected_litellm_response + GPT_5_6_SOL_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_5_6_SOL_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -1961,6 +1994,7 @@ GPT_5_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`tests/test_retry.py` failed because `retries_remaining(3, 1)` returned `4` " "instead of `2`.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -2278,6 +2312,7 @@ GPT_5_6_TERRA_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -2311,7 +2346,8 @@ GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_5_6_TERRA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_5_6_TERRA_AGENT_TEST_CASE.litellm_request, GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response + GPT_5_6_TERRA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_5_6_TERRA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -2420,6 +2456,7 @@ GPT_5_6_TERRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`tests/test_retry.py` fails because `retries_remaining(3, 1)` returns `4` instead " "of `2`.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -2737,6 +2774,7 @@ GPT_6_ASTRA_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -2769,7 +2807,8 @@ GPT_6_ASTRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_ASTRA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_ASTRA_AGENT_TEST_CASE.litellm_request, GPT_6_ASTRA_AGENT_TEST_CASE.expected_litellm_response + GPT_6_ASTRA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_ASTRA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -2878,6 +2917,7 @@ GPT_6_ASTRA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "and `tests/test_retry.py::test_retries_remaining` failed because it returned 4 " "instead of 2.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -3194,6 +3234,7 @@ GPT_6_LUNA_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -3227,7 +3268,8 @@ GPT_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_LUNA_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_LUNA_AGENT_TEST_CASE.litellm_request, GPT_6_LUNA_AGENT_TEST_CASE.expected_litellm_response + GPT_6_LUNA_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_LUNA_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -3336,6 +3378,7 @@ GPT_6_LUNA_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`tests/test_retry.py` failed because `retries_remaining(3, 1)` returned 4 instead " "of 2.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -3587,7 +3630,7 @@ GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_SOL_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_SOL_TEST_CASE.litellm_request, GPT_6_SOL_TEST_CASE.expected_litellm_response + GPT_6_SOL_TEST_CASE.litellm_request, _without_replay_record(GPT_6_SOL_TEST_CASE.expected_litellm_response) ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -3694,6 +3737,7 @@ GPT_6_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "clamping the result at zero; `src/worker.py` calls it. `tests/test_retry.py` " "failed because `retries_remaining(3, 1)` returned 4 instead of 2.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], @@ -4007,6 +4051,7 @@ GPT_6_1_SOL_AGENT_TEST_CASE: Final = replace( }, ], "content": None, + "provider_specific_fields": {"openai_responses_replay": ANY}, }, } ], @@ -4039,7 +4084,8 @@ GPT_6_1_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( GPT_6_1_SOL_AGENT_TEST_CASE, scenario="agent_followup", litellm_request=followup_request( - GPT_6_1_SOL_AGENT_TEST_CASE.litellm_request, GPT_6_1_SOL_AGENT_TEST_CASE.expected_litellm_response + GPT_6_1_SOL_AGENT_TEST_CASE.litellm_request, + _without_replay_record(GPT_6_1_SOL_AGENT_TEST_CASE.expected_litellm_response), ), expected_provider_endpoint="/responses", expected_provider_request={ @@ -4148,6 +4194,7 @@ GPT_6_1_SOL_AGENT_FOLLOWUP_TEST_CASE: Final = replace( "`src/worker.py:12`. `tests/test_retry.py::test_retries_remaining` failed because " "it returned 4 instead of 2; no files were changed.", "role": "assistant", + "provider_specific_fields": {"phase": "final_answer", "openai_responses_replay": ANY}, }, } ], From d302096b5a49630a6dc6e34ccc6616f2464267db Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:15:44 -0700 Subject: [PATCH 08/25] feat(lite): offer each model group's reasoning efforts in pi, OpenCode and Codex (#48949) * feat(lite): offer each model group's reasoning efforts in pi, OpenCode and Codex lite pi, lite opencode and lite codex now read supported_reasoning_efforts from /model_group/info and write them into each client's model config. pi gets reasoning and a thinkingLevelMap, OpenCode gets reasoning_options plus one variant per effort, and Codex fallback entries get supported_reasoning_levels with a default level. The proxy-served Codex catalog on /v1/models?client_version= offers the same levels Codex before 0.138.0 fails the whole catalog on an effort it cannot name, so those releases, and an unreadable version, get only the six efforts every release parses. A group with unknown efforts keeps today's output, and a group with none is written as non-reasoning The strength order and nearest_declared_reasoning_effort move to an SDK-free module so the lite CLI, which loads without initializing the SDK, can import the Codex catalog * fix(lite): describe team models by their public name and keep Codex's none default /model_group/info now looks up a team's own deployments when the caller's team serves the requested name, so a team model reports its reasoning levels. Codex fallback entries default to none when the group lists it, and the proxy builds the Codex catalog in a worker thread * fix(lite): round Codex's default level down and describe a team's shared-name model from its own deployments Move the reasoning strength helper to litellm_core_utils so lite and the Codex catalog import it without the SDK or the router, and let make lint-format-check-changed diff the working tree so a staged rename passes. * fix(lite): keep pi's system role on reasoning model groups pi switches its system prompt to the developer role for any model marked reasoning, so an operator's cache_control_injection_points entry for role system stopped matching on Claude groups. lite pi now sets compat.supportsDeveloperRole false on those entries, so pi sends system as it did before. * fix(codex): treat a client_version too long to parse as an old Codex and keep undescribed stock levels A client_version past Python's integer digit limit made the Codex catalog answer 500. It now gets the levels every Codex parses. A stock Codex level with no description validates again, as on base, and lite codex treats undecodable codex --version output as an unknown version * fix(lite): keep REASONING_EFFORT_STRENGTH_ORDER importable from router_utils and skip the Codex sync on undecodable debug models output --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- Makefile | 2 +- .../reasoning_effort_strength.py | 45 +++++ litellm/proxy/client/cli/README.md | 2 +- litellm/proxy/client/cli/commands/agents.py | 113 +++++++++-- litellm/proxy/client/cli/commands/pi.py | 40 +++- .../proxy/common_utils/codex_model_catalog.py | 97 ++++++++-- litellm/proxy/proxy_server.py | 26 ++- litellm/router.py | 19 +- litellm/router_api.py | 2 +- .../reasoning_effort_capability.py | 41 +--- litellm/types/router.py | 5 +- tests/unit/proxy/client/cli/test_agents.py | 179 +++++++++++++++--- tests/unit/proxy/client/cli/test_pi.py | 70 ++++++- .../common_utils/test_codex_model_catalog.py | 134 +++++++++++++ .../proxy_server/test_routes_model_info.py | 105 ++++++++++ .../proxy/proxy_server/test_routes_models.py | 30 +++ .../public_v1/test_model_hub.py | 2 +- tests/unit/proxy/router_entry_point_stubs.py | 2 +- .../test_reasoning_effort_capability.py | 2 +- tests/unit/test_default_branch.py | 11 ++ .../test_router/test_router_entry_points.py | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 +- 22 files changed, 823 insertions(+), 120 deletions(-) create mode 100644 litellm/litellm_core_utils/reasoning_effort_strength.py diff --git a/Makefile b/Makefile index 937873f8835..48df3c8e215 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,7 @@ lint-install: # the same set CI's ':(glob)litellm/**/*.py' selects. lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) @base_ref=$$($(RESOLVE_BASE)) && \ - changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \ + changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \ files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \ if [ -z "$$files" ]; then \ echo "No changed litellm Python files to format-check."; \ diff --git a/litellm/litellm_core_utils/reasoning_effort_strength.py b/litellm/litellm_core_utils/reasoning_effort_strength.py new file mode 100644 index 00000000000..f7b51b75a9f --- /dev/null +++ b/litellm/litellm_core_utils/reasoning_effort_strength.py @@ -0,0 +1,45 @@ +"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set. + +Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it. +""" + +from collections.abc import Sequence +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from typing_extensions import assert_never + +ReasoningEffortRounding: TypeAlias = Literal["up", "down"] + +REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max") +REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType( + {effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)} +) + + +def nearest_declared_reasoning_effort( + requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" +) -> str: + """Rounding up takes the weakest declared level at least as strong as the request, else the + strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected + call. Rounding down mirrors it: the strongest declared level at most as strong, else the + weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder + and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and + a model that cannot turn it off says so itself. A level outside the strength order is likewise + returned as is for upstream to judge.""" + ranked: Final = sorted( + (effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK), + key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort], + ) + if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked: + return requested + strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested] + match rounding: + case "up": + return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1]) + case "down": + return next( + (effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0] + ) + case _: + assert_never(rounding) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 819908f9e70..1fdf8da7d2b 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -492,7 +492,7 @@ lite codex exec "summarize the repo" Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process. -The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place. +The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug, plus whichever reasoning levels its model group lists on `/model_group/info` the installed Codex can parse. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place. pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. The entry also turns on pi's session header (`x-session-id`, sent by pi 0.80.7 and later), so every chat turn of one pi session shares one session id in the logs and sticks to one deployment when session affinity is on. A custom `pi --session-id` must be at least 8 letters, digits, `_`, or `-` for the proxy to read it. Your own flags come after the injected pin, so `lite pi --model litellm/` wins, and inside the TUI the `/model` picker lists every synced litellm model. diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py index f19fb508c7d..12d97ef6cce 100644 --- a/litellm/proxy/client/cli/commands/agents.py +++ b/litellm/proxy/client/cli/commands/agents.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from itertools import chain from pathlib import Path from types import MappingProxyType -from typing import Final, TypeAlias, TypeVar +from typing import Final, Literal, TypeAlias, TypeVar from urllib.parse import urlsplit import click @@ -41,6 +41,7 @@ from .pi import ( LITELLM_PROXY_API_KEY_ENV, PI_PROVIDER_NAME, ListingFailure, + ModelLimits, PiSyncError, fetch_model_ids, fetch_model_limits, @@ -611,9 +612,27 @@ class _OpenCodeLimit(LiteLLMBaseModel): output: int +class _OpenCodeVariant(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + reasoningEffort: str + + +class _OpenCodeReasoningOption(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["effort"] = "effort" + values: tuple[str, ...] + + class _OpenCodeModel(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + name: str limit: _OpenCodeLimit | None = None + reasoning: Literal[True] | None = None + reasoning_options: tuple[_OpenCodeReasoningOption, ...] | None = None + variants: Mapping[str, _OpenCodeVariant] | None = None class _OpenCodeProviderOptions(LiteLLMBaseModel): @@ -632,15 +651,43 @@ class _OpenCodeConfig(LiteLLMBaseModel): provider: Mapping[str, _OpenCodeProvider] -def _opencode_model_entry(model: ListedModel) -> _OpenCodeModel: - if model.max_input_tokens is None or model.max_output_tokens is None: - return _OpenCodeModel(name=model.id) +_NO_GROUPS: Final[Mapping[str, ModelLimits]] = MappingProxyType({}) + + +def _reasoning_efforts(model_id: str, groups: Mapping[str, ModelLimits]) -> tuple[str, ...] | None: + group: Final = groups.get(model_id) + return group.supported_reasoning_efforts if group is not None else None + + +def _opencode_model_entry(model: ListedModel, groups: Mapping[str, ModelLimits]) -> _OpenCodeModel: + """One OpenCode model entry, with the model group's reasoning efforts as its variants when known. + + OpenCode 1.x builds its own variants for a reasoning model and replaces them only through + `reasoning_options`. 2.x ignores `reasoning` and `reasoning_options` and builds its own variants + unless the entry lists some, so the efforts go in all three, and a model with none lists an + empty `variants`.""" + limit: Final = ( + _OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + if model.max_input_tokens is not None and model.max_output_tokens is not None + else None + ) + efforts: Final = _reasoning_efforts(model.id, groups) + if efforts is None: + return _OpenCodeModel(name=model.id, limit=limit) + if not efforts: + return _OpenCodeModel(name=model.id, limit=limit, variants=MappingProxyType({})) return _OpenCodeModel( - name=model.id, limit=_OpenCodeLimit(context=model.max_input_tokens, output=model.max_output_tokens) + name=model.id, + limit=limit, + reasoning=True, + reasoning_options=(_OpenCodeReasoningOption(values=efforts),), + variants=MappingProxyType({effort: _OpenCodeVariant(reasoningEffort=effort) for effort in efforts}), ) -def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> str: +def opencode_provider_config( + base_url: str, models: Sequence[ListedModel], groups: Mapping[str, ModelLimits] = _NO_GROUPS +) -> str: """OPENCODE_CONFIG_CONTENT declaring the proxy as OpenCode provider `litellm`. One model entry per chat-capable /v1/models row (mode chat, responses, or @@ -648,6 +695,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st is read back through {env:OPENAI_API_KEY}, which build_agent_env exports, so it never lands in the config text. OpenCode merges this inline config over the user's own files, leaving unrelated keys and providers untouched. + `groups` is /model_group/info by model group, where the reasoning efforts come from. """ chat_models: Final = _chat_models(models) provider: Final = _OpenCodeProvider( @@ -657,7 +705,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st baseURL=base_url.rstrip("/") + "/v1", apiKey=f"{{env:{OPENAI_API_KEY_ENV}}}", ), - models=MappingProxyType({m.id: _opencode_model_entry(m) for m in chat_models}), + models=MappingProxyType({m.id: _opencode_model_entry(m, groups) for m in chat_models}), ) config: Final = _OpenCodeConfig(provider=MappingProxyType({OPENCODE_PROVIDER_ID: provider})) return config.model_dump_json(exclude_none=True) @@ -672,7 +720,8 @@ def opencode_model_sync_env( ) -> Mapping[str, str] | ModelSyncSkipped: """Env addition that hands OpenCode the proxy's model list, or why it was skipped. - Fetches /v1/models with the key and packs it into OPENCODE_CONFIG_CONTENT. + Fetches /v1/models with the key, and /model_group/info for each model's reasoning + efforts, and packs them into OPENCODE_CONFIG_CONTENT. An OPENCODE_CONFIG_CONTENT already in the environment is left alone, and a failed fetch is reported rather than raised: OpenCode still launches on the plain OPENAI_* env, just without a synced model list. @@ -682,16 +731,20 @@ def opencode_model_sync_env( listing: Final = _fetch_model_listing(base_url, api_key, get=get) if isinstance(listing, ModelSyncSkipped): return listing - return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)}) + groups: Final = fetch_model_limits(base_url, api_key, get=get) + return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing, groups)}) -def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow: +def _codex_row( + model: ListedModel, upstream: ModelUpstream | None, groups: Mapping[str, ModelLimits] +) -> CodexCatalogRow: return CodexCatalogRow( id=model.id, mode=model.mode, max_input_tokens=model.max_input_tokens, upstream_model=upstream.model if upstream is not None else None, reasoning_effort=upstream.reasoning_effort if upstream is not None else None, + supported_reasoning_efforts=_reasoning_efforts(model.id, groups), ) @@ -700,6 +753,8 @@ def codex_model_catalog( stock: Sequence[CodexStockModel], instructions: str, upstreams: Mapping[str, ModelUpstream], + groups: Mapping[str, ModelLimits] = _NO_GROUPS, + client_version: str | None = None, ) -> str | None: """The `model_catalog_json` body listing the proxy's chat models, or None if there are none. @@ -714,17 +769,19 @@ def codex_model_catalog( deployments pin, and a window no larger than its listed max_input_tokens. A model Codex does not know gets the fallback entry, with the same base instructions Codex itself uses so the agent never - runs without a system prompt. Apart from aliases, the catalog is the one - `/v1/models?client_version=...` serves, built from the installed Codex's own - entries instead of the proxy's vendored ones. + runs without a system prompt, and offers the reasoning efforts `groups` gives its model group + that the installed Codex, at `client_version`, can parse. Apart from aliases, the catalog is + the one `/v1/models?client_version=...` serves, built from the installed Codex's own entries + instead of the proxy's vendored ones. """ - rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models) + rows: Final = tuple(_codex_row(m, upstreams.get(m.id), groups) for m in models) body: Final = codex_models_response_json( rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None, alias_defaults="deployment", + client_version=client_version, ) return body.json if body.listed else None @@ -771,7 +828,7 @@ def _codex_debug_models( encoding="utf-8", timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS, ) - except (OSError, subprocess.TimeoutExpired) as e: + except (OSError, subprocess.TimeoutExpired, UnicodeError) as e: return ModelSyncSkipped(f"`{name} debug models` failed: {e}") if completed.returncode == 0: return completed.stdout @@ -793,6 +850,25 @@ def _stock_codex_models( return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}") +def _codex_version( + binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]] +) -> str | None: + """The version `codex --version` prints (`codex-cli 0.165.0`), or None when it cannot be read.""" + try: + completed: Final = run( + _windows_command(binary, (binary, "--version")), + env=dict(env), + stdin=subprocess.DEVNULL, + capture_output=True, + encoding="utf-8", + timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired, UnicodeError): + return None + words: Final = completed.stdout.split() + return words[-1] if completed.returncode == 0 and words else None + + def codex_model_sync_args( base_env: Mapping[str, str], base_url: str, @@ -811,7 +887,8 @@ def codex_model_sync_args( atomically replaced on every launch. The Codex at `binary` first lists its own models, so the ones the proxy serves keep that Codex's entries, an alias included when every unpaused deployment /v1/model/info lists for it routes - to the same OpenAI model and Codex knows that model, and then + to the same OpenAI model and Codex knows that model, and its version, so a + model it does not know offers only reasoning efforts it can parse, and then reads the file back once before it is handed over. The key never lands in the file. A failed fetch, read, listing, write or read-back is reported rather than raised: Codex still launches with its built-in catalog and takes @@ -834,7 +911,9 @@ def codex_model_sync_args( if isinstance(stock, ModelSyncSkipped): return stock upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get) - catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams) + groups: Final = fetch_model_limits(base_url, api_key, get=get) + version: Final = _codex_version(binary, base_env, run=run) + catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams, groups, version) if catalog is None: return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models") try: diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index 357b82bdf67..37d485dc1b9 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -50,8 +50,11 @@ class PiSyncError: @dataclass(frozen=True, slots=True) class ModelLimits: + """What /model_group/info says about one model group; `supported_reasoning_efforts` is None when unknown.""" + context_window: int | None max_tokens: int | None + supported_reasoning_efforts: tuple[str, ...] | None = None _NonEmptyString = Annotated[str, StringConstraints(min_length=1)] @@ -76,9 +79,12 @@ class _ModelList(LiteLLMBaseModel): class _ModelGroup(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + model_group: str max_input_tokens: float | None = None max_output_tokens: float | None = None + supported_reasoning_efforts: tuple[str, ...] | None = None class _ModelGroupList(LiteLLMBaseModel): @@ -137,8 +143,8 @@ def fetch_model_limits( *, get: Callable[..., requests.Response] = requests.get, ) -> Mapping[str, ModelLimits]: - """Best effort: pi falls back to its own defaults for models without limits, - so an unavailable /model_group/info must not block the launch.""" + """Best effort: pi, OpenCode and Codex fall back to their own defaults for models without + limits or reasoning efforts, so an unavailable /model_group/info must not block the launch.""" url: Final = base_url.rstrip("/") + "/model_group/info" try: resp: Final = get( @@ -156,6 +162,7 @@ def fetch_model_limits( group.model_group: ModelLimits( context_window=int(group.max_input_tokens) if group.max_input_tokens else None, max_tokens=int(group.max_output_tokens) if group.max_output_tokens else None, + supported_reasoning_efforts=group.supported_reasoning_efforts, ) for group in listing.data } @@ -168,6 +175,33 @@ def models_json_path(env: Mapping[str, str]) -> Path: return root / "models.json" +_PI_LADDER_LEVELS: Final = ("minimal", "low", "medium", "high") +_PI_OPT_IN_LEVELS: Final = ("xhigh", "max") +_REASONING_OFF_EFFORT: Final = "none" + + +def _thinking_level_map(efforts: tuple[str, ...]) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized + """Each level pi offers maps to a value: pi drops a level mapped to null and shows xhigh and max only when set.""" + return { + "off": _REASONING_OFF_EFFORT if _REASONING_OFF_EFFORT in efforts else None, + **{level: level if level in efforts else None for level in _PI_LADDER_LEVELS}, + **{level: level for level in _PI_OPT_IN_LEVELS if level in efforts}, + } + + +def _reasoning_fields(limit: ModelLimits | None) -> Mapping[str, JsonValue]: + efforts: Final = limit.supported_reasoning_efforts if limit else None + if not efforts: + return MappingProxyType({}) + return MappingProxyType( + { + "reasoning": True, + "thinkingLevelMap": _thinking_level_map(efforts), + "compat": {"supportsDeveloperRole": False}, + } + ) + + def _model_entry( model_id: str, limits: Mapping[str, ModelLimits] ) -> dict[str, JsonValue]: # mutable-ok: JSON object is serialized @@ -178,7 +212,7 @@ def _model_entry( output: Final[dict[str, JsonValue]] = ( # mutable-ok: JSON field {"maxTokens": limit.max_tokens} if limit and limit.max_tokens else {} ) - return {"id": model_id, **context, **output} + return {"id": model_id, **context, **output, **_reasoning_fields(limit)} def provider_block( diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 63e7178ff99..63fb27d62cf 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -7,7 +7,8 @@ never the OpenAI list shape. This module builds that body from the rows ``/v1/mo lists: a model Codex knows keeps Codex's own entry for it, any other model gets the fallback entry Codex itself uses for an unknown slug, and ``model_info.service_tiers`` becomes the entry's ``service_tiers``, which Codex turns into slash commands (``/ultrafast``) that send ``service_tier`` -upstream. +upstream. A fallback entry offers the model group's ``supported_reasoning_efforts`` as its reasoning +levels, trimmed to the ones the requesting Codex release can parse. The stock entries are Codex 0.159.3's bundled catalog, vendored unchanged from https://github.com/openai/codex/blob/rust-v0.159.3/codex-rs/models-manager/models.json (Apache-2.0) @@ -26,13 +27,16 @@ from pathlib import Path from types import MappingProxyType, NoneType from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias -from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError +from packaging.version import Version +from pydantic import ConfigDict, Field, StringConstraints, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort from litellm.types.llms.base_model import LiteLLMBaseModel if TYPE_CHECKING: from litellm.router import Router + from litellm.router_api import RouterRegistry from litellm.types.proxy.model_listing import ModelInfoResponse CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md") @@ -43,6 +47,10 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"}) _TierId = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] _BODY_PREFIX: Final = '{"models":[' _BODY_SUFFIX: Final = "]}" +_FIRST_CODEX_WITH_CUSTOM_EFFORTS: Final = Version("0.138.0") +_EFFORTS_EVERY_CODEX_PARSES: Final = frozenset({"none", "minimal", "low", "medium", "high", "xhigh"}) +_PREFERRED_DEFAULT_EFFORT: Final = "medium" +_CODEX_UNSET_EFFORT: Final = "none" class CodexServiceTier(LiteLLMBaseModel): @@ -67,6 +75,15 @@ class _ConfiguredTier(LiteLLMBaseModel): _CONFIGURED_TIERS: Final = TypeAdapter(tuple[_TierId | _ConfiguredTier, ...]) +class CodexFallbackReasoningLevel(LiteLLMBaseModel): + """A `ReasoningEffortPreset` as Codex reads it: picking it sends `effort` as the request's reasoning effort.""" + + model_config = ConfigDict(frozen=True) + + effort: str + description: str + + class CodexTruncationPolicy(LiteLLMBaseModel): mode: Literal["bytes"] = "bytes" limit: int = 10_000 @@ -82,10 +99,13 @@ class CodexFallbackModel(LiteLLMBaseModel): same as `codex -m` did. """ + model_config = ConfigDict(frozen=True) + slug: str display_name: str description: NoneType = None - supported_reasoning_levels: tuple[()] = () + default_reasoning_level: str | None = Field(default=None, exclude_if=lambda level: level is None) + supported_reasoning_levels: tuple[CodexFallbackReasoningLevel, ...] = () shell_type: Literal["unified_exec"] = "unified_exec" visibility: Literal["list"] = "list" supported_in_api: Literal[True] = True @@ -153,7 +173,8 @@ class CodexCatalogRow: """What the catalog needs to know about one listed model: its public id, the listing's mode and input limit, the upstream model behind it, the reasoning effort its deployments pin, and the `model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers` - value per deployment behind the id, None where a deployment sets none.""" + value per deployment behind the id, None where a deployment sets none. `supported_reasoning_efforts` + is the model group's, None when unknown.""" id: str mode: str | None = None @@ -162,6 +183,7 @@ class CodexCatalogRow: reasoning_effort: str | None = None display_name: str | None = None service_tiers: tuple[object, ...] = () + supported_reasoning_efforts: tuple[str, ...] | None = None search_tool_support: tuple[bool | None, ...] = () @@ -261,8 +283,40 @@ def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None: return True if per_deployment and None not in per_deployment else None -def _codex_can_drive(row: CodexCatalogRow) -> bool: - return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES) +def _codex_can_drive(model_id: str, mode: str | None) -> bool: + return "*" not in model_id and (mode is None or mode in CODEX_CHAT_MODES) + + +def _parses_custom_efforts(client_version: str | None) -> bool: + try: + return client_version is not None and Version(client_version) >= _FIRST_CODEX_WITH_CUSTOM_EFFORTS + except ValueError: + return False + + +def codex_reasoning_levels( + efforts: Sequence[str], client_version: str | None +) -> tuple[tuple[CodexFallbackReasoningLevel, ...], str | None]: + """The reasoning levels a fallback entry offers for a model group's efforts, and its default level. + + Codex before 0.138.0 fails the whole catalog on an effort it has no name for, so for it, and for + a version that does not parse, only the efforts every release knows are offered. The default is + set whenever a level is offered, because Codex reads a missing one as `none`: `none` itself when the + group accepts it, so a user who picks no level sends what Codex sent before, else medium, else the + nearest weaker declared level, so picking no level never costs more reasoning than Codex's usual medium, + else the weakest declared level, else the first one.""" + parses_any: Final = _parses_custom_efforts(client_version) + offered: Final = tuple(effort for effort in efforts if parses_any or effort in _EFFORTS_EVERY_CODEX_PARSES) + if not offered: + return (), None + nearest: Final = nearest_declared_reasoning_effort(_PREFERRED_DEFAULT_EFFORT, offered, "down") + levels: Final = tuple( + CodexFallbackReasoningLevel(effort=effort, description=f"Sends reasoning_effort={effort} upstream") + for effort in offered + ) + if _CODEX_UNSET_EFFORT in offered: + return levels, _CODEX_UNSET_EFFORT + return levels, nearest if nearest in offered else offered[0] def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> CodexStockModel | None: @@ -309,6 +363,7 @@ def _entry( served: frozenset[str], instructions: str, alias_defaults: CodexAliasDefaults, + client_version: str | None, ) -> CodexCatalogEntry: stock_model: Final = _stock_for(row, stock) stock_tiers: Final = MappingProxyType( @@ -317,9 +372,12 @@ def _entry( configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers) search_tool: Final = offered_search_tool(row.search_tool_support) if stock_model is None: + levels, default_level = codex_reasoning_levels(row.supported_reasoning_efforts or (), client_version) return CodexFallbackModel( slug=row.id, display_name=row.display_name if row.display_name is not None else row.id, + default_reasoning_level=default_level, + supported_reasoning_levels=levels, priority=index, service_tiers=configured if configured is not None else (), context_window=row.max_input_tokens, @@ -406,6 +464,7 @@ def codex_models_response_json( instructions: str | None = None, byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT, alias_defaults: CodexAliasDefaults = "codex", + client_version: str | None = None, ) -> CodexCatalogBody: """Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit. @@ -417,14 +476,15 @@ def codex_models_response_json( keeps every entry. A row listed under another name than the Codex model its `upstream_model` names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps - them, as Codex would use them for the model's own name. + them, as Codex would use them for the model's own name. `client_version` is the requesting Codex's version, which decides the reasoning + levels a fallback entry may offer; None counts as a version too old for custom levels. """ known: Final = bundled_codex_models() if stock is None else stock prompt: Final = codex_base_instructions() if instructions is None else instructions - drivable: Final = tuple(row for row in rows if _codex_can_drive(row)) + drivable: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode)) served: Final = frozenset(row.id for row in drivable) entries: Final = tuple( - _entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable) + _entry(index, row, known, served, prompt, alias_defaults, client_version) for index, row in enumerate(drivable) ) serialized: Final = tuple(_serialized(entry) for entry in entries) kept: Final = _kept_positions(entries, serialized, byte_limit) @@ -435,6 +495,15 @@ def codex_models_response_json( ) +def _group_reasoning_efforts( + row: ModelInfoResponse, lookup_id: str, registry: RouterRegistry | None, team_id: str | None +) -> tuple[str, ...] | None: + if registry is None or not _codex_can_drive(row["id"], row.get("mode")): + return None + group: Final = registry.group(lookup_id, include_hidden=True, team_id=team_id) + return group.supported_reasoning_efforts if group is not None else None + + def _catalog_row( row: ModelInfoResponse, lookup_id: str, label_id: str, llm_router: Router | None, team_id: str | None ) -> CodexCatalogRow: @@ -446,6 +515,7 @@ def _catalog_row( upstream_model=registry.resolve(lookup_id, kind="upstream", team_id=team_id) if registry is not None else None, display_name=registry.group(label_id, field="display_name") if registry is not None else None, service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (), + supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id), search_tool_support=( registry.resolve(lookup_id, kind="search_tool_support", team_id=team_id) if registry is not None else () ), @@ -463,7 +533,8 @@ def codex_catalog_rows( team-scoped rows resolve the way the Anthropic listing's display names do, a `model_group_alias` reading its target's deployments under its own name; `team_id` is the requesting key's team, so the upstream model that picks Codex's stock entry and the tiers are read only off the deployments its - requests can route to. Display names are read under `label_entries`' ids when given.""" + requests can route to, and the reasoning efforts are the ones `/model_group/info` reports for the + entry. Display names are read under `label_entries`' ids when given.""" lookup_ids: Final = MappingProxyType(dict(entries)) label_ids: Final = lookup_ids if label_entries is None else MappingProxyType(dict(label_entries)) return tuple( @@ -480,9 +551,13 @@ def codex_model_list_body( llm_router: Router | None, team_id: str | None = None, label_entries: Sequence[tuple[str, str]] | None = None, + *, + client_version: str | None = None, ) -> str: """The `/v1/models?client_version=...` body, logging the models Codex's byte limit left out.""" - body: Final = codex_models_response_json(codex_catalog_rows(rows, entries, llm_router, team_id, label_entries)) + body: Final = codex_models_response_json( + codex_catalog_rows(rows, entries, llm_router, team_id, label_entries), client_version=client_version + ) if body.left_out: verbose_proxy_logger.warning( "Codex model catalog cut at %d bytes, left out: %s. Models offering a service tier are kept first, " diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d92e8733a3..1bd221bca98 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12337,8 +12337,10 @@ async def model_list( - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a provider's model catalog. When present, the response is Codex's own catalog shape (`{"models": [...]}`) built from the same listing, with - each model's `model_info.service_tiers` as its service tiers; absent, - the OpenAI shape below is returned + each model's `model_info.service_tiers` as its service tiers and, for a + model that Codex release does not know, the reasoning efforts + `/model_group/info` reports that it can parse; absent, the OpenAI shape + below is returned - include_metadata: Include additional metadata in the response with fallback information - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") Defaults to "general" when include_metadata=true @@ -12497,8 +12499,13 @@ async def model_list( admin_listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body( - admin_listing, admin_entries, llm_router, team_id or user_api_key_dict.team_id + content=await asyncio.to_thread( + codex_model_list_body, + admin_listing, + admin_entries, + llm_router, + team_id or user_api_key_dict.team_id, + client_version=client_version, ), media_type="application/json", ) @@ -12573,7 +12580,15 @@ async def model_list( listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above if client_version is not None: return Response( - content=codex_model_list_body(listing, entries, llm_router, team_id or user_api_key_dict.team_id, labels), + content=await asyncio.to_thread( + codex_model_list_body, + listing, + entries, + llm_router, + team_id or user_api_key_dict.team_id, + labels, + client_version=client_version, + ), media_type="application/json", ) @@ -18046,6 +18061,7 @@ def get_model_group_info( registry.deployments( resolve_model_group_alias(registry.aliases(), model) or model, source="routed_group", + team_id=team_id, ) ), } diff --git a/litellm/router.py b/litellm/router.py index f2e68424c8d..3a423b79e49 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12856,7 +12856,10 @@ class Router(DeprecatedRouterMembers): configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None reasoning_efforts_initialized = False reasoning_efforts_unknown = False - model_list: Final = self._core_get_model_list_of_routed_group(model_group) + serves_team_deployments: Final = team_id is not None and bool( + self._team_deployments_named(team_id, model_group) + ) + model_list: Final = self._core_get_model_list_of_routed_group(model_group, team_id) model_group_description: Final[str | None] = next( (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, @@ -12864,7 +12867,8 @@ class Router(DeprecatedRouterMembers): for model in model_list: is_match = False if ( - "model_name" in model + serves_team_deployments + or "model_name" in model and model["model_name"] == model_group or "model_name" in model and self._core_pattern_router.route(model_group) is not None @@ -14213,17 +14217,22 @@ class Router(DeprecatedRouterMembers): lambda name: self._core_wildcard_route_deployments(name, team_id), ) - def _core_get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: + def _core_get_model_list_of_routed_group( + self, model_group: str, team_id: str | None = None + ) -> list[DeploymentTypedDict]: """ The deployments a request the router has already resolved to model_group is - served from: the ones named model_group, the routing group of that name, or - the wildcard route matching it when neither exists. + served from: team_id's own deployments under that public name when the team has + any, else the ones named model_group, the routing group of that name, or the + wildcard route matching it when neither exists. Unlike get_model_list, model_group's own model_group_alias entry is not followed. The router resolves an alias exactly once, so a group reached as an alias target is served by its own deployments, never by a second hop: in the chain X -> T -> U a request to X is served from T's deployments. """ + if team_id is not None and self._team_deployments_named(team_id, model_group): + return self._core_get_all_deployments(model_name=model_group, team_id=team_id) own: Final = cast( # cast-ok: the private alias returns what get_all_deployments returns list[DeploymentTypedDict], self._get_all_deployments(model_name=model_group) ) diff --git a/litellm/router_api.py b/litellm/router_api.py index a1cf1805e90..d62999cf261 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -419,7 +419,7 @@ class RouterRegistry: if source == "named": return r._core_deployments_named(model_name) if source == "routed_group": - return r._core_get_model_list_of_routed_group(model_name) + return r._core_get_model_list_of_routed_group(model_name, team_id) if source == "wildcard": return r._core_wildcard_route_deployments(model_name, team_id) if source == "routable": diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 3c3fad93579..3563fcd96ae 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -27,20 +27,22 @@ it above. from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Final, Literal, TypeAlias, get_args - -from typing_extensions import assert_never +from typing import Final, TypeAlias, get_args import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params +from litellm.litellm_core_utils.reasoning_effort_strength import ( + REASONING_EFFORT_STRENGTH_ORDER, + ReasoningEffortRounding, + nearest_declared_reasoning_effort, +) from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT -ReasoningEffortRounding: TypeAlias = Literal["up", "down"] RequestedReasoningEffort: TypeAlias = str | Mapping[str, object] | None REASONING_EFFORT_ADVERTISEMENT_ORDER: Final[tuple[str, ...]] = get_args(REASONING_EFFORT) @@ -115,35 +117,6 @@ def declared_reasoning_efforts_for_model(model: str, custom_llm_provider: str) - return declared_reasoning_efforts(entry) -REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max") -_STRENGTH_RANK: Final = MappingProxyType({effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}) - - -def nearest_declared_reasoning_effort( - requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up" -) -> str: - """Rounding up takes the weakest declared level at least as strong as the request, else the - strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected - call. Rounding down mirrors it: the strongest declared level at most as strong, else the - weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder - and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and - a model that cannot turn it off says so itself. A level outside the strength order is likewise - returned as is for upstream to judge.""" - ranked: Final = sorted( - (effort for effort in declared if effort in _STRENGTH_RANK), key=lambda effort: _STRENGTH_RANK[effort] - ) - if requested in ranked or requested not in _STRENGTH_RANK or not ranked: - return requested - strength: Final = _STRENGTH_RANK[requested] - match rounding: - case "up": - return next((effort for effort in ranked if _STRENGTH_RANK[effort] >= strength), ranked[-1]) - case "down": - return next((effort for effort in reversed(ranked) if _STRENGTH_RANK[effort] <= strength), ranked[0]) - case _: - assert_never(rounding) - - _ROUNDING_KEY: Final = "reasoning_effort_rounding" _DEFAULT_EFFORT: Final = "default" _EFFORT_KEY: Final = "effort" @@ -163,7 +136,7 @@ def deployment_reasoning_effort_levels(deployment_model_info: object) -> tuple[s def deployment_rounds_reasoning_effort(deployment_model_info: object) -> bool: declared: Final = deployment_reasoning_effort_levels(deployment_model_info) - return declared is not None and any(level in _STRENGTH_RANK for level in declared) + return declared is not None and any(level in REASONING_EFFORT_STRENGTH_ORDER for level in declared) def with_adaptive_thinking_as_effort( diff --git a/litellm/types/router.py b/litellm/types/router.py index 56dcad4051a..f927ae0475b 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,9 +4,11 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import functools import re from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -989,9 +991,10 @@ class AlertingConfig(LiteLLMBaseModel): alerting_threshold: float | None = 300 +@functools.cache def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: """Resolve a class's annotations, keeping each resolved annotation opaque.""" - return get_type_hints(model_class) + return MappingProxyType(get_type_hints(model_class)) class ModelGroupInfo(LiteLLMBaseModel): diff --git a/tests/unit/proxy/client/cli/test_agents.py b/tests/unit/proxy/client/cli/test_agents.py index 6334fc1482e..60a19b11d72 100644 --- a/tests/unit/proxy/client/cli/test_agents.py +++ b/tests/unit/proxy/client/cli/test_agents.py @@ -59,6 +59,9 @@ class _FakeResponse: self.status_code = status_code self.content = json.dumps(body).encode() if body is not None else b"" + def json(self): + return json.loads(self.content) + class _Recorder: def __init__(self, returns=None): @@ -160,24 +163,51 @@ _STOCK_CATALOG = json.dumps({"models": list(_STOCK_MODELS.values())}) class _FakeRun: - """A `codex` that prints `stock` from a bare `debug models` and answers a catalog override with `returncode`. + """A `codex` that prints `stock` from a bare `debug models`, `version` from `--version`, and answers a + catalog override with `returncode`. `stock=None` is a Codex with no `debug models` at all: every call answers with `returncode` and `stderr`. + `version=None` is a `--version` that fails, an exception one that cannot run. """ - def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG): + def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG, version="codex-cli 0.165.0"): self.returncode = returncode self.stderr = stderr self.stock = stock + self.version = version self.calls = [] def __call__(self, args, **kwargs): self.calls.append((args, kwargs)) + if self.stock is not None and "--version" in str(args): + if isinstance(self.version, Exception): + raise self.version + if self.version is None: + return subprocess.CompletedProcess(args, 2, "", "error: unexpected argument '--version'") + return subprocess.CompletedProcess(args, 0, f"{self.version}\n", "") if self.stock is not None and "model_catalog_json=" not in str(args): return subprocess.CompletedProcess(args, 0, self.stock, "") return subprocess.CompletedProcess(args, self.returncode, "", self.stderr) +def _routed_get(captured, listing, groups, model_info=None): + """A proxy answering /model_group/info with `groups` (or `groups` as the status it fails with), + /v1/model/info with `model_info` (raised when it is an exception) and any other URL with `listing`.""" + + def fake_get(url, headers, timeout): + captured.setdefault("urls", []).append(url) + captured["headers"] = headers + if url.endswith("/v1/model/info"): + if isinstance(model_info, Exception): + raise model_info + return model_info if model_info is not None else _FakeResponse(200, {"data": []}) + if not url.endswith("/model_group/info"): + return _FakeResponse(200, listing) + return _FakeResponse(groups) if isinstance(groups, int) else _FakeResponse(200, {"data": list(groups)}) + + return fake_get + + class _FakeJsonResponse: def __init__(self, status_code, payload=None): self.status_code = status_code @@ -534,15 +564,9 @@ class TestOpencodeModelSync: def _listing(*models): return {"object": "list", "data": list(models)} - def _sync(self, listing, base_env=None, base_url="http://localhost:4000/"): + def _sync(self, listing, base_env=None, base_url="http://localhost:4000/", groups=()): captured = {} - - def fake_get(url, headers, timeout): - captured["url"] = url - captured["headers"] = headers - return _FakeResponse(200, listing) - - env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=fake_get) + env = opencode_model_sync_env(base_env or {}, base_url, "sk-key", get=_routed_get(captured, listing, groups)) return captured, env def test_declares_proxy_as_litellm_provider_with_listed_models(self): @@ -550,9 +574,9 @@ class TestOpencodeModelSync: {"id": "gpt-5.5", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, {"id": "claude-opus-4-7", "object": "model", "created": 1, "owned_by": "openai"}, ) - captured, env = self._sync(listing) + captured, env = self._sync(listing, groups=({"model_group": "gpt-5.5", "supported_reasoning_efforts": None},)) - assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["urls"] == ["http://localhost:4000/v1/models", "http://localhost:4000/model_group/info"] assert captured["headers"] == {"Authorization": "Bearer sk-key"} config = json.loads(env["OPENCODE_CONFIG_CONTENT"]) provider = config["provider"]["litellm"] @@ -585,6 +609,56 @@ class TestOpencodeModelSync: assert models["gpt-5.5"]["limit"] == {"context": 400000, "output": 128000} assert "limit" not in models["half"] + @pytest.mark.parametrize( + ("efforts", "reasoning"), + [ + ( + ["low", "high", "max"], + { + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["low", "high", "max"]}], + "variants": { + "low": {"reasoningEffort": "low"}, + "high": {"reasoningEffort": "high"}, + "max": {"reasoningEffort": "max"}, + }, + }, + ), + ( + ["none", "turbo"], + { + "reasoning": True, + "reasoning_options": [{"type": "effort", "values": ["none", "turbo"]}], + "variants": {"none": {"reasoningEffort": "none"}, "turbo": {"reasoningEffort": "turbo"}}, + }, + ), + ([], {"variants": {}}), + (None, {}), + ], + ids=["declared", "unknown-names", "non-reasoning", "unknown"], + ) + def test_model_group_reasoning_efforts_become_exactly_the_opencode_variants(self, efforts, reasoning): + listing = self._listing( + { + "id": "m", + "object": "model", + "created": 1, + "owned_by": "openai", + "max_input_tokens": 9, + "max_output_tokens": 3, + } + ) + _, env = self._sync(listing, groups=({"model_group": "m", "supported_reasoning_efforts": efforts},)) + models = json.loads(env["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] + assert models == {"m": {"name": "m", "limit": {"context": 9, "output": 3}, **reasoning}} + + def test_an_unavailable_model_group_info_keeps_the_listing_only_config(self): + listing = self._listing({"id": "m", "object": "model", "created": 1, "owned_by": "openai"}) + _, failed = self._sync(listing, groups=500) + _, unknown = self._sync(listing, groups=({"model_group": "m", "supported_reasoning_efforts": None},)) + assert failed == unknown + assert json.loads(failed["OPENCODE_CONFIG_CONTENT"])["provider"]["litellm"]["models"] == {"m": {"name": "m"}} + def test_non_chat_models_are_left_out(self): listing = self._listing( {"id": "chat", "object": "model", "created": 1, "owned_by": "openai", "mode": "chat"}, @@ -671,25 +745,13 @@ class TestCodexModelSync: def _row(model_id, **extra): return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra} - def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, model_info=None): + def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, groups=(), model_info=None): captured = {} - - def fake_get(url, headers, timeout): - if headers.get("Authorization") != "Bearer sk-key": - return _FakeResponse(401) - if url.endswith("/v1/model/info"): - if isinstance(model_info, Exception): - raise model_info - return model_info if model_info is not None else _FakeResponse(200, {"data": []}) - captured["url"] = url - captured["headers"] = headers - return _FakeResponse(200, listing) - result = codex_model_sync_args( {"CODEX_HOME": str(codex_home)}, base_url, "sk-key", - get=fake_get, + get=_routed_get(captured, listing, groups, model_info), run=_FakeRun() if run is None else run, ) return captured, result @@ -707,7 +769,11 @@ class TestCodexModelSync: listing = self._listing(self._row("gpt-5.5", mode="chat"), self._row("claude-opus-4-7")) captured, result = self._sync(listing, tmp_path / "codex") - assert captured["url"] == "http://localhost:4000/v1/models" + assert captured["urls"] == [ + "http://localhost:4000/v1/models", + "http://localhost:4000/v1/model/info", + "http://localhost:4000/model_group/info", + ] assert captured["headers"] == {"Authorization": "Bearer sk-key"} path = self._catalog_path(result) assert path == str(tmp_path / "codex" / "litellm-models.json") @@ -980,6 +1046,53 @@ class TestCodexModelSync: assert nullable in entry and entry[nullable] is None assert entry["base_instructions"].startswith("You are a coding agent running in the Codex CLI") + @pytest.mark.parametrize( + ("version", "levels"), + [ + ("codex-cli 0.165.0", ["low", "high", "max"]), + ("codex-cli 0.138.0", ["low", "high", "max"]), + ("codex-cli 0.137.0", ["low", "high"]), + ("codex-cli 0.138.0-alpha.3", ["low", "high"]), + ("codex-cli", ["low", "high"]), + (None, ["low", "high"]), + (OSError("codex vanished"), ["low", "high"]), + (UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), ["low", "high"]), + ], + ids=["current", "first-custom", "pre-custom", "pre-release", "no-number", "failed", "unrunnable", "undecodable"], + ) + def test_unknown_model_offers_the_group_efforts_the_installed_codex_parses(self, tmp_path, version, levels): + _, result = self._sync( + self._listing(self._row("m")), + tmp_path, + run=_FakeRun(version=version), + groups=({"model_group": "m", "supported_reasoning_efforts": ["low", "high", "max"]},), + ) + + assert isinstance(result, ModelSyncArgs) + entry = self._entries(tmp_path)["m"] + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == levels + assert entry["supported_reasoning_levels"][0] == { + "effort": "low", + "description": "Sends reasoning_effort=low upstream", + } + assert entry["default_reasoning_level"] == "low" + + def test_known_models_and_unknown_or_empty_groups_keep_their_entries(self, tmp_path): + listing = self._listing(self._row("gpt-5.5"), self._row("unknown"), self._row("empty")) + groups = ( + {"model_group": "gpt-5.5", "supported_reasoning_efforts": ["max"]}, + {"model_group": "unknown", "supported_reasoning_efforts": None}, + {"model_group": "empty", "supported_reasoning_efforts": []}, + ) + self._sync(listing, tmp_path, groups=groups) + with_groups = self._entries(tmp_path) + self._sync(listing, tmp_path, groups=500) + + assert with_groups == self._entries(tmp_path) + assert with_groups["gpt-5.5"] == {**_STOCK_MODELS["gpt-5.5"], "priority": 0} + assert [with_groups[slug]["supported_reasoning_levels"] for slug in ("unknown", "empty")] == [[], []] + assert all("default_reasoning_level" not in with_groups[slug] for slug in ("unknown", "empty")) + def test_context_window_comes_from_max_input_tokens_for_unknown_models_only(self, tmp_path): listing = self._listing( self._row("big", max_input_tokens=400000), @@ -1094,7 +1207,7 @@ class TestCodexModelSync: run=run, ) assert self._catalog_path(result) == str(tmp_path / "litellm-models.json") - assert len(run.calls) == 2 + assert len(run.calls) == 3 assert all(binary in command for command, _ in run.calls) def test_opencode_dispatch_never_runs_codex(self): @@ -1119,6 +1232,7 @@ class TestCodexModelSync: assert [command for command, _ in run.calls] == [ ("codex", "debug", "models"), + ("codex", "--version"), ("codex", "-c", f"model_catalog_json={json.dumps(path)}", "debug", "models"), ] for _, options in run.calls: @@ -1157,7 +1271,13 @@ class TestCodexModelSync: assert result.reason == "`codex debug models` exited 1: no output" @pytest.mark.parametrize( - "error", [OSError("codex vanished"), subprocess.TimeoutExpired("codex", 10)], ids=["oserror", "timeout"] + "error", + [ + OSError("codex vanished"), + subprocess.TimeoutExpired("codex", 10), + UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), + ], + ids=["oserror", "timeout", "undecodable"], ) def test_unrunnable_preflight_is_reported_not_raised(self, tmp_path, error): def failing_run(*a, **k): @@ -1183,6 +1303,7 @@ class TestCodexModelSync: doubled = override.replace('"', '""') assert [command for command, _ in run.calls] == [ f'{_CMD_PREFIX}""{shim}" "debug" "models""', + f'{_CMD_PREFIX}""{shim}" "--version""', f'{_CMD_PREFIX}""{shim}" "-c" "{doubled}" "debug" "models""', ] diff --git a/tests/unit/proxy/client/cli/test_pi.py b/tests/unit/proxy/client/cli/test_pi.py index e6669564358..f90febb3178 100644 --- a/tests/unit/proxy/client/cli/test_pi.py +++ b/tests/unit/proxy/client/cli/test_pi.py @@ -152,16 +152,25 @@ class TestFetchModelLimits: 200, { "data": [ - {"model_group": "m-a", "max_input_tokens": 131072, "max_output_tokens": 8192}, + { + "model_group": "m-a", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "supported_reasoning_efforts": ["low", "high", "max"], + }, {"model_group": "m-b", "max_input_tokens": None, "max_output_tokens": None}, + {"model_group": "m-c", "supported_reasoning_efforts": []}, ] }, ) limits = fetch_model_limits("http://localhost:4000/", "sk-key", get=fake_get) assert captured["url"] == "http://localhost:4000/model_group/info" - assert limits["m-a"] == ModelLimits(context_window=131072, max_tokens=8192) - assert limits["m-b"] == ModelLimits(context_window=None, max_tokens=None) + assert limits["m-a"] == ModelLimits( + context_window=131072, max_tokens=8192, supported_reasoning_efforts=("low", "high", "max") + ) + assert limits["m-b"] == ModelLimits(context_window=None, max_tokens=None, supported_reasoning_efforts=None) + assert limits["m-c"].supported_reasoning_efforts == () def test_non_200_degrades_to_no_limits(self): assert fetch_model_limits("http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(403)) == {} @@ -219,6 +228,61 @@ class TestProviderBlock: {"id": "m-2"}, ] + @pytest.mark.parametrize( + ("efforts", "thinking_level_map"), + [ + ( + ("low", "medium", "high", "xhigh", "max"), + { + "off": None, + "minimal": None, + "low": "low", + "medium": "medium", + "high": "high", + "xhigh": "xhigh", + "max": "max", + }, + ), + ( + ("none", "low", "high", "max"), + {"off": "none", "minimal": None, "low": "low", "medium": None, "high": "high", "max": "max"}, + ), + ( + ("none", "minimal", "low", "medium", "high"), + {"off": "none", "minimal": "minimal", "low": "low", "medium": "medium", "high": "high"}, + ), + ( + ("high", "turbo"), + {"off": None, "minimal": None, "low": None, "medium": None, "high": "high"}, + ), + ], + ) + def test_reasoning_efforts_become_pi_thinking_levels_and_keep_the_system_role(self, efforts, thinking_level_map): + block = provider_block( + "http://localhost:4000", + ("m-1",), + {"m-1": ModelLimits(context_window=131072, max_tokens=8192, supported_reasoning_efforts=efforts)}, + ) + assert block["models"] == [ + { + "id": "m-1", + "contextWindow": 131072, + "maxTokens": 8192, + "reasoning": True, + "thinkingLevelMap": thinking_level_map, + "compat": {"supportsDeveloperRole": False}, + } + ] + + @pytest.mark.parametrize("efforts", [None, ()]) + def test_unknown_or_no_reasoning_efforts_leave_the_entry_as_it_was(self, efforts): + block = provider_block( + "http://localhost:4000", + ("m-1",), + {"m-1": ModelLimits(context_window=131072, max_tokens=8192, supported_reasoning_efforts=efforts)}, + ) + assert block["models"] == [{"id": "m-1", "contextWindow": 131072, "maxTokens": 8192}] + class TestSyncModelsJson: def test_creates_file_and_parent_dirs(self, tmp_path): diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 75b3ee70aab..509303d9635 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -113,6 +113,94 @@ def test_unknown_model_entry_carries_every_field_codex_deserializes_without_a_de assert (entry["context_window"], entry["base_instructions"], entry["service_tiers"]) == (272000, _PROMPT, []) +_NO_REASONING_BODY = ( + '{"models":[{"slug":"gpt-6-astra","display_name":"gpt-6-astra","description":null,' + '"supported_reasoning_levels":[],"shell_type":"unified_exec","visibility":"list","supported_in_api":true,' + '"priority":0,"service_tiers":[],"availability_nux":null,"upgrade":null,"support_verbosity":false,' + '"supports_reasoning_summaries":false,"supports_parallel_tool_calls":false,"default_verbosity":null,' + '"apply_patch_tool_type":null,"truncation_policy":{"mode":"bytes","limit":10000},' + '"experimental_supported_tools":[],"context_window":272000,"base_instructions":"base prompt",' + '"supports_search_tool":false}]}' +) + + +@pytest.mark.parametrize("efforts", [None, ()], ids=["unknown", "non-reasoning"]) +@pytest.mark.parametrize("client_version", ["0.165.0", "0.120.0", None]) +def test_unknown_or_no_reasoning_efforts_keep_the_fallback_entry_byte_for_byte(efforts, client_version): + """The body an unknown model got before reasoning efforts reached the catalog.""" + assert ( + _body(_row("gpt-6-astra", supported_reasoning_efforts=efforts), client_version=client_version).json + == _NO_REASONING_BODY + ) + + +_ABSENT = object() + + +@pytest.mark.parametrize( + ("efforts", "client_version", "levels", "default"), + [ + (("low", "high", "max"), "0.165.0", ["low", "high", "max"], "low"), + (("high", "xhigh", "max"), "0.165.0", ["high", "xhigh", "max"], "high"), + ( + ("none", "low", "medium", "high", "xhigh", "max"), + "0.165.0", + ["none", "low", "medium", "high", "xhigh", "max"], + "none", + ), + (("low", "medium", "high"), "0.165.0", ["low", "medium", "high"], "medium"), + (("none", "low", "max"), "0.137.0", ["none", "low"], "none"), + (("none",), "0.165.0", ["none"], "none"), + (("low", "turbo"), "0.165.0", ["low", "turbo"], "low"), + (("low", "high", "max"), "0.138.0", ["low", "high", "max"], "low"), + (("low", "high", "max"), "0.137.9", ["low", "high"], "low"), + (("low", "high", "max"), "0.138.0-alpha.2", ["low", "high"], "low"), + (("minimal", "max"), "not-a-version", ["minimal"], "minimal"), + pytest.param(("minimal", "max"), "9" * 5000, ["minimal"], "minimal", id="version-past-int-digit-limit"), + (("low", "turbo"), None, ["low"], "low"), + (("max",), "0.137.0", [], _ABSENT), + ], +) +def test_unknown_model_offers_the_group_efforts_its_codex_release_parses(efforts, client_version, levels, default): + (entry,) = _models(_row("gpt-6-astra", supported_reasoning_efforts=efforts), client_version=client_version) + + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == levels + assert all( + level["description"] == f"Sends reasoning_effort={level['effort']} upstream" + for level in entry["supported_reasoning_levels"] + ) + assert entry.get("default_reasoning_level", _ABSENT) == default + + +@pytest.mark.parametrize( + "levels", + [[{"effort": "medium", "description": "Codex's own medium"}], [{"effort": "medium"}]], + ids=["described", "undescribed"], +) +def test_a_model_codex_knows_keeps_its_own_reasoning_levels(levels): + stock = CodexStockModel.model_validate( + { + "slug": "gpt-5.5", + "display_name": "GPT-5.5", + "priority": 3, + "visibility": "list", + "default_reasoning_level": "medium", + "supported_reasoning_levels": levels, + } + ) + + (entry,) = json.loads( + codex_models_response_json( + (_row("gpt-5.5", supported_reasoning_efforts=("low", "max")),), + stock={"gpt-5.5": stock}, + instructions=_PROMPT, + client_version="0.165.0", + ).json + )["models"] + + assert (entry["default_reasoning_level"], entry["supported_reasoning_levels"]) == ("medium", levels) + + def test_known_model_keeps_codex_stock_entry_listed_under_the_served_id(): (entry,) = _models(_row("gpt-daybreak")) @@ -507,6 +595,9 @@ def test_catalog_rows_read_the_router_by_each_entry_lookup_id(): upstream_model="openai/gpt-5.5", display_name="Team 5.5", service_tiers=(["ultrafast"], None), + supported_reasoning_efforts=router.get_model_group_info( + "model_name_team-1_c0ffee" + ).supported_reasoning_efforts, search_tool_support=(None, None), ), CodexCatalogRow( @@ -538,6 +629,7 @@ def test_catalog_rows_read_an_alias_off_its_target_under_the_alias_name(): max_input_tokens=9, upstream_model="openai/gpt-6-astra", service_tiers=(["ultrafast"],), + supported_reasoning_efforts=router.get_model_group_info("gpt-6-astra").supported_reasoning_efforts, search_tool_support=(None,), ) (entry,) = json.loads(codex_model_list_body(listing, (("gpt-6", "gpt-6"),), router))["models"] @@ -615,6 +707,48 @@ def test_model_list_body_takes_stock_metadata_off_the_deployment_the_key_team_ro assert [entry["slug"] for entry in served("team-2")] == ["coding-model", "coding"] +def test_model_list_body_offers_what_model_group_info_reports_for_each_listed_model(): + """Declared, unknown and non-reasoning groups through a real router, a hidden alias read like its + target, and the requesting Codex's version deciding which declared levels it is offered.""" + vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"} + router = litellm.Router( + model_list=[ + { + "model_name": "declared", + "litellm_params": vllm, + "model_info": {"supports_reasoning": True, "reasoning_effort_levels": ["low", "high", "max"]}, + }, + {"model_name": "unknown", "litellm_params": vllm}, + {"model_name": "non-reasoning", "litellm_params": vllm, "model_info": {"supports_reasoning": False}}, + ], + model_group_alias={"hidden-declared": {"model": "declared", "hidden": True}}, + ) + names = ("declared", "unknown", "non-reasoning", "hidden-declared") + listing = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names + ) + entries = tuple((name, name) for name in names) + + def offered(client_version): + body = codex_model_list_body(listing, entries, router, client_version=client_version) + return { + entry["slug"]: ( + [level["effort"] for level in entry["supported_reasoning_levels"]], + entry.get("default_reasoning_level"), + ) + for entry in json.loads(body)["models"] + } + + assert router.get_model_group_info("declared").supported_reasoning_efforts == ("low", "high", "max") + assert offered("0.165.0") == { + "declared": (["low", "high", "max"], "low"), + "unknown": ([], None), + "non-reasoning": ([], None), + "hidden-declared": (["low", "high", "max"], "low"), + } + assert offered("0.137.0")["declared"] == (["low", "high"], "low") + + def test_catalog_rows_without_a_router_carry_only_the_listing(): listing = ({"id": "plain", "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"},) diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 067b7f0e862..86364d93ed0 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -989,6 +989,111 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch +@pytest.mark.parametrize(("team_id", "efforts"), [(_BYOK_TEAM_ID, ["low", "high", "max"]), ("team-other", None)]) +def test_model_group_info_describes_a_team_model_under_the_public_name_its_team_calls( + client, auth_as, monkeypatch, team_id: str, efforts: list[str] | None +): + """A team's own model is listed under its public name, so its row must describe the team's deployment + behind that name instead of a placeholder, and only for the team that owns it.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.agent_endpoints import model_list_helpers + + router: Final = litellm.Router( + model_list=[ + { + "model_name": _BYOK_INTERNAL_NAME, + "litellm_params": {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"}, + "model_info": { + "team_id": _BYOK_TEAM_ID, + "team_public_model_name": _BYOK_PUBLIC_NAME, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "max"], + }, + } + ] + ) + + async def append_agents_to_model_group(*, model_groups, user_api_key_dict): + return model_groups + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) + ) + + with auth_as(LitellmUserRoles.INTERNAL_USER, user_id=None, team_id=team_id, team_models=[_BYOK_PUBLIC_NAME]): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200 + groups: Final = {group["model_group"]: group for group in response.json()["data"]} + assert groups[_BYOK_PUBLIC_NAME]["supported_reasoning_efforts"] == efforts + + +@pytest.mark.parametrize( + ("team_id", "providers", "efforts", "custom_tags"), + [ + (_BYOK_TEAM_ID, ["fireworks_ai"], ["low", "high", "max"], ["team-tag"]), + ("team-other", ["openai"], [], ["global-tag"]), + ], +) +def test_model_group_info_describes_the_deployments_a_shared_name_routes_the_team_to( + client, auth_as, monkeypatch, team_id: str, providers: list[str], efforts: list[str], custom_tags: list[str] +): + """A team model whose public name is also a global model's name routes the team's requests to the + team's deployment, so the team's row must describe that deployment in every field, while another + team still sees the global model.""" + from litellm.proxy._types import LitellmUserRoles + from litellm.proxy.agent_endpoints import model_list_helpers + + router: Final = litellm.Router( + model_list=[ + { + "model_name": _BYOK_PUBLIC_NAME, + "litellm_params": {"model": "openai/gpt-4.1-mini"}, + "model_info": {"supports_reasoning": False, "custom_tags": ["global-tag"]}, + }, + { + "model_name": _BYOK_INTERNAL_NAME, + "litellm_params": {"model": "fireworks_ai/accounts/fireworks/models/kimi-k3"}, + "model_info": { + "team_id": _BYOK_TEAM_ID, + "team_public_model_name": _BYOK_PUBLIC_NAME, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high", "max"], + "custom_tags": ["team-tag"], + }, + }, + ] + ) + + async def append_agents_to_model_group(*, model_groups, user_api_key_dict): + return model_groups + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) + ) + + with auth_as(LitellmUserRoles.INTERNAL_USER, user_id=None, team_id=team_id, team_models=[_BYOK_PUBLIC_NAME]): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200 + group: Final = {group["model_group"]: group for group in response.json()["data"]}[_BYOK_PUBLIC_NAME] + assert (group["providers"], group["supported_reasoning_efforts"], group["custom_tags"]) == ( + providers, + efforts, + custom_tags, + ) + + # --------------------------------------------------------------------------- # GET /v2/model/info?exclude_auto_routers # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index b91994b2d04..ffd23fa3a6f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -52,6 +52,7 @@ def patched_models(monkeypatch): router.get_routable_upstream_model = MagicMock(return_value="gpt-4") router.get_configured_display_name = MagicMock(return_value=None) router.get_configured_service_tiers = MagicMock(return_value=()) + router.get_model_group_info = MagicMock(return_value=None) router.get_search_tool_support = MagicMock(return_value=()) route_entry_points_to_stubs(router) @@ -451,6 +452,35 @@ def test_codex_format_carries_configured_service_tiers(client, auth_as, patched_ assert all("service_tiers" not in m for m in openai_response.json()["data"]) +@pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) +def test_codex_reasoning_levels_are_the_ones_the_requesting_codex_parses(client, auth_as, monkeypatch, params): + """Both listing paths hand the route's ``client_version`` to the catalog: Codex before 0.138.0 cannot + parse max, so it is offered there only to a newer release.""" + declared = {"supports_reasoning": True, "reasoning_effort_levels": ["low", "max"]} + vllm = {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"} + router = litellm.Router( + model_list=[ + {"model_name": name, "litellm_params": vllm, "model_info": declared} for name in ("gpt-4", "claude-sonnet") + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.model_list) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prisma_client", None) + + with auth_as(): + current = client.get("/v1/models", params={**params, "client_version": "0.165.0"}) + older = client.get("/v1/models", params={**params, "client_version": "0.137.0"}) + + assert [[level["effort"] for level in m["supported_reasoning_levels"]] for m in current.json()["models"]] == [ + ["low", "max"] + ] * 2 + assert [[level["effort"] for level in m["supported_reasoning_levels"]] for m in older.json()["models"]] == [ + ["low"] + ] * 2 + + @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) def test_codex_service_tiers_are_read_for_the_key_team(client, auth_as, patched_models, params): """A tier and the upstream model that picks Codex's stock entry are read off the deployments the diff --git a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py index 11ea33e61c7..143f5bfb83b 100644 --- a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py @@ -30,7 +30,7 @@ class _FakeRouter: def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo | None: return self.infos.get(model_group) - def get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: + def get_model_list_of_routed_group(self, model_group: str, team_id: str | None = None) -> list[DeploymentTypedDict]: return [] diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index afc79f36caa..41acd487b67 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -50,7 +50,7 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: if source == "routing_groups": return router.get_model_list_from_routing_groups(model_name=model_name) if source == "routed_group": - return router.get_model_list_of_routed_group(model_name) + return router.get_model_list_of_routed_group(model_name, **_given(team_id=team_id)) if source == "routable": return router.get_routable_deployments( model_name, *(() if request is None else (request,)), **_given(call_type=call_type) diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 1d8ee338cbb..ce4cb84addf 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -8,7 +8,6 @@ import pytest import litellm from litellm._logging import JsonFormatter, verbose_logger from litellm.router_utils.reasoning_effort_capability import ( - REASONING_EFFORT_STRENGTH_ORDER, RequestedReasoningEffort, deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -17,6 +16,7 @@ from litellm.router_utils.reasoning_effort_capability import ( resolve_supported_reasoning_efforts, thinking_budget_effort, ) +from litellm.litellm_core_utils.reasoning_effort_strength import REASONING_EFFORT_STRENGTH_ORDER class TestDeploymentIsCatalogMapped: diff --git a/tests/unit/test_default_branch.py b/tests/unit/test_default_branch.py index bb04ea2a0fa..577b1126976 100644 --- a/tests/unit/test_default_branch.py +++ b/tests/unit/test_default_branch.py @@ -119,6 +119,17 @@ def test_explicit_base_works_without_remote_access( assert "No changed litellm Python files" in checked.stdout +def test_format_check_reads_staged_renames(remote_and_clone: tuple[Path, Path]) -> None: + _, repo = remote_and_clone + (repo / "litellm" / "before_move.py").write_text("value = 2\n") + _commit(repo, "add module") + _git(repo, "mv", "litellm/before_move.py", "litellm/after_move.py") + checked: Final = _make(repo, "lint-format-check-changed", "BASE_REF=origin/release_branch", "UV_RUN=echo") + assert checked.returncode == 0, checked.stdout + checked.stderr + assert "litellm/after_move.py" in checked.stdout + assert "before_move.py" not in checked.stdout + + def _freshness(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [ diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index 2df5cbc9457..2af7db930c6 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -592,9 +592,9 @@ FORWARDS = [ ((), {"model_name": "gpt"}), ), ( - lambda r: r.registry().deployments("gpt", source="routed_group"), + lambda r: r.registry().deployments("gpt", source="routed_group", team_id="t"), "_core_get_model_list_of_routed_group", - (("gpt",), {}), + (("gpt", "t"), {}), ), ( lambda r: r.registry().deployments("gpt", source="wildcard", team_id="t"), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d542880f868..90a0597cea5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10547,8 +10547,10 @@ export interface paths { * - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a * provider's model catalog. When present, the response is Codex's own * catalog shape (`{"models": [...]}`) built from the same listing, with - * each model's `model_info.service_tiers` as its service tiers; absent, - * the OpenAI shape below is returned + * each model's `model_info.service_tiers` as its service tiers and, for a + * model that Codex release does not know, the reasoning efforts + * `/model_group/info` reports that it can parse; absent, the OpenAI shape + * below is returned * - include_metadata: Include additional metadata in the response with fallback information * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true @@ -21465,8 +21467,10 @@ export interface paths { * - client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a * provider's model catalog. When present, the response is Codex's own * catalog shape (`{"models": [...]}`) built from the same listing, with - * each model's `model_info.service_tiers` as its service tiers; absent, - * the OpenAI shape below is returned + * each model's `model_info.service_tiers` as its service tiers and, for a + * model that Codex release does not know, the reasoning efforts + * `/model_group/info` reports that it can parse; absent, the OpenAI shape + * below is returned * - include_metadata: Include additional metadata in the response with fallback information * - fallback_type: Type of fallbacks to include ("general", "context_window", "content_policy") * Defaults to "general" when include_metadata=true From dacaaad72ed25663ad7f40941c48f653a45289be Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 29 Oct 2026 18:20:47 -0700 Subject: [PATCH 09/25] build: allow filelock 4.x and sync package metadata (#49137) * build: allow filelock 4.x filelock 3.x is the only line the current cap permits, and three CVEs (CVE-2026-112940, CVE-2026-112913, CVE-2026-112907) are fixed only in 4.2.1+. The cap means every downstream that depends on litellm is held on a vulnerable filelock with no way out, since pip has no override. litellm uses FileLock(path, timeout=..., mode=...) and Timeout, in litellm/proxy/client/cli/commands/configure_profiles.py and claude_settings.py. Both keyword arguments still exist in 4.2.4 and Timeout is still exported, so the two call sites are unaffected. The CVEs are in the SQLite-backed ReadWriteLock / AsyncReadWriteLock, which litellm does not use. <5 rather than a floor bump because litellm supports Python 3.10 and filelock 4.x requires >=3.11: widening the cap lets 3.11+ resolve 4.x while 3.10 keeps a 3.x filelock automatically, through filelock's own requires_python. * build: sync core filelock bound and dependency lock --------- Co-authored-by: Speculator55005 <50082482+fas89@users.noreply.github.com> --- packaging/litellm-core/pyproject.toml | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index db50589a675..1141e69c0aa 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "fastuuid>=0.14.0,<1.0; python_version < '3.15'", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<5", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dateutil>=2.8.2,<3.0", diff --git a/pyproject.toml b/pyproject.toml index bff00a2dbf1..de1abeddd7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0; python_version < '3.15'", - "filelock>=3.16.1,<4.0", + "filelock>=3.16.1,<5", "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<4.0.0", "python-dotenv>=1.0.0,<2.0", diff --git a/uv.lock b/uv.lock index f36fd3495a9..2ba97d15327 100644 --- a/uv.lock +++ b/uv.lock @@ -5242,7 +5242,7 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'proxy'", specifier = ">=0.136.3,<1.0" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = ">=0.19.0,<1.0" }, { name = "fastuuid", marker = "python_full_version < '3.15'", specifier = ">=0.14.0,<1.0" }, - { name = "filelock", specifier = ">=3.16.1,<4.0" }, + { name = "filelock", specifier = ">=3.16.1,<5" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = ">=1.133.0,<2.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = ">=2.19.1,<3.0" }, From 636e70708984527aa2a4b89fdeb16886f7cff278 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:20:52 -0700 Subject: [PATCH 10/25] fix(proxy)!: stop answering with the literal string "None" as error type and param on the proxy-server, auth, spend, and key health routes (#39540) * fix(proxy): stop answering with the literal string "None" on the proxy-server, auth, and health routes The error objects these routes return defaulted `type` and `param` to the four-character string "None", so a client's error handler matched no known OpenAI type and fell into its generic branch. Route them through the shared openai_error_payload helpers instead, so `type` comes from the carried type or the HTTP status and `param` serializes as JSON null. * fix(proxy): stop answering with the literal string "None" on the spend and management routes The error objects these routes return defaulted `type` and `param` to the four-character string "None", so a client's error handler matched no known OpenAI type and fell into its generic branch. Route them through the shared openai_error_payload helpers instead, so `type` comes from the carried type or the HTTP status and `param` serializes as JSON null. * test(proxy): pin the mid-stream failure frame and the DB outage body to a JSON null param * fix(proxy): type upstream timeouts as timeout_error and update integration asserts for the null param A 408 with no carried type now answers timeout_error instead of invalid_request_error, so a client branching on type keeps retrying it. Three integration suites still compared auth rejections against param="None". * fix(proxy): keep a 408 on main's invalid_request_error type and pin the transcription file cap's 400 * fix(proxy): type a route failure by the status it answers with after the tag-routing and no-deployment remap * fix(proxy): keep auth_error on proxy-raised 401/403 route failures and ban the string None as an error type or param * fix(proxy): type a hook's 401 or 403 on completions, moderations and transcription by its status, as chat does Only a status the proxy remapped into 401 or 403, such as a tag-routing denial, keeps auth_error. * test(proxy): build the transcription file-cap test on a real router so it holds after the router refactor --- .github/workflows/test-linting.yml | 6 + litellm/proxy/auth/auth_exception_handler.py | 7 +- litellm/proxy/auth/handle_jwt.py | 2 +- litellm/proxy/auth/route_checks.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 12 +- .../common_utils/openai_error_payload.py | 31 ++++- litellm/proxy/db/exception_handler.py | 2 +- .../key_management_endpoints.py | 6 +- .../team_callback_endpoints.py | 11 +- litellm/proxy/proxy_server.py | 106 ++++++++--------- .../spend_management_endpoints.py | 49 ++++---- .../check_no_none_string_error_fields.py | 71 ++++++++++++ .../test_check_no_none_string_error_fields.py | 38 +++++++ .../test_guardrail_submissions.py | 2 +- .../test_jwt_key_mapping_lifecycle.py | 4 +- .../authorization/test_memory_ownership.py | 2 +- .../authorization/test_team_admin_gate.py | 2 +- .../test_unknown_key_generic_401.py | 8 +- .../test_user_self_update_guard.py | 2 +- .../authorization/test_warmed_policy.py | 2 +- .../authorization/test_workflow_ownership.py | 2 +- .../management/test_health_routing.py | 2 +- .../management/test_scim_admin_group.py | 2 +- .../test_user_delete_revokes_keys.py | 2 +- ...st_arize_otel_v2_openinference_sad_edge.py | 2 +- .../test_presidio_entity_masking.py | 2 +- .../security/test_config_overrides.py | 6 +- .../spend/test_global_spend_reset.py | 2 +- .../proxy/auth/test_auth_exception_handler.py | 14 +++ tests/unit/proxy/auth/test_handle_jwt.py | 1 + .../unit/proxy/auth/test_user_api_key_auth.py | 27 +++++ .../test_internal_user_endpoints.py | 2 +- .../test_key_management_endpoints.py | 17 +++ .../test_team_endpoints.py | 2 +- .../proxy_server/test_exception_handlers.py | 2 +- .../proxy/proxy_server/test_routes_audio.py | 107 +++++++++++++++++- .../proxy_server/test_routes_completions.py | 38 +++++++ .../proxy_server/test_streaming_helpers.py | 35 +++++- .../test_spend_management_endpoints.py | 16 +++ ...test_proxy_server_endpoints_and_startup.py | 90 +++++++++++++++ 40 files changed, 606 insertions(+), 130 deletions(-) create mode 100644 tests/code_coverage_tests/check_no_none_string_error_fields.py create mode 100644 tests/code_coverage_tests/test_check_no_none_string_error_fields.py diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index bd7343ea222..88c66f0011d 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -630,6 +630,12 @@ jobs: - name: check_migrations_lock_timeout run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py + - name: check_no_none_string_error_fields + run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py + + - name: test_check_no_none_string_error_fields + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py + - name: check_no_publicly_known_master_key run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index ed743764fce..2ec2274db2b 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor ) from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError +from litellm.proxy.common_utils.openai_error_payload import openai_error_param from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.types.services import ServiceTypes @@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException: message=str(e.detail), internal_message=e.internal_message, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=e.status_code, ) if isinstance(e, HTTPException): return ProxyException( message=getattr(e, "detail", f"Authentication Error({e})"), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED), ) if isinstance(e, ProxyException): @@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException: return ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_401_UNAUTHORIZED, ) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9d800a1fd63..7bd7b1226b1 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException: f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}" ), type=ProxyErrorTypes.auth_provider_unavailable, - param="None", + param=None, code=status.HTTP_503_SERVICE_UNAVAILABLE, ) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 56be44600fd..105acb237af 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -302,7 +302,7 @@ class RouteChecks: f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2b6471952fd..67410e867d6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2586,7 +2586,7 @@ async def user_api_key_auth_builder( raise ProxyException( message="Authentication Error, No api key passed in.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) elif api_key == "": @@ -2594,7 +2594,7 @@ async def user_api_key_auth_builder( raise ProxyException( message="Authentication Error, Malformed API Key passed in. Ensure Key has `Bearer ` prefix.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2835,7 +2835,7 @@ async def user_api_key_auth_builder( raise ProxyException( message=f"Authentication Error, Tried to access route={route}, which is only for MASTER KEY", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2968,7 +2968,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e raise ProxyException( message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) await enforce_key_and_fallback_model_access( @@ -3019,7 +3019,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e "Keys owned by this user cannot be used." ), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3325,7 +3325,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e raise ProxyException( message="Authentication Error, Invalid proxy server token passed", type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) if valid_token_dict is not None: diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index f92fbeef9a1..53b2a05d40f 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -147,8 +147,37 @@ def _is_failover_refusal(exc: Exception) -> bool: return any(refusal in message for refusal in _FAILOVER_REFUSALS) +def answered_status_code(message: str, status_code: int) -> int: + return int(proxy_exception_status_code(message, str(status_code))) + + +def route_failure_proxy_exception( + exc: Exception, + message: str, + default_status_code: int, + headers: dict[str, str] | None = None, # mutable-ok: ProxyException.headers + openai_code: str | None = None, +) -> ProxyException: + raised_status: Final = error_status_code(exc, default_status_code) + answered_status: Final = answered_status_code(message, raised_status) + return ProxyException( + message=message, + type=_route_failure_error_type(exc, raised_status, answered_status), + param=openai_error_param(exc), + headers=headers, + openai_code=openai_code, + code=answered_status, + ) + + +def _route_failure_error_type(exc: Exception, raised_status: int, answered_status: int) -> str: + if answered_status != raised_status and answered_status in _AUTH_FAILURE_STATUS_CODES: + return ProxyErrorTypes.auth_error.value + return openai_error_type(exc, answered_status) + + def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: - answered_status: Final = int(proxy_exception_status_code(message, str(status_code))) + answered_status: Final = answered_status_code(message, status_code) return ProxyException( message=message, type=_catch_all_error_type(exc, answered_status), diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index 3f7ea334a99..baf53604ea5 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -435,7 +435,7 @@ class PrismaDBExceptionHandler: return ProxyException( message=PrismaDBExceptionHandler.database_unavailable_message(e), type=ProxyErrorTypes.no_db_connection, - param="None", + param=None, code=HTTPStatus.SERVICE_UNAVAILABLE.value, ) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 69f28b6d388..a6814b593ca 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -75,7 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) -from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for +from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache @@ -4959,7 +4959,7 @@ async def info_key_fn( raise ProxyException( message="Internal server error from /key/info. Details are in the proxy server logs.", type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -8156,7 +8156,7 @@ async def key_health( raise ProxyException( message=f"Key health check failed: {e}", type=ProxyErrorTypes.internal_server_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index d1e1c4bd25e..5b778392664 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -46,6 +46,7 @@ from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy encrypt_callback_vars, is_sensitive_callback_key, ) +from litellm.proxy.common_utils.openai_error_payload import openai_error_param from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy module exports _get_validated_callback_metadata, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export convert_key_logging_metadata_to_callback, @@ -463,7 +464,7 @@ async def add_team_callbacks( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -606,7 +607,7 @@ async def delete_team_callback( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) else: @@ -747,7 +748,7 @@ async def disable_team_logging( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -838,7 +839,7 @@ async def get_team_callbacks( raise ProxyException( message=getattr(e, "detail", f"Internal Server Error({e})"), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -846,6 +847,6 @@ async def get_team_callbacks( raise ProxyException( message="Internal Server Error, " + str(e), type=ProxyErrorTypes.internal_server_error.value, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bd221bca98..b0ccf74c63b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -529,9 +529,12 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, + attribute_of, headers_with_litellm_call_id, litellm_call_id_headers, + openai_error_param, proxy_exception_for, + route_failure_proxy_exception, with_litellm_call_id, ) from litellm.proxy.common_utils.periodic_reload_schedule import ( @@ -9898,11 +9901,8 @@ async def async_assistants_data_generator(response, user_api_key_dict: UserAPIKe # Including it in the SSE response leaks internal details to clients. error_msg: Final = str(e) - proxy_exception: Final = ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + proxy_exception: Final = route_failure_proxy_exception( + e, str(attribute_of(e, "message", error_msg)), status.HTTP_500_INTERNAL_SERVER_ERROR ) error_returned: Final = json.dumps({"error": proxy_exception.to_dict()}) yield f"data: {error_returned}\n\n" @@ -10712,11 +10712,10 @@ async def async_data_generator( { "error": sse_error_payload(client_error)[1] if isinstance(client_error, HTTPException) - else ProxyException( - message=getattr(client_error, "message", str(client_error)), - type=getattr(client_error, "type", "None"), - param=getattr(client_error, "param", "None"), - code=getattr(client_error, "status_code", 500), + else route_failure_proxy_exception( + client_error, + str(attribute_of(client_error, "message", client_error)), + status.HTTP_500_INTERNAL_SERVER_ERROR, ).to_dict() }, default=str, @@ -13096,14 +13095,12 @@ async def completion( log_llm_api_exception(e, litellm_call_id) if isinstance(e, ProxyException): raise with_litellm_call_id(e, litellm_call_id) - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) @@ -13324,22 +13321,18 @@ async def moderations( if isinstance(e, ProxyException): raise with_litellm_call_id(e, litellm_call_id) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_400_BAD_REQUEST, headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", 500), ) + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, + headers=litellm_call_id_headers(litellm_call_id), + ) async def _audio_speech_chunk_generator( @@ -13477,13 +13470,12 @@ async def audio_speech( detail=e.detail, headers=headers_with_litellm_call_id(e.headers, litellm_call_id), ) - raise ProxyException( - message=getattr(e, "message", f"{e}"), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) @@ -13623,23 +13615,19 @@ async def audio_transcriptions( ) log_llm_api_exception(e, litellm_call_id) if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e.detail)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e.detail)), + status.HTTP_400_BAD_REQUEST, headers=litellm_call_id_headers(litellm_call_id), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg: Final = f"{e}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - headers=litellm_call_id_headers(litellm_call_id), - openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), ) + raise route_failure_proxy_exception( + e, + str(attribute_of(e, "message", e)), + status.HTTP_500_INTERNAL_SERVER_ERROR, + headers=litellm_call_id_headers(litellm_call_id), + openai_code=getattr(e, "code", None), + ) ###################################################################### @@ -17239,7 +17227,7 @@ async def model_streaming_metrics( raise ProxyException( message=CommonProxyErrors.db_not_connected_error.value, type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -17373,7 +17361,7 @@ async def model_metrics( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) startTime = startTime or datetime.now() - timedelta(days=DAYS_IN_A_MONTH) @@ -17488,7 +17476,7 @@ async def model_metrics_slow_responses( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) if api_key is None or api_key == "undefined": @@ -17577,7 +17565,7 @@ async def model_metrics_exceptions( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18791,7 +18779,7 @@ async def login_v2(request: Request): raise ProxyException( message=getattr(e, "detail", str(e)), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: @@ -18799,7 +18787,7 @@ async def login_v2(request: Request): raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18867,7 +18855,7 @@ async def login_v3(request: Request): raise ProxyException( message=getattr(e, "detail", str(e)), type=ProxyErrorTypes.auth_error, - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) else: @@ -18875,7 +18863,7 @@ async def login_v3(request: Request): raise ProxyException( message=error_msg, type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -18940,7 +18928,7 @@ async def login_v3_exchange(request: Request): raise ProxyException( message=str(e), type=ProxyErrorTypes.auth_error, - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 38dcc3b06df..89200792c0b 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -50,6 +50,7 @@ from litellm.proxy.auth.authorization_dependencies import ( LogTeamLookupDependency, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, @@ -636,7 +637,7 @@ async def view_spend_tags( raise ProxyException( message=getattr(e, "detail", f"/spend/tags Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -644,7 +645,7 @@ async def view_spend_tags( raise ProxyException( message="/spend/tags Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2142,7 +2143,7 @@ async def global_get_all_tag_names(): raise ProxyException( message=getattr(e, "detail", f"/spend/all_tag_names Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -2150,7 +2151,7 @@ async def global_get_all_tag_names(): raise ProxyException( message="/spend/all_tag_names Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2224,7 +2225,7 @@ async def global_view_spend_tags( raise ProxyException( message=getattr(e, "detail", f"/spend/tags Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -2232,7 +2233,7 @@ async def global_view_spend_tags( raise ProxyException( message="/spend/tags Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -2447,9 +2448,9 @@ async def calculate_spend(request: SpendCalculateRequest): if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", str(e)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + type=openai_error_type(e, error_status_code(e, status.HTTP_400_BAD_REQUEST)), + param=openai_error_param(e), + code=error_status_code(e, status.HTTP_400_BAD_REQUEST), ) if isinstance(e, litellm.exceptions.ModelNotMappedError): raise ProxyException( @@ -2461,9 +2462,9 @@ async def calculate_spend(request: SpendCalculateRequest): error_msg: Final = f"{e}" raise ProxyException( message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), + type=openai_error_type(e, error_status_code(e, 500)), + param=openai_error_param(e), + code=error_status_code(e, 500), ) @@ -2633,7 +2634,7 @@ async def ui_view_spend_logs( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -2700,7 +2701,7 @@ async def ui_view_spend_logs( raise ProxyException( message="Start date and end date are required", type="bad_request", - param="None", + param=None, code=status.HTTP_400_BAD_REQUEST, ) formats: Final = ["%Y-%m-%d %H:%M:%S", "%Y-%m-%d"] if is_v2 else ["%Y-%m-%d %H:%M:%S"] @@ -3681,7 +3682,7 @@ async def view_spend_logs( raise ProxyException( message=getattr(e, "detail", f"/spend/logs Error({e})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3689,7 +3690,7 @@ async def view_spend_logs( raise ProxyException( message="/spend/logs Error" + str(e), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3716,7 +3717,7 @@ async def global_spend_reset(): raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3747,7 +3748,7 @@ async def global_spend_refresh(): raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_401_UNAUTHORIZED, ) @@ -3818,7 +3819,7 @@ async def global_spend_for_internal_user( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) try: @@ -3879,7 +3880,7 @@ async def global_spend_logs( raise ProxyException( message="Prisma Client is not initialized", type="internal_error", - param="None", + param=None, code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3923,7 +3924,7 @@ async def global_spend_logs( raise ProxyException( message=getattr(e, "detail", f"/global/spend/logs Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3931,7 +3932,7 @@ async def global_spend_logs( raise ProxyException( message="/global/spend/logs Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) @@ -3971,7 +3972,7 @@ async def global_spend(): raise ProxyException( message=getattr(e, "detail", f"/global/spend Error({error_str})"), type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), ) elif isinstance(e, ProxyException): @@ -3979,7 +3980,7 @@ async def global_spend(): raise ProxyException( message="/global/spend Error" + error_str, type="internal_error", - param=getattr(e, "param", "None"), + param=openai_error_param(e), code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) diff --git a/tests/code_coverage_tests/check_no_none_string_error_fields.py b/tests/code_coverage_tests/check_no_none_string_error_fields.py new file mode 100644 index 00000000000..3a75b56fc8d --- /dev/null +++ b/tests/code_coverage_tests/check_no_none_string_error_fields.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import ast +import sys +from collections.abc import Iterator +from itertools import chain +from pathlib import Path +from typing import Final + +ERROR_FIELDS: Final = frozenset({"type", "param"}) +SOURCE_ROOT: Final = Path(__file__).resolve().parents[2] / "litellm" + + +def _is_none_string(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value == "None" + + +def _names_an_error_field(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value in ERROR_FIELDS + + +def _keyword_defaults_to_none_string(call: ast.Call) -> bool: + return any(keyword.arg in ERROR_FIELDS and _is_none_string(keyword.value) for keyword in call.keywords) + + +def _getattr_defaults_to_none_string(call: ast.Call) -> bool: + return ( + isinstance(call.func, ast.Name) + and call.func.id == "getattr" + and len(call.args) == 3 + and _names_an_error_field(call.args[1]) + and _is_none_string(call.args[2]) + ) + + +def _dict_holds_none_string(node: ast.Dict) -> bool: + return any(_names_an_error_field(key) and _is_none_string(value) for key, value in zip(node.keys, node.values)) + + +def _is_violation(node: ast.AST) -> bool: + match node: + case ast.Call(): + return _keyword_defaults_to_none_string(node) or _getattr_defaults_to_none_string(node) + case ast.Dict(): + return _dict_holds_none_string(node) + case _: + return False + + +def violations(source: str, filename: str) -> Iterator[str]: + return (f"{filename}:{node.lineno}" for node in ast.walk(ast.parse(source, filename=filename)) if _is_violation(node)) + + +def _file_violations(path: Path) -> Iterator[str]: + return violations(path.read_text(encoding="utf-8"), str(path.relative_to(SOURCE_ROOT.parent))) + + +def main() -> int: + found: Final = tuple(chain.from_iterable(_file_violations(path) for path in sorted(SOURCE_ROOT.rglob("*.py")))) + if not found: + return 0 + print( + 'An error\'s "type" or "param" is set to the string "None", which OpenAI clients read as a real ' + "error class or field name. Use a named error type and a real None for param instead:" + ) + print("\n".join(found)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/test_check_no_none_string_error_fields.py b/tests/code_coverage_tests/test_check_no_none_string_error_fields.py new file mode 100644 index 00000000000..5d2f04b0942 --- /dev/null +++ b/tests/code_coverage_tests/test_check_no_none_string_error_fields.py @@ -0,0 +1,38 @@ +import importlib.util +from pathlib import Path +from typing import Final + +import pytest + +GUARD_PATH: Final = Path(__file__).resolve().parent / "check_no_none_string_error_fields.py" +_spec: Final = importlib.util.spec_from_file_location("check_no_none_string_error_fields", GUARD_PATH) +assert _spec is not None and _spec.loader is not None +guard: Final = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(guard) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param('raise ProxyException(message="m", type="None", param=None, code=500)\n', id="type_keyword"), + pytest.param('raise ProxyException(message="m", type="x", param="None", code=500)\n', id="param_keyword"), + pytest.param('ProxyException(message="m", type=getattr(e, "type", "None"), param=None)\n', id="type_getattr"), + pytest.param('ProxyException(message="m", type="x", param=getattr(e, "param", "None"))\n', id="param_getattr"), + pytest.param('error = {"message": "m", "type": "x", "param": "None"}\n', id="dict_literal"), + ], +) +def test_a_none_string_error_field_is_reported_with_its_line(source: str) -> None: + assert tuple(guard.violations("x = 1\n" + source, "proxy.py")) == ("proxy.py:2",) + + +@pytest.mark.parametrize( + "source", + [ + pytest.param('ProxyException(message="m", type="auth_error", param=None, code=401)\n', id="real_none_param"), + pytest.param('ProxyException(message="None", type="x", param=getattr(e, "param", None))\n', id="none_message"), + pytest.param('label = getattr(e, "provider", "None")\n', id="other_attribute"), + pytest.param('row = {"model": "None"}\n', id="other_dict_key"), + ], +) +def test_a_real_none_or_an_unrelated_none_string_passes(source: str) -> None: + assert tuple(guard.violations(source, "proxy.py")) == () diff --git a/tests/integration/authorization/test_guardrail_submissions.py b/tests/integration/authorization/test_guardrail_submissions.py index f120028f46d..16b9cbf31dc 100644 --- a/tests/integration/authorization/test_guardrail_submissions.py +++ b/tests/integration/authorization/test_guardrail_submissions.py @@ -310,7 +310,7 @@ def test_non_admin_keys_scoped_to_the_submission_routes_get_403_from_the_admin_c f"Your user_id={_mask_user_id(user_id)}" ), type="auth_error", - param="None", + param=None, code="401", ) ) diff --git a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py index 715f3731175..da830293145 100644 --- a/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py +++ b/tests/integration/authorization/test_jwt_key_mapping_lifecycle.py @@ -114,7 +114,7 @@ class _ChatRequest(BaseModel): class _ProxyError(BaseModel): message: str type: str - param: str + param: str | None code: str @@ -398,7 +398,7 @@ def _assert_unmapped(response: httpx.Response, client_id: str) -> None: assert response.status_code == 403, response.text error: Final = _ErrorResponse.model_validate_json(response.content) expected: Final = f"JWT Key Mapping: No registered mapping for {_CLIENT_CLAIM}='{client_id}'. Access denied." - assert error.error == _ProxyError(message=expected, type="auth_error", param="None", code="403"), response.text + assert error.error == _ProxyError(message=expected, type="auth_error", param=None, code="403"), response.text def _mapping_info(gateway: Gateway, identity: str) -> _JWTMappingResponse: diff --git a/tests/integration/authorization/test_memory_ownership.py b/tests/integration/authorization/test_memory_ownership.py index bad7c3ccb37..249b74363a2 100644 --- a/tests/integration/authorization/test_memory_ownership.py +++ b/tests/integration/authorization/test_memory_ownership.py @@ -24,7 +24,7 @@ def _admin_only_error(route: str, user_id: str) -> dict[str, JsonValue]: f"keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_team_admin_gate.py b/tests/integration/authorization/test_team_admin_gate.py index eff4060631e..adc374a57d7 100644 --- a/tests/integration/authorization/test_team_admin_gate.py +++ b/tests/integration/authorization/test_team_admin_gate.py @@ -600,7 +600,7 @@ def _admin_only_route(route: str) -> tuple[int, JsonValue]: "message": "Authentication Error, Only proxy admin can be used to generate, delete, update info for new " f"keys/users/teams. Route={route}. Your role=internal_user. Your user_id=", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_unknown_key_generic_401.py b/tests/integration/authorization/test_unknown_key_generic_401.py index 8e6ff3fa3ab..025f11d3973 100644 --- a/tests/integration/authorization/test_unknown_key_generic_401.py +++ b/tests/integration/authorization/test_unknown_key_generic_401.py @@ -40,7 +40,7 @@ class _ProxyError(BaseModel): message: str type: str - param: str + param: str | None code: str @@ -260,12 +260,12 @@ def _control_error(response: httpx.Response) -> _ProxyError: def test_requests_without_an_sk_key_keep_their_own_401(gateway: Gateway, peer: Gateway, side: Side) -> None: client: Final = _side(gateway, peer, side).client assert _control_error(client.get("/v1/models")) == _ProxyError( - message="Authentication Error, No api key passed in.", type="auth_error", param="None", code="401" + message="Authentication Error, No api key passed in.", type="auth_error", param=None, code="401" ) assert _control_error(client.get("/v1/models", headers={"Authorization": "Bearer not-a-real-key"})) == _ProxyError( message="LiteLLM Virtual Key expected. Received=not-****-key, expected to start with 'sk-'.", type="auth_error", - param="None", + param=None, code="401", ) @@ -277,7 +277,7 @@ def test_a_blocked_key_keeps_its_own_401_on_both_proxies(gateway: Gateway, peer: expected: Final = _ProxyError( message="Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", type="auth_error", - param="None", + param=None, code="401", ) for candidate in (gateway, peer): diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py index d058b3c6998..2c495beb77e 100644 --- a/tests/integration/authorization/test_user_self_update_guard.py +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -65,7 +65,7 @@ def _ui_session_route_error(user_id: str) -> dict[str, JsonValue]: f"Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_warmed_policy.py b/tests/integration/authorization/test_warmed_policy.py index b4ffc10c3c4..0de4141143e 100644 --- a/tests/integration/authorization/test_warmed_policy.py +++ b/tests/integration/authorization/test_warmed_policy.py @@ -679,7 +679,7 @@ def _expected_scim_write_denial(route: str, user_id: str, user_role: str) -> dic f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/authorization/test_workflow_ownership.py b/tests/integration/authorization/test_workflow_ownership.py index 386d4f1ed27..70919760874 100644 --- a/tests/integration/authorization/test_workflow_ownership.py +++ b/tests/integration/authorization/test_workflow_ownership.py @@ -57,7 +57,7 @@ def _route_refusal(response: httpx.Response, user_id: str) -> None: f"keys/users/teams. Route=/v1/workflows/runs. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, response.text diff --git a/tests/integration/management/test_health_routing.py b/tests/integration/management/test_health_routing.py index 3a60efe4342..b74d32380d0 100644 --- a/tests/integration/management/test_health_routing.py +++ b/tests/integration/management/test_health_routing.py @@ -151,7 +151,7 @@ def test_public_readiness_reports_stalled_database_lookups(gateway: Gateway, tmp "Please retry shortly." ), "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/integration/management/test_scim_admin_group.py b/tests/integration/management/test_scim_admin_group.py index ffbec93d4a8..92016af1ebb 100644 --- a/tests/integration/management/test_scim_admin_group.py +++ b/tests/integration/management/test_scim_admin_group.py @@ -133,7 +133,7 @@ def _expected_scim_write_denial(route: str, user_id: str, user_role: str) -> dic f"keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 324963a0da9..513a80b7319 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -64,7 +64,7 @@ def _blocked_key_error() -> dict[str, JsonValue]: "error": { "message": "Authentication Error, Key is blocked. Update via `/key/unblock` if you're an admin.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } } diff --git a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py index a05b3f0f045..94e9d46ec8f 100644 --- a/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py +++ b/tests/integration/observability/test_arize_otel_v2_openinference_sad_edge.py @@ -177,7 +177,7 @@ def test_arize_otel_v2_d4_unauthenticated_request_has_no_span(gateway: Gateway, "error": { "message": "Authentication Error, No api key passed in.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, body diff --git a/tests/integration/observability/test_presidio_entity_masking.py b/tests/integration/observability/test_presidio_entity_masking.py index ab966873389..ff36e4623f7 100644 --- a/tests/integration/observability/test_presidio_entity_masking.py +++ b/tests/integration/observability/test_presidio_entity_masking.py @@ -325,7 +325,7 @@ def test_long_apply_guardrail_alias_accepts_an_llm_api_scoped_key(gateway: Gatew "error": { "message": "Authentication Error, No api key passed in.", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, unauthenticated.text diff --git a/tests/integration/security/test_config_overrides.py b/tests/integration/security/test_config_overrides.py index 72546c54222..70f9226ea79 100644 --- a/tests/integration/security/test_config_overrides.py +++ b/tests/integration/security/test_config_overrides.py @@ -100,7 +100,7 @@ def _assert_non_admin_refusal(response: httpx.Response, route: str, user_id: str f"new keys/users/teams. Route={route}. Your role=internal_user. Your user_id={masked_user_id}" ), "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, response.text @@ -244,7 +244,7 @@ def test_saved_vault_config_reads_back_masked_and_refuses_non_admins( "message": "user not allowed to access this route, role= proxy_admin_viewer. " "Trying to access: /config_overrides/hashicorp_vault", "type": "auth_error", - "param": "None", + "param": None, "code": "403", } }, viewer_post.text @@ -386,7 +386,7 @@ def test_saved_cyberark_config_reads_back_masked_and_refuses_non_admins( "message": "user not allowed to access this route, role= proxy_admin_viewer. " "Trying to access: /config_overrides/cyberark", "type": "auth_error", - "param": "None", + "param": None, "code": "403", } }, viewer_post.text diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py index c400d90e184..cac66471dea 100644 --- a/tests/integration/spend/test_global_spend_reset.py +++ b/tests/integration/spend/test_global_spend_reset.py @@ -104,7 +104,7 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: "error": { "message": "Authentication Error, Tried to access route=/global/spend/reset, which is only for MASTER KEY", "type": "auth_error", - "param": "None", + "param": None, "code": "401", } }, refused.text diff --git a/tests/unit/proxy/auth/test_auth_exception_handler.py b/tests/unit/proxy/auth/test_auth_exception_handler.py index 92b4c93118f..2523284ec2c 100644 --- a/tests/unit/proxy/auth/test_auth_exception_handler.py +++ b/tests/unit/proxy/auth/test_auth_exception_handler.py @@ -1298,3 +1298,17 @@ def test_as_proxy_exception_keeps_jwt_scope_denial_message_shape(): assert converted.to_dict() == plain.to_dict() assert converted.internal_message == denial.internal_message + + +@pytest.mark.parametrize( + "auth_failure", + [ + pytest.param(HTTPException(status_code=401, detail="Authentication Error, invalid key"), id="http_exception"), + pytest.param(Exception("Invalid proxy server token passed"), id="bare_exception"), + ], +) +def test_auth_failure_error_body_carries_a_json_null_param(auth_failure: Exception) -> None: + body = json.loads(json.dumps({"error": _as_proxy_exception(auth_failure).to_dict()})) + + assert body["error"]["type"] == ProxyErrorTypes.auth_error.value + assert body["error"]["param"] is None diff --git a/tests/unit/proxy/auth/test_handle_jwt.py b/tests/unit/proxy/auth/test_handle_jwt.py index a0ea6ce0403..51cf15996cd 100644 --- a/tests/unit/proxy/auth/test_handle_jwt.py +++ b/tests/unit/proxy/auth/test_handle_jwt.py @@ -4368,6 +4368,7 @@ async def test_removed_signing_key_stops_being_trusted_once_the_stale_window_exp assert exc_info.value.code == "503" assert exc_info.value.type == ProxyErrorTypes.auth_provider_unavailable + assert exc_info.value.param is None @pytest.mark.asyncio diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 117428ee27b..81ea0e3f60b 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -1310,6 +1310,33 @@ async def test_auth_rejects_wrong_or_empty_present_custom_header( assert exc_info.value.code == "401" +@pytest.mark.asyncio +@pytest.mark.parametrize("api_key", [None, ""], ids=["no_key", "no_bearer_prefix"]) +async def test_auth_rejection_without_a_usable_key_has_a_json_null_param( + api_key: str | None, monkeypatch: pytest.MonkeyPatch +): + from fastapi import Request + + from litellm.proxy import proxy_server + from litellm.proxy._types import ProxyErrorTypes, ProxyException + + monkeypatch.setattr(proxy_server, "master_key", "sk-test-master") + monkeypatch.setattr(proxy_server, "general_settings", {}) + request = Request( + scope={"type": "http", "method": "POST", "path": "/chat/completions", "headers": []}, + receive=AsyncMock(return_value={"type": "http.request", "body": b"", "more_body": False}), + ) + request._url = URL(url="/chat/completions") + + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth(request=request, api_key=api_key) + + body = json.loads(json.dumps({"error": exc_info.value.to_dict()})) + assert body["error"]["code"] == "401" + assert body["error"]["type"] == ProxyErrorTypes.auth_error.value + assert body["error"]["param"] is None + + @pytest.mark.asyncio async def test_user_api_key_from_query_param(): """Ensure user_api_key_auth reads API key from `key` query parameter.""" diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index d1bbf444d2c..78a0a765fe6 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4970,7 +4970,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 859824e4a72..b0ef7e64e60 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -23379,6 +23379,23 @@ def test_key_info_database_not_connected_sanitized_500( proxy_server.app.dependency_overrides.pop(user_api_key_auth, None) +def test_key_health_failure_body_carries_a_json_null_param(): + import litellm.proxy.proxy_server as ps + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="hashed-key", metadata={"logging": ["not-a-callback-object"]} + ) + try: + response = client.post("/key/health", headers={"Authorization": "Bearer sk-test"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 500 + error = response.json()["error"] + assert error["type"] == "internal_server_error" + assert error["param"] is None + + @pytest.mark.asyncio async def test_update_key_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None: from fastapi import Request diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index c6d12ade2a0..e33743bb149 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -19095,7 +19095,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/proxy_server/test_exception_handlers.py b/tests/unit/proxy/proxy_server/test_exception_handlers.py index b34c2d2fb08..0f9d379afa1 100644 --- a/tests/unit/proxy/proxy_server/test_exception_handlers.py +++ b/tests/unit/proxy/proxy_server/test_exception_handlers.py @@ -438,7 +438,7 @@ _DB_OUTAGE_503_BODY: Final = { "error": { "message": "Service Unavailable, the authentication database is temporarily unreachable. Please retry shortly.", "type": "no_db_connection", - "param": "None", + "param": None, "code": "503", } } diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index de76c7257cf..59d9490f786 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -14,9 +14,15 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi import HTTPException +from litellm import Router from litellm.proxy import proxy_server +from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import RouterErrors + +_TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']" @pytest.fixture @@ -208,7 +214,8 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path): with auth_as(): response = client.post(path, json=payload) assert response.status_code == 500 - assert len(response.content) > 0 + error = response.json()["error"] + assert (error["type"], error["param"]) == ("internal_server_error", None) def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection): @@ -245,6 +252,104 @@ def test_audio_transcription_happy_path(client, auth_as, patched_transcription, } +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_over_the_file_size_cap_is_rejected_as_a_bad_request( + client, auth_as, patched_transcription, monkeypatch, path +): + from litellm.proxy.common_utils.http_parsing_utils import check_file_size_under_limit + + capped_router = Router( + model_list=[ + {"model_name": "whisper-1", "litellm_params": {"model": "openai/whisper-1", "max_file_size_mb": 0.000001}} + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", capped_router) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", check_file_size_under_limit) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post(path, files=files, data={"model": "whisper-1"}) + assert response.status_code == 400 + error = response.json()["error"] + assert "File size is too large" in error["message"] + assert error["param"] == "file" + + +def test_audio_speech_tag_routing_denial_is_typed_as_a_proxy_auth_error( + client, auth_as, patched_speech_error, monkeypatch +): + async def _deny(*args, **kwargs): + raise ValueError(_TAG_ROUTING_DENIAL) + + monkeypatch.setattr(proxy_server, "route_request", _deny) + with auth_as(): + response = client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"}) + assert response.status_code == 401 + error = response.json()["error"] + assert (error["type"], error["param"]) == ("auth_error", None) + + +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + pytest.param(ValueError(_TAG_ROUTING_DENIAL), 401, "auth_error", id="tag_routing_denial"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + 403, + "permission_error", + id="hook_policy_block", + ), + ], +) +def test_audio_transcription_failure_is_typed_by_who_rejected_it( + client, auth_as, patched_transcription, monkeypatch, failure, expected_status, expected_type +): + async def _deny(*args, **kwargs): + raise failure + + monkeypatch.setattr(proxy_server, "route_request", _deny) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"}) + assert response.status_code == expected_status + error = response.json()["error"] + assert (error["type"], error["param"]) == (expected_type, None) + + +@pytest.mark.parametrize( + "rejection", + [ + pytest.param( + ProxyException( + message="Invalid type for 'metadata': expected an object, but got an integer instead.", + type=ProxyErrorTypes.bad_request_error, + param="metadata", + code=400, + ), + id="bad_metadata", + ), + pytest.param( + ProxyException( + message="tag budgets are unavailable", type=ProxyErrorTypes.no_db_connection, param=None, code=503 + ), + id="tag_budget_store_down", + ), + ], +) +def test_audio_transcription_keeps_the_status_of_a_proxy_rejection( + client, auth_as, patched_transcription, monkeypatch, rejection +): + async def _reject(data, **kwargs): + raise rejection + + monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _reject) + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + with auth_as(): + response = client.post("/v1/audio/transcriptions", files=files, data={"model": "whisper-1"}) + assert response.status_code == int(rejection.code) + assert response.json()["error"] == rejection.to_dict() + + @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) def test_audio_transcription_error(client, auth_as, patched_transcription_error, path): """Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (error).""" diff --git a/tests/unit/proxy/proxy_server/test_routes_completions.py b/tests/unit/proxy/proxy_server/test_routes_completions.py index b5c60c23c02..2b585357549 100644 --- a/tests/unit/proxy/proxy_server/test_routes_completions.py +++ b/tests/unit/proxy/proxy_server/test_routes_completions.py @@ -12,8 +12,10 @@ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock import pytest +from fastapi import HTTPException from litellm.proxy import common_request_processing, proxy_server +from litellm.types.router import RouterErrors from .conftest import normalize # type: ignore[import-not-found] @@ -124,3 +126,39 @@ def test_completion_pipeline_error(client, auth_as, completion_pipeline_raises, response = client.post(path, json=payload) assert response.status_code == 500 assert response.headers.get("content-type", "").startswith("application/json") + + +@pytest.mark.parametrize( + ("failure", "expected_status", "expected_type"), + [ + pytest.param(ValueError("boom"), 500, "internal_server_error", id="unexpected_failure"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + 403, + "permission_error", + id="hook_policy_block", + ), + pytest.param(HTTPException(status_code=401, detail="key expired"), 401, "authentication_error", id="hook_401"), + pytest.param( + ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-4o and tags=['x']"), + 401, + "auth_error", + id="tag_routing_denial", + ), + ], +) +def test_completion_failure_body_types_the_status_it_answers_with( + client, auth_as, monkeypatch, failure, expected_status, expected_type +): + monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", MagicMock(post_call_failure_hook=AsyncMock())) + + async def _raise(self, *args, **kwargs): + raise failure + + monkeypatch.setattr(common_request_processing.ProxyBaseLLMRequestProcessing, "base_process_llm_request", _raise) + with auth_as(): + response = client.post("/v1/completions", json={"model": "gpt-4o", "prompt": "hi"}) + assert response.status_code == expected_status + error = response.json()["error"] + assert (error["type"], error["param"], error["code"]) == (expected_type, None, str(expected_status)) diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 22ada990fe6..c37475bd539 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -32,7 +32,7 @@ import litellm import litellm.proxy.proxy_server as ps from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_request_processing import create_response from litellm.proxy.proxy_server import ( _apply_streaming_chunk_hooks, @@ -946,6 +946,39 @@ def _error_of(frame: str) -> dict[str, object]: return json.loads(frame.removeprefix("data: "))["error"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "raised, expected_error", + [ + pytest.param( + ProxyException(message="Budget has been exceeded", type="budget_exceeded", param=None, code=429), + {"message": "Budget has been exceeded", "type": "budget_exceeded", "param": None, "code": "429"}, + id="proxy_exception_keeps_its_status", + ), + pytest.param( + RuntimeError("upstream blew up"), + {"message": "upstream blew up", "type": "internal_server_error", "param": None, "code": "500"}, + id="unexpected_failure", + ), + ], +) +async def test_a_mid_stream_failure_frame_carries_the_status_the_error_was_raised_with( + monkeypatch, raised: Exception, expected_error: dict[str, object] +): + _patch_logging_flags(monkeypatch) + monkeypatch.setattr(ps.proxy_logging_obj, "post_call_failure_hook", AsyncMock(return_value=None)) + + frames: Final = [ + frame + async for frame in async_data_generator( + response=_async_iter_raises(raised), user_api_key_dict=_user_auth(), request_data={} + ) + ] + + error_frames: Final = [frame for frame in frames if isinstance(frame, str) and frame.startswith('data: {"error":')] + assert [_error_of(frame.removesuffix("\n\n")) for frame in error_frames] == [expected_error] + + @pytest.mark.asyncio @pytest.mark.parametrize( "detail,expected_message,expected_fields", diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 8d53817f51e..6598b94e60f 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -8268,3 +8268,19 @@ async def test_global_spend_refresh_runs_the_refresh_on_the_writer_when_a_reader assert [statement for statement in journal.statements if statement[1].startswith("REFRESH")] == [ ("writer", 'REFRESH MATERIALIZED VIEW "MonthlyGlobalSpend";') ] + + +def test_spend_calculate_rejection_body_is_openai_shaped(client): + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user", api_key="hashed-k" + ) + try: + response = client.post("/spend/calculate", json={}, headers={"Authorization": "Bearer sk-test"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] is None + assert error["code"] == "400" diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 023ef0b4606..62a90109d26 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -7255,6 +7255,39 @@ async def test_async_data_generator_closes_the_guardrail_chain_before_returning_ assert cleanup_ran == [True] +@pytest.mark.asyncio +async def test_async_data_generator_midstream_failure_frame_is_openai_shaped_with_a_json_null_param(): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import async_data_generator + + async def chain_failing_after_first_chunk(**_kwargs): + yield {"choices": [{"delta": {"content": "Hello"}}]} + raise ValueError("audit hook could not reach its sink") + + proxy_logging = _guarded_chain_logging(chain_failing_after_first_chunk) + proxy_logging.post_call_failure_hook = AsyncMock(return_value=None) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging): + frames = [ + frame + async for frame in async_data_generator( + MagicMock(), MagicMock(spec=UserAPIKeyAuth), {"model": "gpt-4o-mini"} + ) + ] + + error_bodies = [json.loads(frame.removeprefix("data: ")) for frame in frames if '"error"' in frame] + assert error_bodies == [ + { + "error": { + "message": "audit hook could not reach its sink", + "type": "internal_server_error", + "param": None, + "code": "500", + } + } + ] + + @pytest.mark.asyncio async def test_async_data_generator_closes_the_guardrail_chain_while_a_keepalive_read_is_pending(): from litellm.proxy._types import UserAPIKeyAuth @@ -14222,6 +14255,63 @@ async def test_moderations_reraises_proxy_exception_unwrapped(): mock_logging.post_call_failure_hook.assert_awaited_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "expected_type", "expected_code"), + [ + pytest.param(HTTPException(status_code=400, detail="bad input"), "invalid_request_error", "400", id="http_400"), + pytest.param(HTTPException(status_code=429, detail="slow down"), "rate_limit_error", "429", id="http_429"), + pytest.param( + HTTPException(status_code=403, detail="Blocked by the content policy"), + "permission_error", + "403", + id="hook_policy_block", + ), + pytest.param(Exception("provider down"), "internal_server_error", "500", id="bare_exception"), + pytest.param( + ValueError("Not allowed to access model due to tags configuration. Passed model=m and tags=['x']"), + "auth_error", + "401", + id="tag_routing_denial", + ), + ], +) +async def test_moderations_failure_body_is_openai_shaped(failure, expected_type, expected_code): + from litellm.proxy._types import ProxyException + + async def passthrough_add_litellm_data(data, **kwargs): + return data + + request = MagicMock() + request.body = AsyncMock(return_value=b'{"input": "hi"}') + fake_logging = MagicMock() + fake_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + fake_logging.post_call_failure_hook = AsyncMock() + + with ( + patch.object( + proxy_server_module, "add_litellm_data_to_request", new=passthrough_add_litellm_data + ), # test-quality-ok: the route reads this module global, no injection point + patch.object( + proxy_server_module, "route_request", new=AsyncMock(side_effect=failure) + ), # test-quality-ok: fakes the provider failure so the real route's error shaping is observable + patch.object( + proxy_server_module, "proxy_logging_obj", new=fake_logging + ), # test-quality-ok: module global, no injection point + pytest.raises(ProxyException) as raised, + ): + await proxy_server_module.moderations( + request=request, + fastapi_response=MagicMock(), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + + body = json.loads(json.dumps({"error": raised.value.to_dict()})) + assert body["error"]["type"] == expected_type + assert body["error"]["param"] is None + assert body["error"]["code"] == expected_code + + @pytest.mark.asyncio async def test_moderations_response_carries_litellm_call_id_header(): from fastapi import Response From 23ad8870b46ab984a922ae72ea7b9410542e11e0 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:24:04 -0700 Subject: [PATCH 11/25] fix(token_counter): price PDF pages for OpenAI and Azure models the way they bill (#47554) * test(token_counter): price PDF pages for OpenAI and Azure models the way they bill * fix(token_counter): price PDF pages for OpenAI and Azure models the way they bill * fix(token_counter): count azure, data-zone and dated GPT names with their family's encoding * fix(token_counter): price only drawn PDF images and keep other gateways' GPT pricing Detect raster images from the page content stream, including inline images on pages with empty resources, and walk each form once. Strip only openai/, azure/ and Azure data-zone prefixes, bound the model family scan, read Azure-only chat models from the cost map, and keep cl100k_base for listed non-chat OpenAI models. * fix(token_counter): price GPT names behind any prefix like the bare name and walk shared forms per resources Prices a PDF by the model name's last path segment, the same name the proxy counts with, keeps listed non-chat models on Claude pricing, walks each form once per inherited resources dictionary so a shared child form drawing an image is not missed, and resolves PDF pricing only when a message carries a PDF * fix(token_counter): keep the rendered-page count for image pages, parse each form once, and match chat models only A PDF page that draws a raster image keeps the rendered-page estimate, since newer GPT models bill such a page near it. The image walk parses each form once and stops at a fixed number of drawn XObjects, so a form shared under many resources dictionaries no longer slows the count. The family prefix match skips non-chat entries, and ft: fine-tunes count like their base model * fix(token_counter): count Azure-only chat models with o200k like other OpenAI chat models * fix(token_counter): count reservation and project ITPM with the serving deployment's model Budget reservation and project ITPM counted with the requested group name, so a GPT-named group served by Claude reserved a PDF at OpenAI's price. They now count with each deployment's model minus its provider prefix, the name count_tokens uses, and take the largest. A custom OpenAI tokenizer with no model counts with cl100k again instead of raising, and listed Azure gpt- chat entries price as OpenAI. * perf(token_counter): find drawn PDF images with a byte scan instead of parsing every operator pypdf parsed every operator of a page's content stream to find the images it draws, so a compressed page of a million operators took about 30 seconds to price for an OpenAI model. A linear scan of the decoded stream for name Do and inline images finds the same draws. * fix(token-counter): count each team deployment by its PDF pricing and honor base_model Budget reservation and the TPM limiter now look up the deployments that serve the caller's team, by base_model when set. A deployment that reads PDF pages and prices them unlike the group name counts with its own model, any other with the group name. A comment between an XObject name and Do now reads as whitespace, and the timing tests moved to CodSpeed benchmarks. * fix(benchmarks): install pypdf in the benchmark environment The PDF token-count benchmarks parse pages with pypdf, which only the proxy-runtime extra installed, so CodSpeed took the no-parser path. * fix(proxy): count PDF pages for deployments served under multi-segment model names * fix(proxy): count a team's wildcard route the way routing serves it and reserve OpenAI chat's per-PDF charge * fix(proxy): count routing-group and alias members with the caller's team, as routing serves them --- litellm/constants.py | 6 + litellm/litellm_core_utils/token_counter.py | 309 ++++++++++++- .../hooks/parallel_request_limiter_v3.py | 27 +- .../spend_tracking/budget_reservation.py | 36 +- litellm/proxy/spend_tracking/input_tokens.py | 111 ++++- pyproject.toml | 1 + .../test_token_counter_pdf_benchmarks.py | 126 +++++ .../litellm_core_utils/test_token_counter.py | 433 +++++++++++++++++- .../hooks/test_parallel_request_limiter_v3.py | 63 ++- .../spend_tracking/test_budget_reservation.py | 48 +- .../proxy/spend_tracking/test_input_tokens.py | 170 +++++++ tests/unit/rust_bridge/test_token_counter.py | 3 +- uv.lock | 2 + 13 files changed, 1272 insertions(+), 63 deletions(-) create mode 100644 tests/benchmarks/test_token_counter_pdf_benchmarks.py diff --git a/litellm/constants.py b/litellm/constants.py index d02114f56b1..466942d35be 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -753,7 +753,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches" ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568 ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000 ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750 +OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13 +OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192 PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64," +PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32 +PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000 +OPENAI_MODEL_NAME_MAX_DASHES: Final = 16 +OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"}) VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs" ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = { "low": 1, diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 5ad4e130e62..ea3e645ffdf 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -7,9 +7,11 @@ import re import struct from bisect import bisect_right from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from itertools import accumulate +from dataclasses import dataclass +from functools import cached_property +from itertools import accumulate, chain, islice from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast import anyio import anyio.lowlevel @@ -33,7 +35,12 @@ from litellm.constants import ( MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES, MAX_TILE_HEIGHT, MAX_TILE_WIDTH, + OPENAI_CHAT_MODES, + OPENAI_MODEL_NAME_MAX_DASHES, + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS, PDF_DATA_URL_PREFIX, + PDF_MAX_FORM_XOBJECT_DEPTH, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, TOKEN_COUNTER_MAX_EXACT_CHARS, @@ -606,6 +613,35 @@ TokenCounterFunction = Callable[[str], int] Type for a function that counts tokens in a string. """ +PdfPagePricing: TypeAlias = Literal["openai", "claude"] + + +class _PdfBox(Protocol): + @property + def width(self) -> float: ... + + @property + def height(self) -> float: ... + + +class _PdfContent(Protocol): + def get_data(self) -> bytes: ... + + +class _PdfPage(Protocol): + @property + def mediabox(self) -> _PdfBox: ... + + def extract_text(self) -> str: ... + + def get_contents(self) -> _PdfContent | None: ... + + +class _PdfReader(Protocol): + @property + def pages(self) -> Sequence[_PdfPage]: ... + + EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") @@ -662,7 +698,7 @@ class _MessageCountParams: if uses_legacy_message_accounting(model): self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n self.tokens_per_name = -1 # if there's a name, the role is omitted - elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms: + elif _is_openai_chat_model(actual_model): self.tokens_per_message = 3 self.tokens_per_name = 1 else: @@ -671,6 +707,63 @@ class _MessageCountParams: self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) self.tool_section = chat_template_tool_section(model) + self._model = model + + @cached_property + def pdf_pricing(self) -> PdfPagePricing: + return pdf_page_pricing(self._model) + + +def _is_openai_chat_model(model: str) -> bool: + return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms + + +def _openai_deployment_name(model: str) -> str: + name: Final = model.rpartition("/")[2] + return name.split(":")[1] if name.startswith("ft:") else name + + +def _openai_model_family(model: str) -> str | None: + words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES) + prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1)) + listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None) + return None if listed is None or _is_listed_non_chat_model(listed) else listed + + +def _untyped_cost_map() -> object: + return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped + + +def _cost_map_entry(model: str) -> Mapping[str, object]: + cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {} + return _pdf_dictionary(cost_map.get(model)) or {} + + +def _is_azure_openai_chat_model(model: str) -> bool: + name: Final = _openai_deployment_name(model) + entry: Final = _cost_map_entry(f"azure/{name}") + return ( + entry.get("litellm_provider") == "azure" + and entry.get("mode") in OPENAI_CHAT_MODES + and (entry.get("supports_pdf_input") is True or name.startswith("gpt-")) + ) + + +def _is_openai_or_azure_model(model: str) -> bool: + if _is_listed_non_chat_model(_openai_deployment_name(model)): + return False + return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) + + +def pdf_page_pricing(model: str | None) -> PdfPagePricing: + return "openai" if _is_openai_or_azure_model(model or "") else "claude" + + +def reads_pdf_pages(model: str) -> bool: + segments: Final = model.split("/") + names: Final = ("/".join(segments[start:]) for start in range(len(segments))) + stated: Final = (_cost_map_entry(name).get("supports_pdf_input") for name in names) + return next((support for support in stated if support is not None), False) is True def token_counter( @@ -842,6 +935,7 @@ def _count_messages( value, use_default_image_token_count, default_token_count, + params.pdf_pricing, ) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -986,7 +1080,7 @@ def _get_exact_count_function( return count_tokens if tokenizer_json["type"] == "openai_tokenizer": - encoding: Final = openai_tokenizer_encoding(model) + encoding: Final = openai_tokenizer_encoding(model or "") def encode_length(text: str) -> int: return _encoding_count(encoding, text) @@ -1006,6 +1100,11 @@ def openai_tokenizer_encoding(model: str) -> "Encoding": return get_encoding(openai_tokenizer_encoding_name(model)) +def _is_listed_non_chat_model(model: str) -> bool: + modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}")) + return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes) + + def openai_tokenizer_encoding_name(model: str) -> str: """The tiktoken encoding name for `model`, without loading the encoding. @@ -1021,7 +1120,9 @@ def openai_tokenizer_encoding_name(model: str) -> str: try: return tiktoken.encoding_name_for_model(model_to_use) except KeyError: - return _suffixed_variant_encoding_name(model_to_use) or "o200k_base" + return _suffixed_variant_encoding_name(model_to_use) or ( + "cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base" + ) def unresolved_tokenizer_model(model: str) -> str: @@ -1079,7 +1180,16 @@ def single_encoding_openai_model(served: Sequence[str | None]) -> str | None: def _openai_model_name(model: str) -> str | None: - return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model) + return ( + _listed_openai_model_name(model) + or _tiktoken_o200k_model_name(model) + or _versioned_openai_model_name(model) + or _azure_openai_chat_model_name(model) + ) + + +def _azure_openai_chat_model_name(model: str) -> str | None: + return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else None _VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)") @@ -1249,7 +1359,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int: return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN) -def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: +def _pdf_dictionary(node: object) -> Mapping[str, object] | None: + return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key + + +def _pdf_value(container: Mapping[str, object], key: str) -> object: + return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in [] + + +_PdfXObjects: TypeAlias = Mapping[str, object] | None +_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfContentSummary: + drawn_names: tuple[str, ...] + draws_inline_image: bool + + +_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects] + + +@dataclass(frozen=True, slots=True) +class _PdfWalk: + frontier: tuple[_PdfDrawing, ...] + seen: frozenset[tuple[int, int]] + summaries: Mapping[int, _PdfContentSummary | None] + draws: int + + +def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects: + resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources")) + if resources is None: + return inherited + return _pdf_dictionary(_pdf_value(resources, "/XObject")) + + +_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r " +_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%" +_PDF_XOBJECT_DRAW: Final = re.compile( + rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])" +) +_PDF_INLINE_IMAGE: Final = re.compile(rb"(? str | None: + try: + return raw.decode(charset) + except UnicodeDecodeError: + return None + + +def _pdf_name(escaped: bytes) -> str: + raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped) + decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS) + return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1")) + + +def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary: + data: Final = content.get_data() + draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1) + return _PdfContentSummary( + drawn_names=tuple(_pdf_name(draw[1]) for draw in draws), + draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None, + ) + + +def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None: + from pypdf.generic import StreamObject + + if not isinstance(form, StreamObject): + return None + return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped + + +def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]: + summary, xobjects = drawing + if xobjects is None: + return () + resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names) + return tuple((xobject, xobjects) for xobject in resolved if xobject is not None) + + +def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]: + xobject, inherited = use + return id(xobject), id(_pdf_xobjects(xobject, inherited)) + + +def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None: + draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier) + if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE: + return None + drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier))) + if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn): + return None + forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"} + unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen) + unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries} + summaries: Final = MappingProxyType( + {**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}} + ) + drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen) + return _PdfWalk( + frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None), + seen=walk.seen | frozenset(forms), + summaries=summaries, + draws=draws, + ) + + +def _pdf_page_draws_raster_image(page: _PdfPage) -> bool: + contents: Final = page.get_contents() + page_dictionary: Final = _pdf_dictionary(page) + if contents is None or page_dictionary is None: + return False + walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned + frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),), + seen=frozenset(), + summaries=MappingProxyType({}), + draws=0, + ) + for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH): + if not walk.frontier: + return False + if any(summary.draws_inline_image for summary, _ in walk.frontier): + return True + if (next_walk := _pdf_next_walk(walk)) is None: + return True + walk = next_walk + return bool(walk.frontier) + + +def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + if _pdf_page_draws_raster_image(page): + return _claude_pdf_page_tokens(page, count_function) + text: Final = page.extract_text() or "" + return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0 + + +def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens( + float(page.mediabox.width), float(page.mediabox.height) + ) + + +def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int: + match pdf_pricing: + case "openai": + return _openai_pdf_page_tokens(page, count_function) + case "claude": + return _claude_pdf_page_tokens(page, count_function) + + +def _count_inline_pdf_tokens( + data_url: str, + count_function: TokenCounterFunction, + pdf_pricing: PdfPagePricing = "claude", +) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None try: @@ -1258,12 +1526,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image") return None try: - reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) - return sum( - count_function(page.extract_text() or "") - + _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) - for page in reader.pages + reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed + _PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) ) + return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None @@ -1273,8 +1539,9 @@ def _count_opaque_document_tokens( data_url: str, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: - pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function) + pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing) if pdf_tokens is not None: return pdf_tokens return calculate_img_tokens( @@ -1289,6 +1556,7 @@ def _count_document_tokens( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: source: Final = document["source"] metadata_tokens: Final = sum( @@ -1301,10 +1569,10 @@ def _count_document_tokens( if isinstance(content, str): return metadata_tokens + count_function(content) return metadata_tokens + _count_content_list( - count_function, content, use_default_image_token_count, default_token_count + count_function, content, use_default_image_token_count, default_token_count, pdf_pricing ) return metadata_tokens + _count_opaque_document_tokens( - _anthropic_image_source_data(source), count_function, use_default_image_token_count + _anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing ) @@ -1312,6 +1580,7 @@ def _count_file_tokens( file_value: object, count_function: TokenCounterFunction, use_default_image_token_count: bool, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one.""" if not isinstance(file_value, Mapping): @@ -1321,7 +1590,9 @@ def _count_file_tokens( name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0 if not isinstance(file_data, str) or not file_data: return name_tokens - return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count) + return name_tokens + _count_opaque_document_tokens( + file_data, count_function, use_default_image_token_count, pdf_pricing + ) def _count_anthropic_content( @@ -1329,6 +1600,7 @@ def _count_anthropic_content( count_function: TokenCounterFunction, use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """ Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). @@ -1363,6 +1635,7 @@ def _count_anthropic_content( field_value, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif isinstance(field_value, dict): tokens += count_function(str(field_value)) @@ -1389,6 +1662,7 @@ def _count_content_list( ], use_default_image_token_count: bool, default_token_count: int | None, + pdf_pricing: PdfPagePricing = "claude", ) -> int: """Recursively count tokens from a list of content blocks.""" try: @@ -1413,12 +1687,14 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] == "file": num_tokens += _count_file_tokens( c.get("file"), count_function, use_default_image_token_count, + pdf_pricing, ) elif c["type"] in ("tool_use", "tool_result", "search_result"): num_tokens += _count_anthropic_content( @@ -1426,6 +1702,7 @@ def _count_content_list( count_function, use_default_image_token_count, default_token_count, + pdf_pricing, ) elif c["type"] in ("thinking", "redacted_thinking"): # Claude extended thinking content block diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index ea0871575e6..404aff84994 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -76,7 +76,11 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit -from litellm.proxy.spend_tracking.input_tokens import tokenizer_models +from litellm.proxy.spend_tracking.input_tokens import ( + inline_pdf_file_count, + models_counted_for, + openai_chat_pdf_file_tokens, +) from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -160,10 +164,10 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None: return resolve_model_group_alias(llm_router.registry().aliases(), model) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # registry().aliases() is the untyped model_group_alias dict -def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]: +def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]: from litellm.proxy.proxy_server import llm_router - return tokenizer_models(model, llm_router) + return models_counted_for(model, llm_router, team_id) def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: @@ -514,6 +518,7 @@ _TPM_FLOOR_FRACTION: Final = 4 # check on data["input"] must exclude these call types, or a Responses call # gets misclassified as an embedding and skips output-token reservation/caps. RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses") +CHAT_COMPLETION_API_CALL_TYPES: Final = (CallTypes.completion.value, CallTypes.acompletion.value) EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding") TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion") RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value) @@ -885,23 +890,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): time_provider: Callable[[], datetime] | None = None, tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db, model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, + counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router, media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ptu_team_ceiling_resolver: Callable[ [str, str], PTUTeamCeiling | None ] = _resolve_ptu_team_ceiling_via_proxy_router, fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings, force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings, - tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router, ): self.internal_usage_cache = internal_usage_cache self._time_provider = time_provider or datetime.now self._tag_rate_limit_resolver = tag_rate_limit_resolver self._model_group_resolver = model_group_resolver + self._counted_models_resolver = counted_models_resolver self._media_model_resolver = media_model_resolver self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver self._fail_closed_resolver = fail_closed_resolver self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver - self._tokenizer_models_resolver = tokenizer_models_resolver if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script( BATCH_RATE_LIMITER_SCRIPT @@ -3948,7 +3953,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): state: Final = data.get("state") return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}" - def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: + def _estimate_precise_input_tokens( + self, data: object, model: str | None, call_type: str | None = None, team_id: str | None = None + ) -> int: """ Model-aware input token estimate for the project ITPM reservation, using ``litellm.token_counter`` -- the same approach the @@ -4014,6 +4021,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): audio_token_estimate: Final = self._estimate_audio_content_tokens(messages) countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages + chat_pdf_files: Final = ( + inline_pdf_file_count(countable_messages) if call_type in CHAT_COMPLETION_API_CALL_TYPES else 0 + ) try: estimate: Final = max( @@ -4029,7 +4039,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): use_default_image_token_count=True, ) ) - for tokenizer_model in self._tokenizer_models_resolver(model or "") + + openai_chat_pdf_file_tokens(tokenizer_model, chat_pdf_files) + for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",)) ), ) return estimate + audio_token_estimate @@ -4086,7 +4097,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): include_media=False, ) raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=requested_model, call_type=call_type + data=data, model=requested_model, call_type=call_type, team_id=user_api_key_dict.team_id ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 57dcb74b36b..fc6fb266d04 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -17,6 +17,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate from litellm.proxy._types import ( @@ -40,12 +41,19 @@ from litellm.proxy.common_utils.user_api_key_cache import ( tag_cache_key, team_membership_reservation_cache_key, ) -from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model, tokenizer_models +from litellm.proxy.spend_tracking.input_tokens import ( + count_input_tokens, + count_input_tokens_for_model, + inline_pdf_file_count, + models_counted_for, + openai_chat_pdf_file_tokens, +) from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget from litellm.types.router import DeploymentTypedDict +from litellm.types.utils import CallTypes @dataclass @@ -271,6 +279,7 @@ async def reserve_budget_for_request( request_body=request_body, route=route, llm_router=llm_router, + team_id=valid_token.team_id, raw_body=raw_body, ) @@ -1592,6 +1601,7 @@ async def count_request_input_tokens( request_body: dict, route: str, llm_router: Router | None, + team_id: str | None, raw_body: bytes | None = None, ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. @@ -1600,21 +1610,35 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - counted_as: Final = MappingProxyType({model: tokenizer_models(model, llm_router) for model in models}) + counted_models: Final = MappingProxyType( + {model: models_counted_for(model, llm_router, team_id) for model in models} + ) counts: Final = await count_input_tokens( request_body=request_body, raw_body=raw_body, - models=tuple(dict.fromkeys(chain.from_iterable(counted_as.values()))), + models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))), + ) + chat_pdf_files: Final = ( + inline_pdf_file_count(request_body.get("messages")) + if CallTypes.acompletion in (get_call_types_for_route(route) or ()) + else 0 ) return MappingProxyType( { - model: count - for model, names in counted_as.items() - if (count := max((counts[name] for name in names if name in counts), default=None)) is not None + model: largest + for model, names in counted_models.items() + if (largest := _largest_count(counts, names, chat_pdf_files)) is not None } ) +def _largest_count(counts: Mapping[str, int], names: Sequence[str], chat_pdf_files: int) -> int | None: + return max( + (counts[name] + openai_chat_pdf_file_tokens(name, chat_pdf_files) for name in names if name in counts), + default=None, + ) + + def _estimate_input_tokens( request_body: dict, route: str, diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 2ae85481f11..59c7fe2a41b 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -13,18 +13,25 @@ from __future__ import annotations import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.token_counter import served_openai_models, unresolved_tokenizer_model +from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX +from litellm.litellm_core_utils.token_counter import ( + deployment_openai_model, + pdf_page_pricing, + reads_pdf_pages, + unresolved_tokenizer_model, +) from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, ) -from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.token_counter import ( @@ -40,6 +47,8 @@ if TYPE_CHECKING: TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000 +_OPENAI_PREFIX: Final = "openai/" + _INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice") @@ -52,25 +61,6 @@ def _approximate_input_size(request_body: Mapping[str, object]) -> int: return sum(len(str(request_body.get(field, ""))) for field in _INPUT_SIZE_FIELDS) -def tokenizer_models(model: str, llm_router: Router | None) -> tuple[str, ...]: - """The names to count a request for `model` with, keeping the largest count. - - An alias resolves to its target group. A group served only by OpenAI models counts as those models, so it gets - their tokenizer. Any other group counts as its OpenAI models, if any, and as the requested name with the encoding - tiktoken gives that name as written, so its other deployments keep that count without loading their vendor's - tokenizer. A name the router does not index counts as written.""" - if llm_router is None: - return (model,) - group: Final = resolve_model_group_alias(llm_router.registry().aliases(), model) or model # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # registry().aliases() is the untyped model_group_alias dict - served: Final = served_openai_models(llm_router.registry().deployments(group, source="named") or ()) - if not served: - return (model,) - openai_served: Final = tuple(served_model for served_model in served if served_model is not None) - if len(openai_served) == len(served): - return openai_served - return (unresolved_tokenizer_model(model), *openai_served) - - async def count_input_tokens( request_body: dict, raw_body: bytes | None, @@ -154,6 +144,83 @@ def _count_input_tokens_for_models( ) +def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: + """The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment + routing would serve it from: the team's or else the shared ones named `model`, its alias target, or its routing + group's members, else those of the one wildcard route that serves it. + + An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed + `openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A + deployment that reads PDF pages and prices them unlike `model` counts as its model minus the provider prefix, the + name `/v1/messages/count_tokens` counts with. Any other deployment counts as the requested name with the encoding + tiktoken gives that name as written, without loading its vendor's tokenizer. A name no deployment serves counts as + written.""" + deployments: Final = _routed_deployments(model, llm_router, team_id) if llm_router else () + return tuple(dict.fromkeys(_counted_name(model, deployment) for deployment in deployments)) or (model,) + + +def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]: + registry: Final = llm_router.registry() + resolved: Final = registry.resolve(model, kind="alias") or model + routing_group: Final = llm_router.get_routing_group(resolved) + members: Final = routing_group.models if routing_group is not None else (resolved,) + named: Final = tuple( + chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members) + ) + return named or registry.deployments(resolved, source="wildcard", team_id=team_id) + + +def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int: + """What OpenAI chat completions bills on top of the counted pages of `pdf_files` inline PDF files, when + `counted_model`, a name from `models_counted_for`, is served by OpenAI itself.""" + return OPENAI_CHAT_PDF_FILE_TOKENS * pdf_files if counted_model.startswith(_OPENAI_PREFIX) else 0 + + +def inline_pdf_file_count(messages: object) -> int: + return sum(1 for part in _content_parts(messages) if _is_inline_pdf_file(part)) + + +def _content_parts(messages: object) -> Iterator[Mapping[str, object]]: + contents: Final = (_as_list(str_keyed(message).get("content")) for message in _as_list(messages)) + return (str_keyed(part) for part in chain.from_iterable(contents)) + + +def _as_list(value: object) -> Sequence[object]: + return value if isinstance(value, list) else () + + +def _is_inline_pdf_file(part: Mapping[str, object]) -> bool: + file_data: Final = str_keyed(part.get("file")).get("file_data") + return part.get("type") == "file" and isinstance(file_data, str) and file_data.startswith(PDF_DATA_URL_PREFIX) + + +def _counted_name(model: str, deployment: object) -> str: + openai_model: Final = deployment_openai_model(deployment) + if openai_model is not None and _served_by_openai(deployment): + return _OPENAI_PREFIX + openai_model.removeprefix(_OPENAI_PREFIX) + if openai_model is not None: + return openai_model + served: Final = next(iter(deployment_pricing_models(deployment)), model) + if pdf_page_pricing(served) != pdf_page_pricing(model) and reads_pdf_pages(served): + return served.partition("/")[2] or served + return unresolved_tokenizer_model(model) + + +def _served_by_openai(deployment: object) -> bool: + params: Final = str_keyed(str_keyed(deployment).get("litellm_params")) + model: Final = params.get("model") + provider: Final = params.get("custom_llm_provider") + if not isinstance(model, str): + return False + try: + _, served_provider, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=provider if isinstance(provider, str) else None + ) + except litellm.exceptions.BadRequestError: + return False + return served_provider == "openai" + + def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: try: if "messages" in request_body: diff --git a/pyproject.toml b/pyproject.toml index de1abeddd7e..6a21fd9b26b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -311,6 +311,7 @@ benchmarks = [ "pytest-codspeed==4.3.0", "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", + "pypdf>=6.21.2,<7.0", ] [build-system] diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..b2a86459438 --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,126 @@ +"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed. + +Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size.""" + +import base64 +from itertools import accumulate, chain +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import token_counter + +TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." + + +def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str: + escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(), + *xobjects, + ) + header: Final = b"%PDF-1.4\n" + objects: Final = tuple( + f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1) + ) + offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1]))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n" + ).encode() + return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + + +def _form(resources_number: int, drawing: bytes) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _count_pdf(pdf_base64: str) -> int: + document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}} + return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}]) + + +def _page_of_drawing_operators() -> str: + return _pdf_base64("", "<< >>", b" q Q" * 50_000) + + +def _page_of_forms_sharing_resources() -> str: + form_count: Final = 300 + shared_resources: Final = ( + "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" + ).encode() + forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count)) + return _pdf_base64( + TEXT_LINE, + "<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>", + b" q 10 0 0 10 0 0 cm /X0 Do Q", + (forms[0], shared_resources, *forms), + ) + + +def _page_of_parents_sharing_a_child_form() -> str: + parent_count: Final = 100 + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child)) + + +PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators() +PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources() +PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form() +BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b"")) +TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b"")) +LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000 +NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8 +UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE) + + +@pytest.mark.benchmark +def test_pdf_page_of_drawing_operators_and_no_image() -> None: + assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None: + assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None: + assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_token_counter_long_unknown_model_name() -> None: + assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index f03c7421899..c2f8e12a726 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -30,7 +30,11 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import ( + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _encoding_count, @@ -2219,11 +2223,11 @@ def test_anthropic_image_block_without_source_raises(): ) -def _count_user_content(content: list[dict]) -> int: +def _count_user_content(content: list[dict], model: str = "anthropic/claude-fable-5") -> int: from litellm.litellm_core_utils.token_counter import token_counter return token_counter( - model="anthropic/claude-fable-5", + model=model, messages=[{"role": "user", "content": content}], use_default_image_token_count=True, ) @@ -2319,14 +2323,27 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries(): ) -def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str: +def _pdf_base64( + pages: tuple[str, ...], + width: int = 612, + height: int = 792, + xobjects: tuple[bytes, ...] = (), + drawing: bytes = b"", + page_resources: str | None = None, +) -> str: + """`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every + page's content stream ends with `drawing`. `page_resources` replaces every page's /Resources dictionary.""" + first_xobject: Final = 4 + 2 * len(pages) + page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else "" + resources: Final = page_resources if page_resources is not None else f"<< /Font << /F1 3 0 R >>{page_xobjects} >>" + def page_objects(index: int, text: str) -> tuple[bytes, bytes]: escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") - stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream" page: Final = ( f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] " - f"/Resources << /Font << /F1 3 0 R >> >> /Contents {4 + 2 * index} 0 R >>" + f"/Resources {resources} /Contents {4 + 2 * index} 0 R >>" ).encode() return content, page @@ -2336,6 +2353,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> f"<< /Type /Pages /Count {len(pages)} /Kids [ {kids} ] >>".encode(), b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", *chain.from_iterable(page_objects(index, text) for index, text in enumerate(pages)), + *xobjects, ) header: Final = b"%PDF-1.4\n" objects: Final = tuple( @@ -2414,6 +2432,401 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest. assert priced_as_image < priced_by_page +_REVENUE_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." +_HEADCOUNT_LINE: Final = "Headcount is flat and the office lease was renewed for three years." +_ONE_PIXEL_IMAGE: Final = ( + b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 " + b"/Length 1 >>\nstream\n\x00\nendstream" +) + + +_DRAW_X0: Final = b" q 10 0 0 10 0 0 cm /X0 Do Q" +_DRAW_X1: Final = b"/X1 Do" + + +def _form_xobject(resources_number: int, drawing: bytes = _DRAW_X1) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _xobject_resources(xobject_number: int) -> bytes: + return f"<< /XObject << /X1 {xobject_number} 0 R >> >>".encode() + + +def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]: + return {"type": "document", "source": _pdf_source(pdf_base64)} + + +def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]: + file_data: Final = "data:application/pdf;base64," + pdf_base64 + return {"type": "file", "file": {"filename": "report.pdf", "file_data": file_data}} + + +def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]: + return {"type": "tool_result", "tool_use_id": "t1", "content": [_as_pdf_document(pdf_base64)]} + + +def _pdf_cost(model: str, pdf_base64: str, wrap: Callable[[str], Mapping[str, object]] = _as_pdf_document) -> int: + return _count_user_content([wrap(pdf_base64)], model) - _count_user_content([wrap(_pdf_base64(()))], model) + + +def _text_cost(model: str, *lines: str) -> int: + return sum( + _count_user_content([{"type": "text", "text": line}], model) - _count_user_content([], model) for line in lines + ) + + +def _rendered_page_cost(width: int = 612, height: int = 792) -> int: + return _pdf_cost("my-private-model", _pdf_base64(("",), width, height)) + + +@pytest.mark.parametrize( + "wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"] +) +@pytest.mark.parametrize( + "model", + ["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"], +) +def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing( + model: str, wrap: Callable[[str], Mapping[str, object]] +): + """OpenAI models bill a PDF page with text as its text plus 13 tokens and a blank page as nothing, with no page + image. Measured 2026-10-20 on Azure chat completions gpt-5.6-sol (1, 2, and 10 pages of 15 text tokens billed + +28, +56, +280, and 1, 2, or 10 blank pages billed the same as none) and on OpenAI /v1/responses/input_tokens + for gpt-4o, gpt-4.1, gpt-5, gpt-5.5, o3, and o4-mini. + + Before the fix every model got Claude's pricing, so count_tokens answered about 1586 for a one-page text PDF + that Azure gpt-5.6-sol billed at 61. + """ + one_page: Final = _pdf_cost(model, _pdf_base64((_REVENUE_LINE,)), wrap) + + assert one_page == _text_cost(model, _REVENUE_LINE) + 13 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE, _HEADCOUNT_LINE)), wrap) == ( + _text_cost(model, _REVENUE_LINE, _HEADCOUNT_LINE) + 2 * 13 + ) + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,) * 10), wrap) == 10 * one_page + assert _pdf_cost(model, _pdf_base64(("",) * 5), wrap) == 0 + assert _pdf_cost(model, _pdf_base64(("", _REVENUE_LINE, "")), wrap) == one_page + + +@pytest.mark.parametrize( + "xobjects", + [(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)], + ids=["image_on_the_page", "image_inside_a_form"], +) +def test_openai_model_prices_a_page_that_draws_a_raster_image_as_its_text_plus_the_rendered_page( + xobjects: tuple[bytes, ...], +): + """A page embedding a raster image keeps the rendered-page estimate every model got before, so a reservation + never undercounts it: on 2026-10-20 /v1/responses/input_tokens billed a page with one drawn image at 1620 on + gpt-5.6-sol and gpt-6-sol, above OpenAI's own high-detail page image (765 for Letter), while a vector-drawn + chart page billed no image. + """ + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == _rendered_page_cost() + assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == ( + _rendered_page_cost(300, 300) + ) + + +def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws(): + """Pages sharing one /Resources all list its logo, but only the page that paints it bills a page image: measured + 2026-10-20 on Azure gpt-5.6-sol, a 3-page PDF drawing a logo on page 1 billed 403, not three page images.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,))) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + form_listing_an_unused_image: Final = (_form_xobject(7, b""), _xobject_resources(8), _ONE_PIXEL_IMAGE) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=form_listing_an_unused_image, drawing=_DRAW_X0)) == 0 + + +def test_openai_model_prices_no_image_for_forms_that_draw_none(): + """A vector-drawn form XObject bills no page image, and a form that lists itself cannot loop the walk.""" + model: Final = "gpt-4o" + self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6)) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + 13 + ) + + +_INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q" + + +def test_openai_model_prices_a_page_that_draws_an_inline_image_like_an_image_xobject(): + """An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured + 2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o.""" + model: Final = "gpt-4o" + form_drawing_inline_image: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(_INLINE_IMAGE)} >>\nstream\n".encode() + + _INLINE_IMAGE + + b"\nendstream" + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == ( + _rendered_page_cost() + ) + assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == ( + _rendered_page_cost() + ) + + +def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk(): + """A chain of 2,000 nested form XObjects used to exhaust the stack, which dropped the whole document's text and + priced it as one image. The walk now stops at a fixed depth and prices the page as if it drew an image.""" + model: Final = "gpt-4o" + depth: Final = 2000 + first: Final = 6 + chain_objects: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * level + 1), _xobject_resources(first + 2 * level + 2)) for level in range(depth) + ) + ) + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == ( + _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) + + +@pytest.mark.parametrize("drawing", [b" /A Do /B Do", b" /B Do /A Do"]) +def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_one_parent(drawing: bytes): + """A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two + parents share reaches the image only through the parent whose resources list it.""" + model: Final = "gpt-4o" + child_without_resources: Final = ( + b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream" + ) + parents_and_child: Final = ( + _form_xobject(7, b"/C Do"), + b"<< /XObject << /C 10 0 R >> >>", + _form_xobject(9, b"/C Do"), + b"<< /XObject << /C 10 0 R /Y 11 0 R >> >>", + child_without_resources, + _ONE_PIXEL_IMAGE, + ) + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=parents_and_child, + drawing=drawing, + page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>", + ), + ) == _rendered_page_cost() + + +@pytest.mark.parametrize( + ("drawing", "draws_image"), + [(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)], + ids=["comment-before-do", "do-inside-a-comment"], +) +def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool): + """A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after + it nor draws anything itself.""" + model: Final = "gpt-4o" + + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == ( + _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13) + ) + + +@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"]) +def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str): + """A name written with #xx escapes reads the same in the content stream and in /Resources, decoded the way pypdf + decodes it, so an image drawn under such a name still prices the page as rendered.""" + model: Final = "gpt-4o" + + assert _pdf_cost( + model, + _pdf_base64( + ("",), + xobjects=(_ONE_PIXEL_IMAGE,), + drawing=f" {escaped_name} Do".encode(), + page_resources=f"<< /XObject << {escaped_name} 6 0 R >> >>", + ), + ) == _rendered_page_cost() + + +@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)]) +def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered( + parent_count: int, draws_too_many: bool +): + """A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects + between them, so a page of forms nested to blow up the walk cannot stall the request.""" + model: Final = "gpt-4o" + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2)) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form_xobject(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + + assert _pdf_cost( + model, + _pdf_base64( + (_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources + ), + ) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13) + + +_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.6-sol", + "gpt-5.4-mini", + "gpt-6-sol", + "azure/gpt-41", + "azure/gpt-5.6-sol", + "eu/gpt-5.6-sol", + "gpt-5.6-sol-2026-07-09", + ], +) +def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str): + """OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on + /v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a + non-Latin sample like this one that is 249 tokens in o200k_base and 297 in cl100k_base.""" + o200k: Final = tiktoken.get_encoding("o200k_base") + + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE)) + + +@pytest.mark.parametrize(("deployment_name", "model"), [("azure/gpt-4", "gpt-4"), ("eu/gpt-35-turbo", "gpt-35-turbo")]) +def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deployment_name: str, model: str): + assert token_counter_new(model=deployment_name, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"] +) +def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str): + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize( + "model", + [ + "openrouter/openai/gpt-4o", + "github_copilot/gpt-4o", + "azure_ai/gpt-4o", + "azure/responses/gpt-5.6-sol", + "azure/o_series/o3-mini", + ], +) +def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str): + """The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes + a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 on + 2026-10-20, closer to OpenAI's pricing than to Claude's 1534.""" + bare_name: Final = model.rpartition("/")[2] + pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0) + + assert _pdf_cost(model, pdf_with_a_drawn_image) == _pdf_cost(bare_name, pdf_with_a_drawn_image) + assert _pdf_cost(model, _pdf_base64(("",))) == 0 + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE) + ) + + +@pytest.mark.parametrize( + "model", + [ + "text-embedding-ada-002-v2", + "tts-1", + "whisper-1", + "azure/whisper-1", + "github_copilot/text-embedding-3-small-inference", + "azure_text/gpt-35-turbo-instruct", + "ft:davinci-002:my-org::abc123", + ], +) +def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str): + assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len( + tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE) + ) + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + + +@pytest.mark.parametrize( + ("fine_tune", "base_model"), + [ + ("ft:gpt-4o-mini-2024-07-18:my-org::abc123", "gpt-4o-mini-2024-07-18"), + ("openai/ft:gpt-4.1-2025-04-14:my-org:support:def456", "gpt-4.1-2025-04-14"), + ], +) +def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_model: str): + pdf: Final = _pdf_base64((_REVENUE_LINE,)) + + assert _pdf_cost(fine_tune, pdf) == _pdf_cost(base_model, pdf) == _text_cost(base_model, _REVENUE_LINE) + 13 + assert token_counter_new(model=fine_tune, text=_NON_LATIN_SAMPLE) == token_counter_new( + model=base_model, text=_NON_LATIN_SAMPLE + ) + + +@pytest.mark.parametrize( + "model", + [ + "", + "anthropic/claude-fable-5", + "DeepSeek-V4-Flash-0731", + "my-private-model", + "azure/mistral-large-latest", + "command-r-plus", + ], +) +def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str): + """Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps + Claude's text plus rendered page image, 1534 tokens for a blank Letter page.""" + assert _pdf_cost(model, _pdf_base64(("",))) == 1534 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 1534 + + +def test_messages_reach_token_count_prices_pdf_pages_like_token_counter(): + from litellm.litellm_core_utils.token_counter import messages_reach_token_count + + messages: Final = [{"role": "user", "content": [_as_pdf_document(_pdf_base64((_REVENUE_LINE, "")))]}] + total: Final = token_counter_new(model="azure/gpt-5.6-sol", messages=messages) + + assert total == token_counter_new(model="azure/gpt-5.6-sol", messages=[{"role": "user", "content": []}]) + ( + _text_cost("azure/gpt-5.6-sol", _REVENUE_LINE) + 13 + ) + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total) is True + assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total + 1) is False + + def _png_data_url(width: int, height: int) -> str: ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big") return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() @@ -2747,6 +3160,14 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t } +def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None: + sample: Final = "A custom tokenizer passed with no model still counts." + + assert litellm.token_counter(model=None, custom_tokenizer=litellm.utils.select_tokenizer("gpt-4o"), text=sample) == ( + len(tiktoken.get_encoding("cl100k_base").encode(sample)) + ) + + def test_process_exits_cleanly_while_a_background_tokenizer_retry_is_downloading(tmp_path: Path) -> None: assert _run_in_memory_hub(HUB_RETRY_DURING_EXIT_SCRIPT, {}, "", tmp_path) == {"retry_reached_hub": True} diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 933d1f48ad4..b66e7f3d091 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -46,7 +46,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, ) -from litellm.proxy.spend_tracking.input_tokens import tokenizer_models +from litellm.proxy.spend_tracking.input_tokens import models_counted_for from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_ENV_VAR from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token from litellm.router_utils.ptu_shares import PTUTeamCeiling @@ -9101,6 +9101,63 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit( assert "expensive" in str(denied.value.detail) +@pytest.mark.asyncio +async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_deployments() -> None: + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(cache), + counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"), + ) + messages: Final = _one_page_pdf_message() + caller: Final = UserAPIKeyAuth( + api_key=hash_token("sk-pdf-itpm"), + project_id="pdf-itpm", + project_metadata={"model_itpm_limit": {"gpt-4o": 1_000_000}, "model_otpm_limit": {"gpt-4o": 1_000_000}}, + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=cache, + data={"model": "gpt-4o", "messages": messages, "max_tokens": 10}, + call_type="acompletion", + ) + + stash: Final = get_request_stash() + assert stash is not None + by_name: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), counted_models_resolver=lambda model, team_id: (model,) + ) + claude_count: Final = by_name._estimate_precise_input_tokens({"messages": messages}, "claude-haiku-5-5") + assert claude_count > by_name._estimate_precise_input_tokens({"messages": messages}, "gpt-4o") + assert stash.itpm_reserved_tokens == claude_count + + +def test_project_itpm_adds_what_openai_chat_bills_on_top_of_a_pdf_files_pages_on_chat_completions_only() -> None: + from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + router: Final = Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test"}} + for name, model in (("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol")) + ] + ) + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(DualCache()), + counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id), + ) + data: Final = {"messages": _one_page_pdf_message()} + + assert handler._estimate_precise_input_tokens(data, "on-openai", "acompletion") == ( + handler._estimate_precise_input_tokens(data, "on-azure", "acompletion") + OPENAI_CHAT_PDF_FILE_TOKENS + ) + assert handler._estimate_precise_input_tokens(data, "on-openai", "anthropic_messages") == ( + handler._estimate_precise_input_tokens(data, "on-azure", "anthropic_messages") + ) + + def _extraction_request(state: str, call_id: str) -> dict[str, object]: return { "model": "invoice-reader", @@ -9184,7 +9241,7 @@ _MULTILINGUAL_REQUEST: Final[dict[str, object]] = { def _project_itpm_estimate(router: Router, model: str) -> int: handler: Final = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(DualCache()), - tokenizer_models_resolver=partial(tokenizer_models, llm_router=router), + counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id), ) return handler._estimate_precise_input_tokens(_MULTILINGUAL_REQUEST, model, "acompletion") @@ -9193,7 +9250,7 @@ def test_project_itpm_counts_a_group_mixing_in_a_non_openai_model_by_its_name_as router: Final = Router( model_list=[ {"model_name": "gpt-6.1-sol", "litellm_params": {"model": model, "api_key": "sk-test"}} - for model in ("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5") + for model in ("openai/gpt-6.1-sol", "cohere/command-r-plus") ] ) messages: Final = _MULTILINGUAL_REQUEST["messages"] diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 035b7d9dcd3..a87ac7fa7c0 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -429,6 +429,24 @@ def test_budget_window_start_reads_json_encoded_windows(window: str, expected: d assert get_budget_window_start(window) == expected +@pytest.mark.asyncio +async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_group() -> None: + from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + request_body: Final = {"model": "gpt-4o", "messages": _one_page_pdf_message()} + router: Final = Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}] + ) + + counts: Final = await count_request_input_tokens( + request_body=request_body, route="/chat/completions", llm_router=router, team_id=None + ) + + assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")} + assert counts["gpt-4o"] > count_input_tokens_for_model(request_body=request_body, model="gpt-4o") + + _MULTILINGUAL_MESSAGES: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}] @@ -446,10 +464,38 @@ async def _input_tokens(router: Router, model: str) -> dict[str, int]: request_body={"model": model, "messages": _MULTILINGUAL_MESSAGES}, route="/v1/chat/completions", llm_router=router, + team_id=None, ) ) +async def _reserved_input_tokens(router: Router, model: str, route: str, messages: list[dict[str, object]]) -> int: + counts: Final = await count_request_input_tokens( + request_body={"model": model, "messages": messages}, route=route, llm_router=router, team_id=None + ) + return counts[model] + + +@pytest.mark.asyncio +async def test_an_openai_chat_request_reserves_what_openai_chat_bills_on_top_of_each_pdf_files_pages() -> None: + from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS + from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message + + router: Final = _router_with_groups(("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol")) + two_pdfs: Final = [*_one_page_pdf_message(), *_one_page_pdf_message()] + + assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", two_pdfs) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", two_pdfs) + + 2 * OPENAI_CHAT_PDF_FILE_TOKENS + ) + assert await _reserved_input_tokens(router, "on-openai", "/v1/messages", two_pdfs) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/messages", two_pdfs) + ) + assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) == ( + await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) + ) + + @pytest.mark.asyncio async def test_alias_group_of_an_openai_model_reserves_on_that_models_token_count(): router: Final = _router_with_groups(("gpt-6.1-sol", "openai/gpt-6.1-sol"), ("prod-gpt-6", "openai/gpt-6.1-sol")) @@ -516,7 +562,7 @@ async def test_release_named_group_or_alias_no_openai_model_serves_reserves_on_a async def test_group_mixing_in_a_non_openai_model_reserves_on_its_name_as_written_and_its_openai_models( group: str, counted_as: str ): - served: Final = ("openai/gpt-6.1-sol", "anthropic/claude-sonnet-5-5") + served: Final = ("openai/gpt-6.1-sol", "cohere/command-r-plus") router: Final = _router_with_groups(*((group, model) for model in served)) reserved: Final = (await _input_tokens(router, group))[group] diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index c7f13651471..40aee7bb00d 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import json from types import MappingProxyType from typing import Final @@ -12,7 +13,9 @@ from litellm.proxy.spend_tracking.input_tokens import ( TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS, count_input_tokens, count_input_tokens_for_model, + models_counted_for, ) +from litellm.router import Router CL100K_MODEL: Final = "gpt-4" @@ -30,6 +33,173 @@ async def test_large_input_is_still_counted() -> None: assert isinstance(counts, MappingProxyType) +def _one_page_pdf_message() -> list[dict[str, object]]: + stream: Final = b"BT /F1 12 Tf 72 720 Td (Revenue grew eleven percent while churn fell.) Tj ET" + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 4 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> " + b"/Contents 5 0 R >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + ) + objects: Final = tuple(f"{n} 0 obj\n".encode() + body + b"\nendobj\n" for n, body in enumerate(bodies, start=1)) + header: Final = b"%PDF-1.4\n" + offsets: Final = tuple(len(header) + sum(len(obj) for obj in objects[:index]) for index in range(len(objects))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{offsets[-1] + len(objects[-1])}\n%%EOF\n" + ).encode() + pdf: Final = base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + return [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"filename": "report.pdf", "file_data": f"data:application/pdf;base64,{pdf}"}}, + {"type": "text", "text": "Summarize the report."}, + ], + } + ] + + +def _router(*deployments: tuple[str, str]) -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test", "api_base": "https://x"}} + for name, model in deployments + ] + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model", "counted_with"), + [ + ("gpt-4o", "anthropic/claude-haiku-5-5", "claude-haiku-5-5"), + ("pdf-reports", "azure/gpt-5.6-sol", "azure/gpt-5.6-sol"), + ("pdf-reports", "azure/responses/gpt-5.6-sol", "azure/responses/gpt-5.6-sol"), + ("pdf-reports", "openrouter/openai/gpt-5.6-sol", "openrouter/openai/gpt-5.6-sol"), + ("pdf-reports", "openai/gpt-5.6-sol", "openai/gpt-5.6-sol"), + ("pdf-reports", "gpt-5.6-sol", "openai/gpt-5.6-sol"), + ], + ids=[ + "gpt-named-group-served-by-claude", + "custom-group-served-by-azure-gpt", + "custom-group-served-by-azure-responses-gpt", + "custom-group-served-by-openrouter-gpt", + "custom-group-served-by-openai-gpt", + "custom-group-served-by-unprefixed-openai-gpt", + ], +) +def test_a_model_group_counts_like_the_deployment_that_serves_it( + group: str, deployment_model: str, counted_with: str +) -> None: + request_body: Final = {"model": group, "messages": _one_page_pdf_message()} + + assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,) + assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model( + request_body=request_body, model=group + ) + + +@pytest.mark.parametrize( + ("group", "deployment_model"), + [ + ("my-claude", "anthropic/claude-haiku-5-5"), + ("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"), + ("my-cohere", "cohere/command-r-plus"), + ("gpt-4o", "cohere/command-r-plus"), + ], + ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"], +) +def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group( + group: str, deployment_model: str +) -> None: + assert models_counted_for(group, _router((group, deployment_model)), None) == (group,) + + +def test_a_custom_named_deployment_counts_with_its_base_model(): + router: Final = Router( + model_list=[ + { + "model_name": "reports", + "litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"}, + "model_info": {"base_model": "azure/gpt-5.6-sol"}, + }, + {"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}}, + ] + ) + + assert models_counted_for("reports", router, None) == ("azure/gpt-5.6-sol",) + assert models_counted_for("drafts", router, None) == ("drafts",) + + +def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is(): + router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol")) + + assert models_counted_for("mixed", router, None) == ("mixed", "azure/gpt-5.6-sol") + assert models_counted_for("not-deployed", router, None) == ("not-deployed",) + assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",) + + +def test_a_team_request_counts_with_the_deployment_its_team_is_served_by(): + router: Final = Router( + model_list=[ + {"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}, + { + "model_name": "reports-team-a", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "reports"}, + }, + ] + ) + + assert models_counted_for("reports", router, "team-a") == ("azure/gpt-5.6-sol",) + assert models_counted_for("reports", router, "team-b") == ("reports",) + + +@pytest.mark.parametrize("requested", ["reports", "latest-reports"], ids=["routing-group", "alias-of-routing-group"]) +def test_a_team_request_to_a_routing_group_counts_with_the_member_deployment_its_team_is_served_by(requested: str): + router: Final = Router( + model_list=[ + { + "model_name": "drafts", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x", "api_base": "https://x"}, + }, + { + "model_name": "drafts-team-a", + "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "drafts"}, + }, + ], + routing_groups=[{"group_name": "reports", "models": ["drafts"], "routing_strategy": "simple-shuffle"}], + model_group_alias={"latest-reports": "reports"}, + ) + + assert models_counted_for(requested, router, "team-a") == models_counted_for( + requested, _router((requested, "anthropic/claude-haiku-5-5")), None + ) + assert models_counted_for(requested, router, "team-b") == ("azure/gpt-5.6-sol",) + + +def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard(): + router: Final = Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "anthropic/*", "api_key": "x"}}, + { + "model_name": "team-a-gpt", + "litellm_params": {"model": "azure/gpt-*", "api_key": "x", "api_base": "https://x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "gpt-*"}, + }, + ] + ) + + assert models_counted_for("gpt-5.6-sol", router, "team-a") == ("azure/gpt-5.6-sol",) + assert models_counted_for("gpt-5.6-sol", router, "team-b") == models_counted_for("gpt-5.6-sol", router, None) + assert "azure/gpt-5.6-sol" not in models_counted_for("gpt-5.6-sol", router, "team-b") + + def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: model: Final = "claude-sonnet-5-5" body_without_tool_choice: Final = { diff --git a/tests/unit/rust_bridge/test_token_counter.py b/tests/unit/rust_bridge/test_token_counter.py index 068d389ebbd..ba9c0b90498 100644 --- a/tests/unit/rust_bridge/test_token_counter.py +++ b/tests/unit/rust_bridge/test_token_counter.py @@ -144,9 +144,10 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None) ("gemini/gemini-2.5-pro", "cl100k_base"), ("mistral/mistral-large-latest", "cl100k_base"), ("my-router-alias", "cl100k_base"), - ("azure/gpt-4o", "o200k_base"), ("command-r-plus", "cl100k_base"), ("gpt-4o", "o200k_base"), + ("azure/gpt-4o", "o200k_base"), + ("eu/gpt-5.6-sol", "o200k_base"), ("gpt-4o-mini", "o200k_base"), ("gpt-4o-2024-08-06", "o200k_base"), ("gpt-4.1", "o200k_base"), diff --git a/uv.lock b/uv.lock index 2ba97d15327..87b8fc95486 100644 --- a/uv.lock +++ b/uv.lock @@ -5109,6 +5109,7 @@ admin-mcp = [ benchmarks = [ { name = "a2a-sdk" }, { name = "mcp" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -5337,6 +5338,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1 benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, + { name = "pypdf", specifier = ">=6.21.2,<7.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ] From 89ca8d83f89b876db0353069034a1377a94cde05 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:28:16 -0700 Subject: [PATCH 12/25] fix(ci): measure PR gates from the target branch's current tip (#49078) * fix(ci): measure PR gates from the target branch's current tip The lint gate base, the ui lint and ui unit test scopes, and the spend logs guard compared the event's pull_request.base.sha with the head to find the merge base. GitHub can leave base.sha weeks behind the base branch, and when that old commit is an ancestor of a head that merged newer main, compare returns the old commit itself, so the strict gates blamed main's own code on the PR. Compare base.ref with the head instead, passed through env, so GitHub resolves the branch tip when the job runs. This is the same base make check uses locally. The ui unit test push guard now keys on BASE_REF, which is empty on push and workflow_dispatch. The regression test runs every workflow step that reads merge_base_commit against a fake gh backed by a temp repo with a stale base.sha, for main and a litellm_/ base with a slash. * fix(ci): check out the base branch's current guard code in the PR guards * fix(ci): read the PR's current base branch so re-runs survive a deleted base Every merge-base step now asks the pulls API for the PR's base branch instead of reading the event's base.ref, which a re-run keeps after a stacked base is merged and deleted. The guard checkouts go back to base.sha, which stays reachable. * test(ci): mark the step script path Final * fix(ci): keep the spend logs guard on main and run both guard scripts from the base branch tip * fix(ci): measure from the event's base branch unless it was deleted, and test what each step hands on A re-run after a retarget off main kept following the PR's current base, so lint, ui-unit and the cost map guard could pass without measuring against main. They now read the current base only when the event's base branch no longer exists. The tests run each merge base step whole and check the value its consumer gets * fix(ci): follow a renamed base branch to its new name before the compare --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/cost-map-guard.yml | 10 +- .github/workflows/spend-logs-guard.yml | 6 +- .github/workflows/test-linting.yml | 19 +- .github/workflows/test-unit.yml | 8 +- tests/unit/test_lint_workflow_diff_gates.py | 393 +++++++++++++++++++- tests/unit/test_select_ui_test_scope.py | 65 +++- 6 files changed, 461 insertions(+), 40 deletions(-) diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml index fb86237e504..f26c649aad2 100644 --- a/.github/workflows/cost-map-guard.yml +++ b/.github/workflows/cost-map-guard.yml @@ -8,6 +8,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -21,16 +22,19 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - - name: Fetch the pull request head, its merge base, and the base branch tip + - name: Check out the base branch tip and fetch the pull request head and its merge base id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')" + base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)" + target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')" merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target" + git checkout -q --detach "$target" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" echo "target=$target" >> "$GITHUB_OUTPUT" - name: Set up uv diff --git a/.github/workflows/spend-logs-guard.yml b/.github/workflows/spend-logs-guard.yml index cc3bda9b880..bdba95f6a5d 100644 --- a/.github/workflows/spend-logs-guard.yml +++ b/.github/workflows/spend-logs-guard.yml @@ -19,17 +19,17 @@ jobs: steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: - ref: ${{ github.event.pull_request.base.sha }} + ref: ${{ github.event.pull_request.base.ref }} persist-credentials: false - name: Fetch the pull request head, its merge base and its reviews id: revisions env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} run: | - merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" + merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json" echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 88c66f0011d..c3388430589 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -48,11 +48,13 @@ jobs: if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; } - MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$MERGE_BASE" retry git fetch --no-tags --depth=1 origin "$MERGE_BASE" echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" @@ -264,6 +266,7 @@ jobs: name: ui permissions: contents: read + pull-requests: read runs-on: ubuntu-latest timeout-minutes: 8 defaults: @@ -281,16 +284,12 @@ jobs: id: changed env: GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - # base.sha is the base branch tip from when the PR was opened, while - # actions/checkout leaves HEAD on a merge of the PR into the *current* - # base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit - # landed since, so a PR that touches no UI file still gets linted - # against hundreds of other people's files. Diff the PR head against its - # own merge base instead, which is exactly what this PR changed. - merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$merge_base" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" : > "$RUNNER_TEMP/prettier_files.txt" diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a5013155eae..9ebb9bab018 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1150,18 +1150,20 @@ jobs: env: CI: "true" GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | full_suite() { npm run test -- --run --pool forks --maxWorkers=14; } - if [ -z "$BASE_SHA" ]; then + if [ -z "$PR_NUMBER" ]; then echo "Push to $GITHUB_REF_NAME: running the full suite" full_suite exit 0 fi - merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') test -n "$merge_base" git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" changed_files=() diff --git a/tests/unit/test_lint_workflow_diff_gates.py b/tests/unit/test_lint_workflow_diff_gates.py index 5d3c0d80aa6..816ef6ce707 100644 --- a/tests/unit/test_lint_workflow_diff_gates.py +++ b/tests/unit/test_lint_workflow_diff_gates.py @@ -1,19 +1,198 @@ from __future__ import annotations import fnmatch +import itertools +import os import re import shlex import subprocess +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass from pathlib import Path -from typing import Final +from types import MappingProxyType +from typing import Final, TypeAlias import pytest import yaml +from pydantic import BaseModel, ConfigDict, Field WORKFLOW: Final = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "test-linting.yml" +WORKFLOWS_DIR: Final = WORKFLOW.parent +REPO_ROOT: Final = WORKFLOW.parents[2] DIFF_GATE: Final = re.compile(r'git diff --name-only --diff-filter=\w+ "\$GATE_BASE_SHA" HEAD -- (.+?) \|') GATES: Final = tuple(tuple(shlex.split(gate.group(1))) for gate in DIFF_GATE.finditer(WORKFLOW.read_text())) PYTHON_GATES: Final = tuple(gate for gate in GATES if gate[0].startswith(":(glob)")) +REPOSITORY: Final = "BerriAI/litellm" +UI_DASHBOARD: Final = "ui/litellm-dashboard" +STACKED_BASE_REF: Final = "litellm_/stacked" +RELEASE_BASE_REF: Final = "stable/next" +BASE_REFS: Final = ("main", STACKED_BASE_REF, RELEASE_BASE_REF) +RENAMED_BASE_REF: Final = "litellm_renamed_before_the_rerun" +GITHUB_EXPRESSION: Final = re.compile(r"\$\{\{\s*(.+?)\s*\}\}") +FAKE_GH: Final = r"""#!/usr/bin/env bash +set -euo pipefail +endpoint=${2%%\?*} +endpoint=${endpoint#"repos/$GITHUB_REPOSITORY/"} +commit() { git -C "$UPSTREAM" rev-parse --verify --quiet "$1^{commit}"; } +renamed() { git -C "$UPSTREAM" config --get "renamed.$1.to" || printf '%s\n' "$1"; } +case "$1 $endpoint ${3-} ${4-}" in + "api compare/"*"..."*" --jq .merge_base_commit.sha") + basehead=${endpoint#compare/} + git -C "$UPSTREAM" merge-base "$(commit "${basehead%%...*}")" "$(commit "${basehead#*...}")" ;; + "api pulls/1 --jq .base.ref") + printf '%s\n' "$LIVE_BASE_REF" ;; + "api branches/"*) + branch=$(renamed "${endpoint#branches/}") + sha=$(commit "refs/heads/$branch") || { [ "${3-}" = --silent ] || printf '{"message":"Not Found"}\n'; exit 1; } + case "${3-} ${4-}" in + "--silent ") ;; + "--jq .name") printf '%s\n' "$branch" ;; + "--jq .commit.sha") printf '%s\n' "$sha" ;; + *) echo "fake gh cannot answer: $*" >&2; exit 64 ;; + esac ;; + "api pulls/1/reviews --paginate --slurp") + printf '[[]]\n' ;; + *) + echo "fake gh cannot answer: $*" >&2 + exit 64 ;; +esac +""" +FAKE_ARGV_RECORDER: Final = r"""#!/usr/bin/env bash +printf '%s\n' "$@" > "$RUNNER_TEMP/$(basename "$0").argv" +""" + + +class WorkflowStep(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str = "" + name: str = "" + uses: str = "" + with_: Mapping[str, str] = Field(default_factory=dict, alias="with") + env: Mapping[str, str] = Field(default_factory=dict) + run: str = "" + + +class RunDefaults(BaseModel): + model_config = ConfigDict(frozen=True) + + working_directory: str = Field(default=".", alias="working-directory") + + +class JobDefaults(BaseModel): + model_config = ConfigDict(frozen=True) + + run: RunDefaults = Field(default_factory=RunDefaults) + + +class WorkflowJob(BaseModel): + model_config = ConfigDict(frozen=True) + + defaults: JobDefaults = Field(default_factory=JobDefaults) + steps: tuple[WorkflowStep, ...] = () + + +class PullRequestTrigger(BaseModel): + model_config = ConfigDict(frozen=True) + + branches: tuple[str, ...] = ("**",) + + +class WorkflowTriggers(BaseModel): + model_config = ConfigDict(frozen=True) + + pull_request: PullRequestTrigger = Field(default_factory=PullRequestTrigger) + + +class Workflow(BaseModel): + model_config = ConfigDict(frozen=True) + + on: WorkflowTriggers + jobs: Mapping[str, WorkflowJob] + + +@dataclass(frozen=True, slots=True) +class MergeBaseStep: + workflow: str + job_id: str + job: WorkflowJob + step: WorkflowStep + base_branches: tuple[str, ...] + + @property + def job_key(self) -> str: + return f"{self.workflow}:{self.job_id}" + + +@dataclass(frozen=True, slots=True) +class StaleBaseUpstream: + path: Path + stale_base: str + head: str + tip_by_ref: Mapping[str, str] + merge_base_by_ref: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class Runner: + root: Path + upstream: StaleBaseUpstream + live_base_ref: str + context: Mapping[str, str] + + @property + def checkout(self) -> Path: + return self.root / "checkout" + + @property + def tools(self) -> Path: + return self.root / "bin" + + @property + def temp(self) -> Path: + return self.root / "runner_temp" + + @property + def github_output(self) -> Path: + return self.root / "github_output" + + @property + def github_env(self) -> Path: + return self.root / "github_env" + + +@dataclass(frozen=True, slots=True) +class StepRun: + step: MergeBaseStep + runner: Runner + + +Observation: TypeAlias = str | frozenset[str] | tuple[str, str] + + +@dataclass(frozen=True, slots=True) +class Consumer: + observe: Callable[[StepRun], Observation] + expect: Callable[[StaleBaseUpstream, str], Observation] + + +def _workflow(path: Path) -> Workflow: + return Workflow.model_validate(yaml.load(path.read_text(), Loader=yaml.BaseLoader)) + + +def _merge_base_steps_in(path: Path) -> Iterator[MergeBaseStep]: + workflow: Final = _workflow(path) + for job_id, job in workflow.jobs.items(): + yield from ( + MergeBaseStep(path.name, job_id, job, step, workflow.on.pull_request.branches) + for step in job.steps + if "merge_base_commit" in step.run + ) + + +MERGE_BASE_STEPS: Final = tuple( + itertools.chain.from_iterable(map(_merge_base_steps_in, sorted(WORKFLOWS_DIR.glob("*.y*ml")))) +) def _git(cwd: Path, *args: str) -> str: @@ -104,6 +283,218 @@ def test_e2e_harness_gate_fires_on_any_e2e_or_harness_file_its_dependency_manife assert selected == frozenset({trigger}) +def _event_context(pull_request: Mapping[str, str]) -> Mapping[str, str]: + return MappingProxyType( + { + "github.repository": REPOSITORY, + "github.token": "fake-token", + **{f"github.event.pull_request.{field}": value for field, value in pull_request.items()}, + } + ) + + +def _context_value(context: Mapping[str, str], expression: str) -> str: + if expression not in context: + pytest.fail(f"the fake runner has no value for ${{{{ {expression} }}}}") + return context[expression] + + +def _resolved(text: str, context: Mapping[str, str]) -> str: + return GITHUB_EXPRESSION.sub(lambda expression: _context_value(context, expression.group(1)), text) + + +def _install_tool(path: Path, script: str) -> None: + path.write_text(script) + path.chmod(0o755) + + +def _runner(tmp_path: Path, upstream: StaleBaseUpstream, pull_request: Mapping[str, str], live_base_ref: str) -> Runner: + runner: Final = Runner(tmp_path, upstream, live_base_ref, _event_context(pull_request)) + runner.tools.mkdir() + runner.temp.mkdir() + for name, script in (("gh", FAKE_GH), ("npm", FAKE_ARGV_RECORDER), ("uv", FAKE_ARGV_RECORDER)): + _install_tool(runner.tools / name, script) + return runner + + +def _check_out_like_the_job(job: WorkflowJob, runner: Runner) -> None: + checkout: Final = next(step for step in job.steps if step.uses.startswith("actions/checkout@")) + ref: Final = _resolved(checkout.with_.get("ref", "${{ github.event.pull_request.head.sha }}"), runner.context) + _git(runner.root, "init", "-q", str(runner.checkout)) + _git(runner.checkout, "remote", "add", "origin", f"file://{runner.upstream.path}") + _git(runner.checkout, "fetch", "-q", "--no-tags", "--depth=1", "origin", ref) + _git(runner.checkout, "checkout", "-q", "--detach", "FETCH_HEAD") + + +def _run_step(step: WorkflowStep, job: WorkflowJob, context: Mapping[str, str], runner: Runner) -> None: + result: Final = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", _resolved(step.run, context)], + cwd=runner.checkout / job.defaults.run.working_directory, + env={ + **{name: _resolved(value, context) for name, value in step.env.items()}, + "PATH": f"{runner.tools}{os.pathsep}{os.environ['PATH']}", + "HOME": str(runner.root), + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_WORKSPACE": str(REPO_ROOT), + "GITHUB_OUTPUT": str(runner.github_output), + "GITHUB_ENV": str(runner.github_env), + "RUNNER_TEMP": str(runner.temp), + "UPSTREAM": str(runner.upstream.path), + "LIVE_BASE_REF": runner.live_base_ref, + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def _key_values(path: Path) -> Mapping[str, str]: + return MappingProxyType( + {key: value for key, _, value in (line.partition("=") for line in path.read_text().splitlines())} + ) + + +def _argv_recorded(runner: Runner, tool: str) -> tuple[str, ...]: + return tuple((runner.temp / f"{tool}.argv").read_text().splitlines()) + + +def _gate_base_exported(run: StepRun) -> str: + return _key_values(run.runner.github_env)["GATE_BASE_SHA"] + + +def _ui_files_queued_for_prettier(run: StepRun) -> frozenset[str]: + return frozenset((run.runner.temp / "prettier_files.txt").read_text().splitlines()) + + +def _ui_files_passed_to_vitest_related(run: StepRun) -> frozenset[str]: + argv: Final = _argv_recorded(run.runner, "npm") + return frozenset(argv[argv.index("related") + 1 : argv.index("--run")]) + + +def _guard_base_argument_and_checkout(run: StepRun) -> tuple[str, str]: + guard: Final = next(step for step in run.step.job.steps if step.name == "Run the guard") + outputs: Final = { + f"steps.{run.step.step.id}.outputs.{name}": value + for name, value in _key_values(run.runner.github_output).items() + } + _run_step(guard, run.step.job, MappingProxyType({**run.runner.context, **outputs}), run.runner) + argv: Final = _argv_recorded(run.runner, "uv") + return argv[argv.index("--base") + 1], _sha(run.runner.checkout, "HEAD") + + +def _merge_base(upstream: StaleBaseUpstream, ref: str) -> str: + return upstream.merge_base_by_ref[ref] + + +def _ui_files_changed_since_merge_base(upstream: StaleBaseUpstream, ref: str) -> frozenset[str]: + merge_base: Final = upstream.merge_base_by_ref[ref] + changed: Final = _git(upstream.path, "diff", "--name-only", f"--relative={UI_DASHBOARD}", merge_base, upstream.head) + return frozenset(changed.split()) + + +def _merge_base_and_tip(upstream: StaleBaseUpstream, ref: str) -> tuple[str, str]: + return upstream.merge_base_by_ref[ref], upstream.tip_by_ref[ref] + + +CONSUMERS: Final = MappingProxyType( + { + "test-linting.yml:python": Consumer(_gate_base_exported, _merge_base), + "test-linting.yml:ui": Consumer(_ui_files_queued_for_prettier, _ui_files_changed_since_merge_base), + "test-unit.yml:ui-unit": Consumer(_ui_files_passed_to_vitest_related, _ui_files_changed_since_merge_base), + "spend-logs-guard.yml:spend-logs-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), + "cost-map-guard.yml:cost-map-guard": Consumer(_guard_base_argument_and_checkout, _merge_base_and_tip), + } +) + + +def _sha(repo: Path, revision: str) -> str: + return _git(repo, "rev-parse", revision).strip() + + +def _commit_ui_file(repo: Path, name: str) -> None: + target: Final = repo / UI_DASHBOARD / "src" / f"{name}.ts" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"export const {name} = 1;\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", name) + + +@pytest.fixture(scope="module") +def upstream(tmp_path_factory: pytest.TempPathFactory) -> StaleBaseUpstream: + path: Final = tmp_path_factory.mktemp("upstream") + _git(path, "init", "-q", "-b", "main") + _git(path, "config", "user.email", "t@t") + _git(path, "config", "user.name", "t") + _commit_ui_file(path, "main_when_the_pull_request_was_opened") + _git(path, "branch", "feature") + _git(path, "branch", RELEASE_BASE_REF) + _commit_ui_file(path, "main_since") + _git(path, "checkout", "-q", "-b", STACKED_BASE_REF) + _commit_ui_file(path, "stacked_base_since") + _git(path, "checkout", "-q", "feature") + _commit_ui_file(path, "pull_request_work") + _git(path, "merge", "-q", "--no-ff", "-m", "merge the newer base", STACKED_BASE_REF) + _git(path, "config", f"renamed.{RENAMED_BASE_REF}.to", STACKED_BASE_REF) + return StaleBaseUpstream( + path=path, + stale_base=_sha(path, "main~1"), + head=_sha(path, "feature"), + tip_by_ref=MappingProxyType({ref: _sha(path, ref) for ref in BASE_REFS}), + merge_base_by_ref=MappingProxyType( + {ref: _git(path, "merge-base", ref, "feature").strip() for ref in BASE_REFS} + ), + ) + + +def test_every_workflow_that_reads_a_merge_base_has_its_steps_discovered() -> None: + assert MERGE_BASE_STEPS + assert frozenset(step.workflow for step in MERGE_BASE_STEPS) == frozenset( + path.name for path in WORKFLOWS_DIR.glob("*.y*ml") if "merge_base_commit" in path.read_text() + ) + + +def _runs_for(step: MergeBaseStep, base_ref: str) -> bool: + return any(fnmatch.fnmatchcase(base_ref, pattern) for pattern in step.base_branches) + + +BASE_REF_CASES: Final = ( + ("main", "main", "main"), + ("stacked", STACKED_BASE_REF, STACKED_BASE_REF), + ("rerun_after_the_stacked_base_was_deleted", "litellm_deleted_after_merge", "main"), + ("rerun_after_the_stacked_base_was_renamed", RENAMED_BASE_REF, STACKED_BASE_REF), + ("rerun_after_a_retarget_off_main_to_a_stacked_branch", "main", STACKED_BASE_REF), + ("rerun_after_a_retarget_off_main_to_a_release_branch", "main", RELEASE_BASE_REF), +) +MERGE_BASE_CASES: Final = tuple( + pytest.param(step, event_base_ref, live_base_ref, id=f"{step.job_key}:{step.step.name}-{case}") + for step, (case, event_base_ref, live_base_ref) in itertools.product(MERGE_BASE_STEPS, BASE_REF_CASES) + if _runs_for(step, event_base_ref) +) + + +@pytest.mark.parametrize(("step", "event_base_ref", "live_base_ref"), MERGE_BASE_CASES) +def test_merge_base_step_hands_its_consumer_the_events_base_unless_that_branch_is_gone( + upstream: StaleBaseUpstream, tmp_path: Path, step: MergeBaseStep, event_base_ref: str, live_base_ref: str +) -> None: + pull_request: Final = MappingProxyType( + { + "number": "1", + "base.ref": event_base_ref, + "base.sha": upstream.stale_base, + "head.sha": upstream.head, + "head.ref": "feature", + "user.type": "User", + } + ) + runner: Final = _runner(tmp_path, upstream, pull_request, live_base_ref) + _check_out_like_the_job(step.job, runner) + _run_step(step.step, step.job, runner.context, runner) + consumer: Final = CONSUMERS[step.job_key] + measured_ref: Final = event_base_ref if event_base_ref in upstream.tip_by_ref else live_base_ref + assert consumer.observe(StepRun(step, runner)) == consumer.expect(upstream, measured_ref) + + UI_API_TYPES_DETECT: Final[str] = next( step["run"] for step in yaml.safe_load(WORKFLOW.read_text())["jobs"]["ui-api-types"]["steps"] diff --git a/tests/unit/test_select_ui_test_scope.py b/tests/unit/test_select_ui_test_scope.py index 4605aab956c..8f5a4b464bd 100644 --- a/tests/unit/test_select_ui_test_scope.py +++ b/tests/unit/test_select_ui_test_scope.py @@ -18,8 +18,12 @@ that consumes it, running the step's real shell against stubbed `gh`, `git` and from __future__ import annotations import os +import re import subprocess +from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType +from typing import Final import pytest import yaml @@ -28,6 +32,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SCOPE_SCRIPT = REPO_ROOT / ".github" / "scripts" / "select_ui_test_scope.sh" WORKFLOW = REPO_ROOT / ".github" / "workflows" / "test-unit.yml" STEP_NAME = "Run UI unit tests (Vitest)" +EXPRESSION = re.compile(r"\$\{\{\s*(.+?)\s*\}\}") +PULL_REQUEST_EVENT: Final = MappingProxyType( + {"number": "1", "base.ref": "main", "base.sha": "basesha", "head.sha": "headsha"} +) FULL_SUITE_ARGV = ["run", "test", "--", "--run", "--pool", "forks", "--maxWorkers=14"] @@ -73,13 +81,26 @@ def test_an_empty_change_set_fails_open_to_the_full_suite() -> None: assert scope([]) == "full" -def _step_script() -> str: - workflow = yaml.safe_load(WORKFLOW.read_text()) - steps = workflow["jobs"]["ui-unit"]["steps"] - script = next(step["run"] for step in steps if step.get("name") == STEP_NAME) - resolved = script.replace("${{ github.repository }}", "BerriAI/litellm") - assert "${{" not in resolved, "the step uses an Actions expression this harness does not resolve" - return resolved +def _resolve(text: str, pull_request: Mapping[str, str] | None) -> str: + def value(match: re.Match[str]) -> str: + expression: Final = match.group(1) + if expression == "github.repository": + return "BerriAI/litellm" + if expression == "github.token": + return "token" + field: Final = expression.removeprefix("github.event.pull_request.") + assert field != expression, f"the step uses ${{{{ {expression} }}}}, which this harness does not resolve" + return "" if pull_request is None else pull_request[field] + + return EXPRESSION.sub(value, text) + + +def _step_script_and_env(pull_request: Mapping[str, str] | None) -> tuple[str, dict[str, str]]: + workflow: Final = yaml.safe_load(WORKFLOW.read_text()) + steps: Final = workflow["jobs"]["ui-unit"]["steps"] + step: Final = next(step for step in steps if step.get("name") == STEP_NAME) + env: Final = {name: _resolve(str(value), pull_request) for name, value in step.get("env", {}).items()} + return _resolve(step["run"], pull_request), env def _stub(bin_dir: Path, name: str, body: str) -> None: @@ -88,7 +109,9 @@ def _stub(bin_dir: Path, name: str, body: str) -> None: stub.chmod(0o755) -def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> tuple[int, str, list[list[str]]]: +def _run_step( + tmp_path: Path, changed: list[str], pull_request: Mapping[str, str] | None = PULL_REQUEST_EVENT +) -> tuple[int, str, list[list[str]]]: """Run the workflow step's real shell; return its status, stdout and every npm argv.""" bin_dir = tmp_path / "bin" bin_dir.mkdir() @@ -100,20 +123,22 @@ def _run_step(tmp_path: Path, changed: list[str], base_sha: str = "basesha") -> _stub(bin_dir, "git", 'if [ "$1" = diff ]; then cat "$CHANGED_FILES"; fi') _stub(bin_dir, "npm", 'printf "%s\\n" "$@" >>"$NPM_LOG"; printf "\\0" >>"$NPM_LOG"') - step = tmp_path / "step.sh" - step.write_text(_step_script()) + step_script, step_env = _step_script_and_env(pull_request) + script: Final = tmp_path / "step.sh" + script.write_text(step_script) - env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env["BASE_SHA"] = base_sha - env["HEAD_SHA"] = "headsha" - env["GITHUB_WORKSPACE"] = str(REPO_ROOT) - env["GITHUB_REF_NAME"] = "release_branch" - env["CHANGED_FILES"] = str(changed_file) - env["NPM_LOG"] = str(npm_log) + env: Final = { + **os.environ, + **step_env, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "GITHUB_WORKSPACE": str(REPO_ROOT), + "GITHUB_REF_NAME": "release_branch", + "CHANGED_FILES": str(changed_file), + "NPM_LOG": str(npm_log), + } result = subprocess.run( - ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(step)], + ["bash", "--noprofile", "--norc", "-eo", "pipefail", str(script)], cwd=tmp_path, capture_output=True, text=True, @@ -155,7 +180,7 @@ def test_step_runs_the_full_suite_when_a_changed_file_is_outside_src(tmp_path: P def test_step_runs_the_full_suite_on_a_push(tmp_path: Path) -> None: - returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], base_sha="") + returncode, output, invocations = _run_step(tmp_path, ["src/app/page.tsx"], pull_request=None) assert returncode == 0, output assert invocations == [FULL_SUITE_ARGV] From a5b2198dc0f386d23664d46eb0757cba59962c22 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:28:26 -0700 Subject: [PATCH 13/25] fix(wandb): GLM-5.2 max_input_tokens from the CoreWeave Forge page (#49148) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b5d09d6ff4f..e8f6766ed25 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -65126,7 +65126,7 @@ "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, "max_output_tokens": 262144, - "max_input_tokens": 1049000, + "max_input_tokens": 262000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, "cache_read_input_token_cost": 1.4e-07, @@ -65134,7 +65134,7 @@ "litellm_provider": "wandb", "mode": "chat", "supports_vision": false, - "source": "https://wandb.ai/site/pricing/tokens/" + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference" }, "deepinfra/openai/gpt-oss-120b-Turbo": { "max_output_tokens": 131072, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b5d09d6ff4f..e8f6766ed25 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -65126,7 +65126,7 @@ "wandb/zai-org/GLM-5.2": { "supports_reasoning": true, "max_output_tokens": 262144, - "max_input_tokens": 1049000, + "max_input_tokens": 262000, "input_cost_per_token": 7.6e-07, "output_cost_per_token": 2.42e-06, "cache_read_input_token_cost": 1.4e-07, @@ -65134,7 +65134,7 @@ "litellm_provider": "wandb", "mode": "chat", "supports_vision": false, - "source": "https://wandb.ai/site/pricing/tokens/" + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference" }, "deepinfra/openai/gpt-oss-120b-Turbo": { "max_output_tokens": 131072, From 594f862bf3c859b1dcfe3fb9496ce22fdd72b49f Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:29:14 -0700 Subject: [PATCH 14/25] fix(streaming): rebuild each choice of an n>1 chat stream on its own (#49002) * fix(streaming): rebuild each choice of an n>1 chat stream on its own stream_chunk_builder folded every choice of an n>1 chat stream into choice 0, so post-call guardrails such as tool_permission re-emitted one choice holding every choice's tool calls, and spend logs and logging callbacks kept only the first choice's text. Each choice is now assembled from its own chunks, while usage, cost, and hidden params still read the whole stream. Cisco AI Defense compares each choice's streamed text on its own so safe n>1 streams still pass through unchanged. * fix(streaming): keep bridged reasoning summaries on choice 0 and infer an unreported choice finish reason * fix(streaming): match MCP follow-ups and bridged summary text to the rebuilt choices MCP auto-execute now answers only the first choice's tool calls, since its follow-up turn continues that choice alone. The Responses bridge streams a space before each reasoning summary part after the first, matching the non-streamed reply. build_base_response now takes the per-choice streams instead of having its choices replaced afterwards * fix(streaming): space every later reasoning summary part, deny per judged call, and rebuild in one text pass * fix(guardrails): key tool_permission denials for custom tool calls too, and type the stream rebuild's messages * fix(mcp): continue the first choice that called a tool when an n>1 reply auto-executes MCP tools * fix(noma): scan each choice of a multi-choice reply on its own * fix(noma): keep masked choices when a later choice's scan fails open on a stream --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../transformation.py | 10 +- .../streaming_chunk_builder_utils.py | 433 ++++++++++-------- .../chat/guardrail_translation/handler.py | 5 - .../cisco_ai_defense/cisco_ai_defense.py | 63 ++- .../guardrails/guardrail_hooks/noma/noma.py | 64 ++- .../guardrail_hooks/tool_permission.py | 25 +- .../mcp/litellm_proxy_mcp_handler.py | 40 +- ...responses_transformation_transformation.py | 39 ++ .../test_streaming_chunk_builder_utils.py | 112 ++++- .../test_cisco_ai_defense_chat.py | 56 +++ .../guardrails/guardrail_hooks/test_noma.py | 200 +++++++- .../guardrail_hooks/test_tool_permission.py | 111 +++++ .../mcp/test_litellm_proxy_mcp_handler.py | 80 ++++ tests/unit/test_main.py | 7 +- 14 files changed, 950 insertions(+), 295 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 580e55273f6..bb59b4617af 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state self._structured_output: Final = structured_output self._first_message_item_done: bool = False + self._reasoning_summary_streamed: bool = False def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place replays_openai_history: bool = False, structured_output: bool = False, + reasoning_summary_streamed: bool = False, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. @@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): tool_call_index_map: Per-stream output_index -> sequential tool_call index map replays_openai_history: Record the completed output's item sequence for the next turn structured_output: Only the first message item became content + reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text Returns: ModelResponseStream: OpenAI-formatted streaming chunk @@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) else: raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}") + elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed: + return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))]) elif event_type == "response.reasoning_summary_text.delta": content_part = parsed_chunk.get("delta", None) if content_part: return ModelResponseStream( choices=[ StreamingChoices( - index=cast(int, parsed_chunk.get("summary_index")), + index=0, delta=Delta(reasoning_content=content_part), ) ] @@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): tool_call_index_map=self._tool_call_index_map, replays_openai_history=self._replays_openai_history, structured_output=self._structured_output, + reasoning_summary_streamed=self._reasoning_summary_streamed, ) ) if self._structured_output and _is_message_item_done(chunk): self._first_message_item_done = True + if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"): + self._reasoning_summary_streamed = True return self._with_served_service_tier(self._with_stream_scoped_id(translated)) def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool: diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8e1791c3c92..c69d9277d65 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -3,7 +3,7 @@ import builtins import time from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass, replace -from itertools import chain, groupby +from itertools import chain, groupby, takewhile from types import MappingProxyType from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast @@ -257,7 +257,7 @@ def apply_grounding_request_counts( class ChunkProcessor: - def __init__(self, chunks: list, messages: Sequence | None = None): + def __init__(self, chunks: list, messages: Sequence[object] | None = None): self.chunks = self._sort_chunks(chunks) self.messages = messages self.first_chunk = chunks[0] @@ -390,6 +390,30 @@ class ChunkProcessor: case _: return "assistant" + @staticmethod + def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object: + finish_reason = unreported + for chunk in chunks: + if "choices" in chunk and len(chunk["choices"]) > 0: + chunk_finish_reason = None + if hasattr(chunk["choices"][0], "finish_reason"): + chunk_finish_reason = chunk["choices"][0].finish_reason + elif "finish_reason" in chunk["choices"][0]: + chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) + if chunk_finish_reason is not None: + finish_reason = chunk_finish_reason + return finish_reason + + @staticmethod + def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices: + return Choices( + index=index, + finish_reason=ChunkProcessor._finish_reason_of_chunks( + chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop" + ), + message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""}, + ) + @staticmethod def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str: """ @@ -414,7 +438,11 @@ class ChunkProcessor: None, ) - def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse: + def build_base_response( + self, + chunks: Sequence["_BaseChunk"], + choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]], + ) -> ModelResponse: chunk = self.first_chunk id: Final = ChunkProcessor._get_chunk_id(chunks) object: Final = chunk["object"] @@ -426,16 +454,7 @@ class ChunkProcessor: service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks) role: Final = ChunkProcessor._get_role_from_chunks(chunks) - finish_reason = "stop" - for chunk in chunks: - if "choices" in chunk and len(chunk["choices"]) > 0: - chunk_finish_reason = None - if hasattr(chunk["choices"][0], "finish_reason"): - chunk_finish_reason = chunk["choices"][0].finish_reason - elif "finish_reason" in chunk["choices"][0]: - chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"]) - if chunk_finish_reason is not None: - finish_reason = chunk_finish_reason + finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks) # Initialize the response dictionary response = ModelResponse( @@ -455,13 +474,17 @@ class ChunkProcessor: if service_tier is not None else MappingProxyType({}) ), - "choices": [ - { - "index": 0, - "message": {"role": role, "content": ""}, - "finish_reason": finish_reason, - } - ], + "choices": ( + [ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams] + if len(choice_streams) > 1 + else [ + { + "index": 0, + "message": {"role": role, "content": ""}, + "finish_reason": finish_reason, + } + ] + ), "usage": { "prompt_tokens": 0, # Modify as needed "completion_tokens": 0, # Modify as needed @@ -1520,11 +1543,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]: return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields} +def _stream_choice_index(choice: object) -> int: + index: Final = _field_of(choice, "index") + return index if isinstance(index, int) else 0 + + +def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool: + return any( + "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] + for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks) + ) + + +def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]: + indices: Final = sorted( + frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks))) + ) + if len(indices) <= 1: + return ((0, chunks),) + return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices) + + +def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]: + return tuple( + _BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index]) + for chunk in chunks + ) + + +def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None: + choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices")) + parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk))) + return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None + + +def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]: + tool_call_chunks: Final = [ + chunk + for chunk in chunks + if any( + "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None + for choice in chunk.get("choices") or () + ) + ] + + if tool_call_chunks: + tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks) + yield "content", None + yield "tool_calls", tool_calls_list + + function_call_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "function_call" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["function_call"] is not None + ] + + if function_call_chunks: + yield "content", None + yield "function_call", processor.get_combined_function_call_content(function_call_chunks) + + content_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "content" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["content"] is not None + ] + + if content_chunks: + yield "content", processor.get_combined_content(content_chunks) + + thinking_blocks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "thinking_blocks" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["thinking_blocks"] is not None + ] + + if thinking_blocks: + yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks) + + reasoning_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "reasoning_content" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["reasoning_content"] is not None + ] + + if reasoning_chunks: + yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks) + + typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream + reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks)) + if reasoning_items: + yield "reasoning_items", list(reasoning_items) + + annotation_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "annotations" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["annotations"] is not None + ] + + message_annotations, provider_annotations = ( + partition_annotations( + list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks)) + ) + if annotation_chunks + else (None, ()) + ) + if message_annotations is not None: + yield "annotations", message_annotations + + audio_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "audio" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["audio"] is not None + ] + + if audio_chunks: + yield "audio", processor.get_combined_audio_content(audio_chunks) + + # Handle image chunks from models like gemini-2.5-flash-image + # See: https://github.com/BerriAI/litellm/issues/19478 + image_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "images" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["images"] is not None + ] + + if image_chunks: + # Images come complete in a single chunk, collect all images from all chunks + all_images: Final = [] + for chunk in image_chunks: + all_images.extend(chunk["choices"][0]["delta"]["images"]) + yield "images", all_images + + # Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations) + # See: https://github.com/BerriAI/litellm/issues/17737 + provider_specific_chunks: Final = [ + chunk + for chunk in chunks + if chunk.get("choices") + and "provider_specific_fields" in chunk["choices"][0]["delta"] + and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None + ] + + provider_field_dicts: Final = tuple( + fields + for chunk in provider_specific_chunks + for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],) + if isinstance(fields, dict) + ) + streamed_citations: Final = tuple( + fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None + ) + citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {} + provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {} + combined_provider_fields: Final = { + key: value + for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts) + for key, value in fields.items() + if key != "citation" + } + + if combined_provider_fields: + yield "provider_specific_fields", combined_provider_fields + + def stream_chunk_builder( chunks: list, messages: Sequence | None = None, - start_time=None, - end_time=None, + start_time: object = None, + end_time: object = None, logging_obj: Optional["Logging"] = None, count_prompt_tokens: Callable[[], int] | None = None, ) -> ModelResponse | TextCompletionResponse | None: @@ -1561,26 +1761,15 @@ def stream_chunk_builder( usage_model: Final = usage_tokenizer_model( model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None ) - # Initialize the response dictionary - response: Final = processor.build_base_response(chunks) + choice_streams: Final = _choice_streams(chunks) + response: Final = processor.build_base_response(chunks, choice_streams) + rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams))) - # Fast path for the common text-only streaming case: - # avoid repeated multi-pass list scans over chunks. - simple_content_parts: Final[list[str]] = [] - is_simple_text_stream = True - for chunk in chunks: - if not chunk.get("choices"): - continue - - if (part := _simple_text_part(chunk["choices"])) is None: - is_simple_text_stream = False - break - if part: - simple_content_parts.append(part) - - if is_simple_text_stream: - if simple_content_parts: - response["choices"][0]["message"]["content"] = "".join(simple_content_parts) + simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams) + if None not in simple_texts: + for (choice, _), simple_text in zip(rebuilt_choices, simple_texts): + if simple_text: + choice.message.content = simple_text completion_output = get_content_from_model_response(response) usage = processor.calculate_usage( chunks=chunks, @@ -1619,165 +1808,9 @@ def stream_chunk_builder( processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj) return response - tool_call_chunks: Final = [ - chunk - for chunk in chunks - if any( - "tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None - for choice in chunk.get("choices") or () - ) - ] - - if len(tool_call_chunks) > 0: - tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks) - _choice = response.choices[0] - _choice.message.content = None - _choice.message.tool_calls = tool_calls_list - - function_call_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "function_call" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["function_call"] is not None - ] - - if len(function_call_chunks) > 0: - _choice = response.choices[0] - _choice.message.content = None - _choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks) - - content_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "content" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["content"] is not None - ] - - if len(content_chunks) > 0: - response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks) - - thinking_blocks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "thinking_blocks" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["thinking_blocks"] is not None - ] - - if len(thinking_blocks) > 0: - response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content( - thinking_blocks - ) - - reasoning_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "reasoning_content" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["reasoning_content"] is not None - ] - - if len(reasoning_chunks) > 0: - response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content( - reasoning_chunks - ) - - typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream - reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks)) - if reasoning_items: - response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items) - - annotation_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "annotations" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["annotations"] is not None - ] - - if len(annotation_chunks) > 0: - # Merge annotations from ALL chunks — providers may spread - # them across multiple streaming chunks or send them only in - # the final chunk. - all_annotations: Final[list[object]] = [] - for ac in annotation_chunks: - all_annotations.extend(ac["choices"][0]["delta"]["annotations"]) - message_annotations, provider_annotations = partition_annotations(all_annotations) - if message_annotations is not None: - response["choices"][0]["message"]["annotations"] = message_annotations - if provider_annotations: - response.choices[0].message.provider_specific_fields = { - **(response.choices[0].message.provider_specific_fields or {}), - "annotations": list(provider_annotations), - } - - audio_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "audio" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["audio"] is not None - ] - - if len(audio_chunks) > 0: - _choice = response.choices[0] - _choice.message.audio = processor.get_combined_audio_content(audio_chunks) - - # Handle image chunks from models like gemini-2.5-flash-image - # See: https://github.com/BerriAI/litellm/issues/19478 - image_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "images" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["images"] is not None - ] - - if len(image_chunks) > 0: - # Images come complete in a single chunk, collect all images from all chunks - all_images: Final = [] - for chunk in image_chunks: - all_images.extend(chunk["choices"][0]["delta"]["images"]) - response["choices"][0]["message"]["images"] = all_images - - # Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations) - # See: https://github.com/BerriAI/litellm/issues/17737 - provider_specific_chunks: Final = [ - chunk - for chunk in chunks - if chunk.get("choices") - and "provider_specific_fields" in chunk["choices"][0]["delta"] - and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None - ] - - if len(provider_specific_chunks) > 0: - provider_field_dicts: Final = tuple( - fields - for chunk in provider_specific_chunks - for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],) - if isinstance(fields, dict) - ) - streamed_citations: Final = tuple( - fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None - ) - citation_fields: Final = ( - {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {} - ) - combined_provider_fields: Final = { - key: value - for fields in (citation_fields, *provider_field_dicts) - for key, value in fields.items() - if key != "citation" - } - - if combined_provider_fields: - _choice = response.choices[0] - _choice.message.provider_specific_fields = { - **(_choice.message.provider_specific_fields or {}), - **combined_provider_fields, - } + for choice, choice_chunks in rebuilt_choices: + for field, value in _assembled_message_fields(processor, choice_chunks): + choice.message[field] = value completion_output = get_content_from_model_response(response) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3070bedd30d..5c5204ef2f4 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): responses_so_far: Sequence["ModelResponseStream"], litellm_logging_obj: "LiteLLMLoggingObj | None", ) -> "ModelResponse": - """``stream_chunk_builder`` folds every choice of a stream into one index-0 - choice, so the stream is rebuilt one choice index at a time (every chunk - kept, its choices narrowed to that index, so usage-only chunks still - count) and the rebuilt choices are stitched into one response, each - carrying the index the stream gave it.""" choice_indices: Final = tuple( sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices)) ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py index 45e785fd53c..3d555707159 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py +++ b/litellm/proxy/guardrails/guardrail_hooks/cisco_ai_defense/cisco_ai_defense.py @@ -22,6 +22,7 @@ import os from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass, replace from datetime import datetime +from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -120,6 +121,11 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" +def _streamed_choice_index(choice: object) -> int: + index: Final[object] = getattr(choice, "index", None) + return index if isinstance(index, int) else 0 + + class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): """ Cisco AI Defense guardrail integration. @@ -585,29 +591,44 @@ class CiscoAIDefenseGuardrail(CiscoAIDefenseMcpMixin, CustomGuardrail): @classmethod def _extract_streaming_chunk_scan_text(cls, chunks: Sequence[object]) -> str: + streamed_choices: Final = tuple( + choice + for choice in chain.from_iterable(getattr(chunk, "choices", None) or () for chunk in chunks) + if getattr(choice, "delta", None) is not None + ) + choice_indices: Final = sorted(frozenset(map(_streamed_choice_index, streamed_choices))) + choice_texts: Final = ( + cls._extract_streamed_deltas_scan_text( + tuple( + getattr(choice, "delta", None) + for choice in streamed_choices + if _streamed_choice_index(choice) == index + ) + ) + for index in choice_indices + ) + return " ".join(text for text in choice_texts if text) + + @classmethod + def _extract_streamed_deltas_scan_text(cls, deltas: Sequence[object]) -> str: original_text = "" argument_text = "" - for chunk in chunks: - choices = getattr(chunk, "choices", None) or [] - for c in choices: - delta: object | None = getattr(c, "delta", None) - if delta is None: - continue - text = getattr(delta, "content", None) - if isinstance(text, str): - original_text += text - reasoning_text = " ".join(cls._extract_message_reasoning_parts(delta)) - if reasoning_text: - original_text += reasoning_text - for tc in getattr(delta, "tool_calls", None) or []: - args = cls._extract_tool_call_arguments(tc) - if args: - argument_text += args - fc: object | None = getattr(delta, "function_call", None) - if fc is not None: - args = cls._extract_function_call_arguments(fc) - if args: - argument_text += args + for delta in deltas: + text = getattr(delta, "content", None) + if isinstance(text, str): + original_text += text + reasoning_text = " ".join(cls._extract_message_reasoning_parts(delta)) + if reasoning_text: + original_text += reasoning_text + for tc in getattr(delta, "tool_calls", None) or []: + args = cls._extract_tool_call_arguments(tc) + if args: + argument_text += args + fc: object | None = getattr(delta, "function_call", None) + if fc is not None: + args = cls._extract_function_call_arguments(fc) + if args: + argument_text += args return " ".join(part for part in (original_text, argument_text) if part) # ------------------------------------------------------------------ diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index 85f476ecd62..d69fe4068e0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -246,20 +246,38 @@ class NomaGuardrail(CustomGuardrail): ) -> str | None: """Shared logic for processing LLM response checks""" - start_time: Final = datetime.now() - extra_data: Final = self.get_guardrail_dynamic_request_body_params(request_data) - if not isinstance(response, litellm.ModelResponse): return None - content = None - for choice in response.choices: - if isinstance(choice, litellm.Choices) and choice.message.content: - content = choice.message.content - break + checked_contents: Final = tuple( + [ + await self._process_choice_check(request_data, response, position, content, user_auth, event_type) + for position, content in self._choice_texts(response) + ] + ) + return next(iter(checked_contents), None) - if not content or not isinstance(content, str): - return None + @staticmethod + def _choice_texts(response: litellm.ModelResponse) -> tuple[tuple[int, str], ...]: + return tuple( + (position, choice.message.content) + for position, choice in enumerate(response.choices) + if isinstance(choice, litellm.Choices) + and isinstance(choice.message.content, str) + and choice.message.content + ) + + async def _process_choice_check( + self, + request_data: dict[str, object], + response: litellm.ModelResponse, + position: int, + content: str, + user_auth: UserAPIKeyAuth, + event_type: GuardrailEventHooks | None, + ) -> str: + start_time: Final = datetime.now() + extra_data: Final = self.get_guardrail_dynamic_request_body_params(request_data) payload: Final = { "input": [ @@ -306,7 +324,7 @@ class NomaGuardrail(CustomGuardrail): anonymized_content: Final = self._extract_anonymized_content(response_json, ASSISTANT_ROLE) if anonymized_content: # Replace the LLM response content with anonymized version - self._replace_llm_response_content(response, anonymized_content) + self._replace_llm_response_content(response, anonymized_content, position) verbose_proxy_logger.debug("Noma guardrail anonymized LLM response: %s", anonymized_content) return anonymized_content @@ -486,21 +504,14 @@ class NomaGuardrail(CustomGuardrail): messages[i]["content"] = anonymized_content break - def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str): - """ - Replace the LLM response content with anonymized version. - - Args: - response: The original LLM response - anonymized_content: The anonymized content to replace with - """ + def _replace_llm_response_content(self, response: LLMResponse, anonymized_content: str, position: int): + """Replace the content of the choice at ``position`` with its anonymized version.""" if not isinstance(response, litellm.ModelResponse): return - # Replace content in all choices - for choice in response.choices: - if isinstance(choice, litellm.Choices) and choice.message.content: - choice.message.content = anonymized_content + choice: Final = response.choices[position] + if isinstance(choice, litellm.Choices): + choice.message.content = anonymized_content async def _check_user_message_background( self, @@ -815,6 +826,7 @@ class NomaGuardrail(CustomGuardrail): ) if isinstance(assembled_model_response, ModelResponse): + texts_before_check: Final = self._choice_texts(assembled_model_response) try: processed_response: Final = await self._check_llm_response( request_data, @@ -828,7 +840,11 @@ class NomaGuardrail(CustomGuardrail): if self.block_failures: raise verbose_proxy_logger.error("Noma streaming post-call hook failed: %s", e) - for chunk in all_chunks: + if self._choice_texts(assembled_model_response) == texts_before_check: + for chunk in all_chunks: + yield chunk + return + async for chunk in MockResponseIterator(model_response=assembled_model_response): yield chunk return diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index fa92a0c90e0..e8c094c0c37 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -36,6 +36,7 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( ) from litellm.types.utils import ( CallTypesLiteral, + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, Function, @@ -744,6 +745,14 @@ class ToolPermissionGuardrail(CustomGuardrail): return ToolResult(tool_use_id=tool_call.id, content=error_message, is_error=True) + @staticmethod + def _judged_tool_call( + tool_call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, + ) -> tuple[str | None, ...]: + if isinstance(tool_call, ChatCompletionMessageCustomToolCall): + return (tool_call.id, tool_call.type, tool_call.custom.name, tool_call.custom.input) + return (tool_call.id, tool_call.type, tool_call.function.name, tool_call.function.arguments) + def _modify_response_with_permission_errors( self, response: ModelResponse, @@ -761,11 +770,10 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.info("Blocking %s unauthorized tool uses", len(denied_tools)) - # Create a mapping of tool_use_id to error result - error_results: Final = {} - for tool_use, error in denied_tools: - error_result = self._create_permission_error_result(tool_use, error) - error_results[tool_use.id] = error_result + error_results: Final = { + self._judged_tool_call(tool_use): self._create_permission_error_result(tool_use, error) + for tool_use, error in denied_tools + } # Modify the response content for choice_index, choice in enumerate(response.choices): @@ -775,9 +783,8 @@ class ToolPermissionGuardrail(CustomGuardrail): # Rewrite tool_calls for tool_call in choice.message.tool_calls or []: - tool_call_id = tool_call.id - if tool_call_id in error_results: - error_result = error_results[tool_call_id] + error_result = error_results.get(self._judged_tool_call(tool_call)) + if error_result is not None: error_messages.append(error_result.content) else: filtered_tool_calls.append(tool_call) @@ -788,7 +795,7 @@ class ToolPermissionGuardrail(CustomGuardrail): getattr(choice.message, "function_call", None), choice_index ) if legacy_tool_call is not None: - legacy_error_result = error_results.get(legacy_tool_call.id) + legacy_error_result = error_results.get(self._judged_tool_call(legacy_tool_call)) if legacy_error_result is not None: choice.message.function_call = None error_messages.append(legacy_error_result.content) diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index c01d07fb5c3..c88a8c1cc28 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -35,10 +35,7 @@ from litellm.types.llms.openai import ToolParam as ResponsesToolParam from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.utils import ( CallTypes, - ChatCompletionMessageCustomToolCall, - ChatCompletionMessageToolCall, Choices, - Message, ModelResponse, StandardLoggingMCPToolCall, ) @@ -587,29 +584,18 @@ class LiteLLM_Proxy_MCP_Handler: _extract_tool_calls_from_response = extract_tool_calls_from_response + @staticmethod + def _continued_choice(response: ModelResponse) -> Choices | None: + choices: Final = tuple(choice for choice in response.choices if isinstance(choice, Choices)) + return next((choice for choice in choices if choice.message.tool_calls), next(iter(choices), None)) + @staticmethod def extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]: - """Extract tool calls from a chat completion response.""" - tool_calls: Final[list[object]] = [] - - try: - for choice in response.choices: - message: Message | None = getattr(choice, "message", None) - if message is None: - continue - tool_call_entries: ( - Sequence[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None - ) = getattr(message, "tool_calls", None) - if tool_call_entries: - for tool_call in tool_call_entries: - if hasattr(tool_call, "model_dump"): - tool_calls.append(tool_call.model_dump()) - else: - tool_calls.append(tool_call) - except Exception: - verbose_logger.exception("Failed to extract tool calls from chat completion response") - - return tool_calls + """Extract the tool calls of the first choice that made any, since the follow-up turn continues it alone.""" + continued_choice: Final = LiteLLM_Proxy_MCP_Handler._continued_choice(response) + if continued_choice is None: + return [] + return [tool_call.model_dump() for tool_call in continued_choice.message.tool_calls or ()] _extract_tool_calls_from_chat_response = extract_tool_calls_from_chat_response @@ -1018,9 +1004,9 @@ class LiteLLM_Proxy_MCP_Handler: message_to_append: dict | None = None try: - first_choice: Final = response.choices[0] - if isinstance(first_choice, Choices) and getattr(first_choice, "message", None): - message_to_append = first_choice.message.model_dump(exclude_none=True) + continued_choice: Final = LiteLLM_Proxy_MCP_Handler._continued_choice(response) + if continued_choice is not None: + message_to_append = continued_choice.message.model_dump(exclude_none=True) # Ensure tool_calls have arguments field (required by OpenAI API) if message_to_append.get("tool_calls"): for tool_call in message_to_append["tool_calls"]: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index d59f59703ad..4e6ed16e360 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -418,6 +418,45 @@ def test_openai_responses_chunk_parser_reasoning_summary(): assert delta.function_call is None +def test_bridged_reasoning_summary_parts_rebuild_into_the_one_choice_they_streamed_on(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + events = ( + {"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0}, + { + "type": "response.reasoning_summary_text.delta", + "delta": "First part.", + "output_index": 0, + "summary_index": 0, + }, + {"type": "response.reasoning_summary_part.done", "output_index": 0, "summary_index": 0}, + {"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 1}, + { + "type": "response.reasoning_summary_text.delta", + "delta": "Second part.", + "output_index": 0, + "summary_index": 1, + }, + {"type": "response.reasoning_summary_part.added", "output_index": 1, "summary_index": 0}, + { + "type": "response.reasoning_summary_text.delta", + "delta": "Next item.", + "output_index": 1, + "summary_index": 0, + }, + {"type": "response.output_text.delta", "delta": "Answer", "output_index": 2, "content_index": 0}, + ) + + rebuilt = litellm.stream_chunk_builder(chunks=[iterator.chunk_parser(event) for event in events]) + + assert [(choice.index, choice.message.reasoning_content, choice.message.content) for choice in rebuilt.choices] == [ + (0, "First part. Second part. Next item.", "Answer") + ] + + def test_chunk_parser_string_output_text_delta_produces_text(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( OpenAiResponsesToChatCompletionStreamIterator, diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index dd1f495b0d0..28587bf4749 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1304,12 +1304,118 @@ def test_stream_chunk_builder_keeps_each_choices_tool_call_arguments_apart(): ] ) - assert [(tool_call.id, tool_call.function.arguments) for tool_call in response.choices[0].message.tool_calls] == [ - ("call_a", '{"fruit": "persimmon"}'), - ("call_b", '{"fruit": "durian"}'), + assert [ + (choice.index, [(tool_call.id, tool_call.function.arguments) for tool_call in choice.message.tool_calls]) + for choice in response.choices + ] == [ + (0, [("call_a", '{"fruit": "persimmon"}')]), + (1, [("call_b", '{"fruit": "durian"}')]), ] +def test_stream_chunk_builder_rebuilds_each_choice_of_a_multi_choice_text_stream_on_its_own(): + def chunk(choice_index: int, content: str, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + object="chat.completion.chunk", + created=1234567890, + model="gpt-4.1-mini", + choices=[StreamingChoices(index=choice_index, delta=Delta(content=content), finish_reason=finish_reason)], + ) + + response = stream_chunk_builder( + chunks=[ + chunk(0, "The persim"), + chunk(1, "The"), + chunk(1, " persimmon is"), + chunk(0, "mon is ripe.", finish_reason="stop"), + chunk(1, " ripe and", finish_reason="length"), + ] + ) + + assert [(choice.index, choice.message.content, choice.finish_reason) for choice in response.choices] == [ + (0, "The persimmon is ripe.", "stop"), + (1, "The persimmon is ripe and", "length"), + ] + + +def test_stream_chunk_builder_gives_a_choice_without_a_finish_chunk_the_finish_reason_its_content_implies(): + def chunk(choice_index: int, delta: Delta, finish_reason: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + object="chat.completion.chunk", + created=1234567890, + model="gpt-4.1-mini", + choices=[StreamingChoices(index=choice_index, delta=delta, finish_reason=finish_reason)], + ) + + def tool_call(call_id: str) -> list[ChatCompletionDeltaToolCall]: + return [ + ChatCompletionDeltaToolCall( + id=call_id, index=0, type="function", function=Function(name="lookup_fruit", arguments="{}") + ) + ] + + response = stream_chunk_builder( + chunks=[ + chunk(0, Delta(tool_calls=tool_call("call_a"))), + chunk(1, Delta(tool_calls=tool_call("call_b"))), + chunk(2, Delta(content="Kiwis are fuzzy.")), + chunk(0, Delta(), finish_reason="tool_calls"), + ] + ) + + assert [(choice.index, choice.finish_reason) for choice in response.choices] == [ + (0, "tool_calls"), + (1, "tool_calls"), + (2, "stop"), + ] + + +def test_stream_chunk_builder_keeps_each_choices_function_call_annotations_and_images_on_that_choice(): + def chunk(choice_index: int, delta: Delta) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-123", + object="chat.completion.chunk", + created=1234567890, + model="gpt-4.1-mini", + choices=[StreamingChoices(index=choice_index, delta=delta, finish_reason=None)], + ) + + citation: Final = { + "type": "url_citation", + "url_citation": {"start_index": 0, "end_index": 5, "url": "https://example.com/kiwi", "title": "Kiwi"}, + } + image: Final = {"type": "image_url", "index": 0, "image_url": {"url": "data:image/png;base64,a2l3aQ=="}} + + response = stream_chunk_builder( + chunks=[ + chunk(0, Delta(function_call={"name": "lookup_fruit", "arguments": '{"fruit": '})), + chunk(1, Delta(content="Kiwis are")), + chunk(0, Delta(function_call={"arguments": '"kiwi"}'})), + chunk(1, Delta(content=" fuzzy.", annotations=[citation])), + chunk(1, Delta(images=[image])), + ] + ) + + function_choice, cited_choice = response.choices + assert ( + function_choice.index, + function_choice.message.content, + function_choice.message.function_call.name, + function_choice.message.function_call.arguments, + getattr(function_choice.message, "annotations", None), + getattr(function_choice.message, "images", None), + ) == (0, None, "lookup_fruit", '{"fruit": "kiwi"}', None, None) + assert ( + cited_choice.index, + cited_choice.message.content, + cited_choice.message.function_call, + cited_choice.message.annotations, + cited_choice.message.images, + ) == (1, "Kiwis are fuzzy.", None, [citation], [image]) + + def test_get_combined_tool_content_joins_many_dict_shaped_argument_fragments_in_order(): processor = ChunkProcessor.__new__(ChunkProcessor) first_fragments = [f"a{i};" for i in range(300)] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py index 4d1f254aef9..5ce78fd05d1 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_cisco_ai_defense_chat.py @@ -2108,6 +2108,62 @@ class TestCiscoAIDefenseStreamingBypass: f"Original: {chunks!r}, received: {received!r}" ) + @pytest.mark.asyncio + async def test_streaming_safe_n2_response_inspects_each_choice_and_yields_original_chunks(self): + g = _make_guardrail(event_hook=["pre_call", "post_call"]) + chunks = [ + ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), finish_reason=finish_reason, index=index)], + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + ) + for index, part, finish_reason in ( + (0, "The persim", None), + (1, "The", None), + (0, "mon is ripe.", "stop"), + (1, " persimmon is ripe.", "stop"), + ) + ] + + received, post_mock = await _streaming_setup(g, chunks, cisco_response=_safe_response()) + + inspected = post_mock.call_args.kwargs["json"]["messages"] + assert [m["content"] for m in inspected if m["role"] == "assistant"] == [ + "The persimmon is ripe.", + "The persimmon is ripe.", + ] + assert received == chunks + + @pytest.mark.asyncio + async def test_streaming_safe_n2_response_with_a_function_call_choice_yields_original_chunks(self): + g = _make_guardrail(event_hook=["pre_call", "post_call"]) + chunks = [ + ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=delta, finish_reason=None, index=index)], + created=1234567890, + model="gpt-4", + object="chat.completion.chunk", + ) + for index, delta in ( + (0, Delta(function_call={"name": "lookup_fruit", "arguments": '{"fruit": '})), + (1, Delta(content="Kiwis are")), + (0, Delta(function_call={"arguments": '"kiwi"}'})), + (1, Delta(content=" fuzzy.")), + ) + ] + + received, post_mock = await _streaming_setup(g, chunks, cisco_response=_safe_response()) + + inspected = post_mock.call_args.kwargs["json"]["messages"] + assert [m["content"] for m in inspected if m["role"] == "assistant"] == [ + '{"fruit": "kiwi"}', + "Kiwis are fuzzy.", + ] + assert received == chunks + @pytest.mark.asyncio async def test_streaming_redact_does_not_replay_tool_call_arguments(self): g = _make_guardrail( diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py index ee369f72f01..574550e1bdd 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_noma.py @@ -1,4 +1,5 @@ import copy +from itertools import chain import os from typing import cast from unittest.mock import AsyncMock, MagicMock, patch @@ -18,7 +19,7 @@ import litellm.proxy.guardrails.guardrail_hooks.noma.noma as noma_legacy_module from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage from litellm.proxy.guardrails.init_guardrails import init_guardrails_v2 from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import Choices, Message +from litellm.types.utils import Choices, Delta, Message, ModelResponseStream, StreamingChoices @pytest.fixture @@ -1848,12 +1849,207 @@ class TestNomaAnonymizationLogic: ) anonymize_guardrail._replace_llm_response_content( - response, "Your email is *******" + response, "Your email is *******", 0 ) assert response.choices[0].message.content == "Your email is *******" +def _noma_reply_per_text(verdict_for_text): + def reply(*args, **kwargs): + response = MagicMock() + response.json.return_value = verdict_for_text(kwargs["json"]["input"][0]["content"][0]["text"]) + response.raise_for_status = MagicMock() + return response + + return reply + + +class TestNomaMultiChoiceResponses: + @pytest.mark.asyncio + async def test_streamed_two_choice_reply_is_blocked_when_only_the_second_choice_is_flagged(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + monitor_mode=False, + block_failures=True, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + scanned_texts = [] + + def verdict_for_text(text): + scanned_texts.append(text) + return {"aggregatedScanResult": "launch code" in text, "scanResult": []} + + async def upstream(): + for index, part in ((0, "Kiwis are fuzzy."), (1, "The launch code is 0000.")): + yield ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), finish_reason="stop", index=index)], + created=1234567890, + model="gpt-4.1-mini", + object="chat.completion.chunk", + ) + + with patch.object(guardrail.async_handler, "post", side_effect=_noma_reply_per_text(verdict_for_text)): + with pytest.raises(NomaBlockedMessage): + async for _ in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=upstream(), + request_data={"litellm_call_id": "call-1", "metadata": {}}, + ): + pass + + assert scanned_texts == ["Kiwis are fuzzy.", "The launch code is 0000."] + + @pytest.mark.asyncio + async def test_each_choice_gets_its_own_anonymized_text(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + anonymize_input=True, + monitor_mode=False, + block_failures=True, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + + def verdict_for_text(text): + return { + "aggregatedScanResult": False, + "scanResult": [ + { + "role": "assistant", + "type": "message", + "results": { + "anonymizedContent": {"anonymized": text.replace("test@example.com", "*******")}, + "sensitiveData": {"PII": {"result": False, "status": "SUCCESS"}}, + }, + } + ], + } + + response = ModelResponse( + id="resp_1", + choices=[ + Choices(index=0, finish_reason="stop", message=Message(role="assistant", content="Mail test@example.com")), + Choices( + index=1, finish_reason="stop", message=Message(role="assistant", content="Write test@example.com") + ), + ], + model="gpt-4.1-mini", + ) + + with patch.object(guardrail.async_handler, "post", side_effect=_noma_reply_per_text(verdict_for_text)): + await guardrail.async_post_call_success_hook( + data={"litellm_call_id": "call-1", "metadata": {}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=response, + ) + + assert [choice.message.content for choice in response.choices] == ["Mail *******", "Write *******"] + + @pytest.mark.asyncio + async def test_fail_open_stream_keeps_redactions_made_before_a_later_choice_fails(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + anonymize_input=True, + monitor_mode=False, + block_failures=False, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + + def verdict_for_text(text): + if text.startswith("Write"): + raise httpx.ConnectError("Noma is unreachable") + return { + "aggregatedScanResult": False, + "scanResult": [ + { + "role": "assistant", + "type": "message", + "results": { + "anonymizedContent": {"anonymized": text.replace("test@example.com", "*******")}, + "sensitiveData": {"PII": {"result": False, "status": "SUCCESS"}}, + }, + } + ], + } + + async def upstream(): + for index, part in ((0, "Mail test@example.com"), (1, "Write back soon")): + yield ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), finish_reason="stop", index=index)], + created=1234567890, + model="gpt-4.1-mini", + object="chat.completion.chunk", + ) + + with patch.object(guardrail.async_handler, "post", side_effect=_noma_reply_per_text(verdict_for_text)): + streamed = [ + chunk + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=upstream(), + request_data={"litellm_call_id": "call-1", "metadata": {}}, + ) + ] + + streamed_choices = tuple(chain.from_iterable(part.choices for part in streamed)) + text_by_choice = { + index: "".join(choice.delta.content or "" for choice in streamed_choices if choice.index == index) + for index in (0, 1) + } + assert text_by_choice == {0: "Mail *******", 1: "Write back soon"} + + @pytest.mark.asyncio + async def test_fail_open_stream_replays_the_original_chunks_when_nothing_was_redacted(self): + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + anonymize_input=True, + monitor_mode=False, + block_failures=False, + guardrail_name="noma-post-call", + event_hook="post_call", + default_on=True, + ) + originals = [ + ModelResponseStream( + id="resp_1", + choices=[StreamingChoices(delta=Delta(content=part), index=0)], + created=1234567890, + model="gpt-4.1-mini", + object="chat.completion.chunk", + ) + for part in ("Kiwis", " are", " fuzzy.") + ] + + async def upstream(): + for chunk in originals: + yield chunk + + with patch.object(guardrail.async_handler, "post", side_effect=httpx.ConnectError("Noma is unreachable")): + streamed = [ + chunk + async for chunk in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=upstream(), + request_data={"litellm_call_id": "call-1", "metadata": {}}, + ) + ] + + assert len(streamed) == len(originals) + assert all(sent is original for sent, original in zip(streamed, originals)) + + class TestNomaAnonymizationFlow: """Test full anonymization flow with real Noma response objects""" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py index 2ee4c8c72c0..21376068686 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -5,6 +5,7 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics) import json import logging import re +from itertools import chain from typing import Final from unittest.mock import patch @@ -25,10 +26,16 @@ from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( PermissionError, ) from litellm.types.utils import ( + ChatCompletionCustomToolCallPayload, + ChatCompletionDeltaToolCall, + ChatCompletionMessageCustomToolCall, ChatCompletionMessageToolCall, Choices, + Delta, + Function, ModelResponse, ModelResponseStream, + StreamingChoices, ) @@ -425,6 +432,41 @@ class TestToolPermissionGuardrail: assert isinstance(choice.message.content, str) assert "berri" in choice.message.content + @pytest.mark.asyncio + async def test_rewrite_denies_only_the_choice_whose_call_is_denied_when_choices_share_a_call_id(self): + guardrail = ToolPermissionGuardrail( + guardrail_name="choice-guardrail", + rules=[{"id": "deny_bash", "tool_name": "Bash", "decision": "deny"}], + default_action="allow", + on_disallowed_action="rewrite", + event_hook=GuardrailEventHooks.post_call, + ) + response = ModelResponse( + choices=[ + Choices( + index=index, + finish_reason="tool_calls", + message={ + "tool_calls": [ + {"id": "call_shared", "type": "function", "function": {"name": name, "arguments": "{}"}} + ] + }, + ) + for index, name in enumerate(("Bash", "Read")) + ] + ) + + await guardrail.async_post_call_success_hook( + data={"guardrails": ["choice-guardrail"]}, user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + kept: Final = [ + [tool_call.function.name for tool_call in choice.message.tool_calls or ()] + for choice in response.choices + if isinstance(choice, Choices) + ] + assert kept == [[], ["Read"]] + @pytest.mark.asyncio async def test_async_post_call_success_hook_missing_arguments_blocks_param_rule( self, @@ -866,6 +908,46 @@ class TestToolPermissionGuardrail: f"got: {chunks[0].choices[0].delta.content!r}" ) + @pytest.mark.asyncio + async def test_async_post_call_streaming_iterator_hook_re_emits_each_choice_of_an_n2_tool_call_stream(self): + def chunk(choice_index: int, tool_call: ChatCompletionDeltaToolCall) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-n2-tools", + created=1700000000, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[StreamingChoices(index=choice_index, delta=Delta(tool_calls=[tool_call]))], + ) + + def fragment(arguments: str, name: str | None = None) -> ChatCompletionDeltaToolCall: + return ChatCompletionDeltaToolCall( + id="call_shared" if name else None, + index=0, + type="function", + function=Function(name=name, arguments=arguments), + ) + + async def _stream(): + for streamed in ( + chunk(0, fragment("", name="Bash")), + chunk(1, fragment("", name="Bash")), + chunk(0, fragment('{"command": "ls"}')), + chunk(1, fragment('{"command": "pwd"}')), + ): + yield streamed + + re_emitted: Final = [ + chunk + async for chunk in self.guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_stream(), request_data={} + ) + ] + + assert [ + (choice.index, [(call.function.name, call.function.arguments) for call in choice.delta.tool_calls]) + for choice in chain.from_iterable(streamed.choices for streamed in re_emitted) + ] == [(0, [("Bash", '{"command": "ls"}')]), (1, [("Bash", '{"command": "pwd"}')])] + def test_modify_response_with_permission_errors(self): # Setup a response with one tool_call tool_call = ChatCompletionMessageToolCall( @@ -952,6 +1034,35 @@ class TestToolPermissionGuardrail: assert "Permission denied" in choice.message.content + def test_modify_response_with_permission_errors_keeps_custom_tool_call_in_another_choice(self): + denied_call = ChatCompletionMessageToolCall( + id="call_bash", type="function", function=Function(name="Bash", arguments="{}") + ) + custom_call = ChatCompletionMessageCustomToolCall( + id="call_patch", custom=ChatCompletionCustomToolCallPayload(name="apply_patch", input="*** Begin Patch") + ) + response = ModelResponse( + choices=[ + Choices(index=0, message={"role": "assistant", "content": None, "tool_calls": [denied_call]}), + Choices(index=1, message={"role": "assistant", "content": None, "tool_calls": [custom_call]}), + ] + ) + denied_tools = [ + ( + denied_call, + PermissionError(tool_name="Bash", rule_id="deny_bash", message="Tool 'Bash' denied by rule 'deny_bash'"), + ) + ] + + self.guardrail._modify_response_with_permission_errors(response, denied_tools) + + kept = [ + [call.id for call in choice.message.tool_calls or []] + for choice in response.choices + if isinstance(choice, Choices) + ] + assert kept == [[], ["call_patch"]] + class TestToolPermissionGuardrailIntegration: """Integration tests for Tool Permission Guardrail""" diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 229d5cbe676..2f27622d77c 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -131,6 +131,86 @@ def test_extract_tool_calls_from_chat_response_handles_tool_calls(): assert tool_calls[0]["function"]["name"] == "foo" +def test_chat_follow_up_answers_only_tool_calls_in_its_assistant_turn(): + response = ModelResponse( + id="resp-n2", + choices=[ + { + "index": index, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": "foo", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + } + for index, call_id in enumerate(("call-a", "call-b")) + ], + model="gpt", + created=0, + object="chat.completion", + ) + tool_results = [ + {"tool_call_id": tool_call["id"], "name": tool_call["function"]["name"], "result": "done"} + for tool_call in LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response) + ] + + follow_up = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat( + [{"role": "user", "content": "hi"}], response, tool_results + ) + + assistant_call_ids = [tool_call["id"] for tool_call in follow_up[1]["tool_calls"]] + answered_call_ids = [message["tool_call_id"] for message in follow_up if message["role"] == "tool"] + assert answered_call_ids == assistant_call_ids == ["call-a"] + + +def test_chat_follow_up_continues_the_choice_that_called_a_tool_when_choice_0_answered(): + response = ModelResponse( + id="resp-n2-second", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "no tool needed"}, + "finish_reason": "stop", + }, + { + "index": 1, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call-b", "type": "function", "function": {"name": "foo", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + }, + ], + model="gpt", + created=0, + object="chat.completion", + ) + tool_results = [ + {"tool_call_id": tool_call["id"], "name": tool_call["function"]["name"], "result": "done"} + for tool_call in LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response) + ] + + follow_up = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat( + [{"role": "user", "content": "hi"}], response, tool_results + ) + + assistant_call_ids = [tool_call["id"] for tool_call in follow_up[1]["tool_calls"]] + answered_call_ids = [message["tool_call_id"] for message in follow_up if message["role"] == "tool"] + assert answered_call_ids == assistant_call_ids == ["call-b"] + + +def test_extract_tool_calls_from_chat_response_without_choices_finds_none(): + response = ModelResponse(id="resp-empty", choices=[], model="gpt", created=0, object="chat.completion") + + assert LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response) == [] + + def test_create_follow_up_messages_for_chat_appends_tool_results(): original_messages = [{"role": "user", "content": "hi"}] response = ModelResponse( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 11104cf7bca..571f890a4e6 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -2831,9 +2831,10 @@ def test_stream_chunk_builder_keeps_tool_calls_carried_only_by_a_later_choice_of response = stream_chunk_builder(chunks=chunks) - tool_calls = response.choices[0].message.tool_calls - assert tool_calls is not None - assert [(call.id, call.function.name, call.function.arguments) for call in tool_calls] == [ + text_choice, tool_choice = response.choices + assert (text_choice.index, text_choice.message.content, text_choice.message.tool_calls) == (0, "hello world", None) + assert tool_choice.index == 1 + assert [(call.id, call.function.name, call.function.arguments) for call in tool_choice.message.tool_calls] == [ ("call_1", "lookup_fruit", '{"fruit":"kiwi"}') ] From 326f17594efb084b9d6080551147c1d12aed032e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:29:30 -0700 Subject: [PATCH 15/25] fix(batches): return real batch token counts next to OpenAI's batch usage fields (#47740) * fix(batches): read OpenAI batch usage into the batch token counts * fix(batches): serve OpenAI's batch usage shape from retrieve and the cost poller * fix(batches): keep the chat usage keys next to OpenAI's batch usage fields Batch usage now carries every chat key it had plus OpenAI's five batch fields with the same counts, so poller-billed batches keep prompt_tokens and cache-write tokens, and older proxies still read the new rows. Usage with missing or null detail counts reads its real input and output counts. * test(batches): check the batch usage counts in the e2e terminal-state test * fix(types): apply nested include and exclude to both batch usage shapes * fix(batches): log no cost for a batch retrieve that is not final yet A retrieve now reads OpenAI's batch usage into the chat counts, so a retrieve that saw the batch completed before its output file arrived priced those counts for the logging callbacks, and the final retrieve priced the whole batch again. A batch that is not final now logs no cost, as it did before. Batch usage dumps also pass exclude_unset and exclude_defaults through to the usage keys. * fix(batches): pass through every usage detail field the provider sends Batch dumps rebuilt input_tokens_details and output_tokens_details from the chat keys, which dropped any detail field past cached_tokens and reasoning_tokens that the merge base passed through. The dump now starts from the provider's stored details and only sets the two counts derived from the chat keys * fix(batches): skip sync loggers on the cost poller's own retrieve and honor exclude_none on provider details * fix(batches): keep sync logger events at $0 on poller-owned retrieves, pass unparsed provider counts through, log batch usage in chat shape * fix(batches): store only chat usage extras on batch spend rows Spend rows for batch retrieves copied OpenAI's batch fields (input_tokens, output_tokens and their details) into metadata.additional_usage_values next to the chat counts they duplicate. Build the row from the batch's chat usage, shared with the logging payload through batch_chat_usage. --- litellm/litellm_core_utils/litellm_logging.py | 19 +- .../spend_tracking/spend_tracking_utils.py | 9 +- litellm/types/utils.py | 151 +++++++++++++ tests/e2e/batches/batch_client.py | 13 +- tests/e2e/batches/test_batches_e2e.py | 7 + .../test_litellm_logging.py | 110 ++++++++- .../common_utils/test_check_batch_cost.py | 17 +- .../test_spend_tracking_utils.py | 59 +++++ tests/unit/types/test_types_utils.py | 213 ++++++++++++++++++ 9 files changed, 575 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 34c98efc22f..4dd33fa5565 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -202,6 +202,7 @@ from litellm.types.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, + batch_chat_usage, ) from litellm.types.videos.main import VideoObject from litellm.utils import get_base_model_from_metadata, print_verbose @@ -2091,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass): elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set router_model_id = hidden_params["model_id"] + if isinstance(priced_result, LiteLLMBatch) and ( + not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result) + ): + return 0.0 + # Fallback: extract router_model_id from litellm_params when not available # from the result object. ResponsesAPIResponse objects (used by /v1/responses # streaming) don't carry _hidden_params["model_id"] like ModelResponse does. @@ -3346,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass): finally: self._restore_correlation_context() + def _is_batch_poll_without_logging(self, result: object) -> bool: + litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {} + return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True + def _success_handler_body( self, result: object = None, @@ -3860,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass): ## CALCULATE COST FOR BATCH JOBS if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch): - litellm_params = self.litellm_params or {} - litellm_metadata: Final = litellm_params.get("litellm_metadata") or {} - if ( - litellm_metadata.get("batch_ignore_default_logging", False) is True - ): # polling job will query these frequently, don't spam db logs + if self._is_batch_poll_without_logging(result): return batch_cost: Final = kwargs.get("batch_cost", None) @@ -7481,7 +7487,8 @@ def get_standard_logging_object_payload( # Extract usage as a plain dict, avoiding Pydantic round-trip raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict( response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj, - combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")), + combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")) + or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None), ) usage_dict: Final = ( {**raw_usage_dict, "output_image_count": len(init_response_obj.data)} diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 634a805d72b..09b7ce7ee9a 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -87,6 +87,7 @@ from litellm.types.utils import ( AzureSpillover, CallTypes, CostBreakdown, + LiteLLMBatch, LlmProviders, StandardLoggingGuardrailInformation, StandardLoggingKubernetesPodRouting, @@ -96,6 +97,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, StandardLoggingVectorStoreRequest, VectorStoreSearchResponse, + batch_chat_usage, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -626,7 +628,12 @@ def get_logging_payload( usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) elif not is_unbilled_non_inference_call(call_type, metadata, response_obj_dict): # Use response_obj_dict instead of response_obj to avoid calling .get() on Pydantic models - _usage: Final = response_obj_dict.get("usage", None) or {} + chat_usage_of_batch: Final = batch_chat_usage(response_obj) if isinstance(response_obj, LiteLLMBatch) else None + _usage: Final = ( + chat_usage_of_batch.model_dump() + if chat_usage_of_batch is not None + else response_obj_dict.get("usage", None) or {} + ) if isinstance(_usage, litellm.Usage): usage = dict(_usage) elif isinstance(_usage, dict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1055ffe7b2f..181ed5ea6db 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -20,6 +20,7 @@ from openai._models import BaseModel as OpenAIObject from openai.types.audio.transcription_create_params import ( FileTypes as FileTypes, ) +from openai.types.batch_usage import BatchUsage, InputTokensDetails, OutputTokensDetails from openai.types.chat.chat_completion import ChatCompletion as ChatCompletion from openai.types.completion_usage import ( CompletionTokensDetails, @@ -44,6 +45,7 @@ from pydantic import ( JsonValue, PrivateAttr, SkipValidation, + ValidationError, field_serializer, field_validator, ) @@ -4785,6 +4787,112 @@ class LiteLLMFineTuningJob(FineTuningJob): self._hidden_params = kwargs.get("_hidden_params", {}) +class _BatchInputTokensDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + cached_tokens: int | None = None + + +class _BatchOutputTokensDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + reasoning_tokens: int | None = None + + +class _BatchTokenCounts(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + output_tokens: int + total_tokens: int | None = None + input_tokens_details: _BatchInputTokensDetails | None = None + output_tokens_details: _BatchOutputTokensDetails | None = None + + +def _parse_batch_token_counts(value: object) -> _BatchTokenCounts | None: + if not isinstance(value, Mapping): + return None + try: + return _BatchTokenCounts.model_validate(value) + except ValidationError: + return None + + +def _prompt_details_with_cached_tokens( + details: PromptTokensDetailsWrapper | None, cached_tokens: int | None +) -> PromptTokensDetailsWrapper | None: + if not cached_tokens: + return details + return (details or PromptTokensDetailsWrapper()).model_copy(update={"cached_tokens": cached_tokens}) + + +def _completion_details_with_reasoning_tokens( + details: CompletionTokensDetailsWrapper | None, reasoning_tokens: int | None +) -> CompletionTokensDetailsWrapper | None: + if not reasoning_tokens: + return details + return (details or CompletionTokensDetailsWrapper()).model_copy(update={"reasoning_tokens": reasoning_tokens}) + + +def _usage_with_batch_token_counts(usage: Usage, counts: _BatchTokenCounts) -> Usage: + return usage.model_copy( + update={ + "prompt_tokens": counts.input_tokens, + "completion_tokens": counts.output_tokens, + "total_tokens": ( + counts.input_tokens + counts.output_tokens if counts.total_tokens is None else counts.total_tokens + ), + "prompt_tokens_details": _prompt_details_with_cached_tokens( + usage.prompt_tokens_details, + counts.input_tokens_details.cached_tokens if counts.input_tokens_details else None, + ), + "completion_tokens_details": _completion_details_with_reasoning_tokens( + usage.completion_tokens_details, + counts.output_tokens_details.reasoning_tokens if counts.output_tokens_details else None, + ), + } + ) + + +class _ProviderBatchUsageDetails(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + input_tokens_details: Mapping[str, JsonValue] | None = None + output_tokens_details: Mapping[str, JsonValue] | None = None + + +def _provider_batch_usage_details(usage: Usage) -> _ProviderBatchUsageDetails: + try: + return _ProviderBatchUsageDetails.model_validate(usage) + except ValidationError: + return _ProviderBatchUsageDetails() + + +def _has_unparsed_batch_token_counts(usage: Usage) -> bool: + provider_fields: Final = usage.model_extra or {} + sent_batch_counts: Final = "input_tokens" in provider_fields or "output_tokens" in provider_fields + return sent_batch_counts and _parse_batch_token_counts(provider_fields) is None + + +def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage: + provider_details: Final = _provider_batch_usage_details(usage) + cached_tokens: Final = usage.prompt_tokens_details.cached_tokens if usage.prompt_tokens_details else None + reasoning_tokens: Final = ( + usage.completion_tokens_details.reasoning_tokens if usage.completion_tokens_details else None + ) + return BatchUsage( + input_tokens=usage.prompt_tokens, + input_tokens_details=InputTokensDetails.model_validate( + {**(provider_details.input_tokens_details or {}), "cached_tokens": cached_tokens or 0} + ), + output_tokens=usage.completion_tokens, + output_tokens_details=OutputTokensDetails.model_validate( + {**(provider_details.output_tokens_details or {}), "reasoning_tokens": reasoning_tokens or 0} + ), + total_tokens=usage.total_tokens, + ) + + class LiteLLMBatch(Batch): _hidden_params: dict = {} @@ -4798,6 +4906,40 @@ class LiteLLMBatch(Batch): usage: Usage | None = None + @field_validator("usage", mode="before") + @classmethod + def _usage_from_batch_token_counts(cls, value: object) -> object: + counts: Final = _parse_batch_token_counts(value) + return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts) + + @field_serializer("usage") + def _serialize_usage_with_batch_fields( + self, usage: Usage | None, info: FieldSerializationInfo + ) -> Mapping[str, object] | None: + if not isinstance(usage, Usage): + return usage + chat_fields: Final = usage.model_dump( + mode=info.mode, + include=info.include, + exclude=info.exclude, + exclude_unset=info.exclude_unset, + exclude_defaults=info.exclude_defaults, + exclude_none=info.exclude_none, + ) + if _has_unparsed_batch_token_counts(usage): + return chat_fields + return { + **chat_fields, + **_batch_usage_from_chat_usage(usage).model_dump( + mode=info.mode, + include=info.include, + exclude=info.exclude, + exclude_unset=info.exclude_unset, + exclude_defaults=info.exclude_defaults, + exclude_none=info.exclude_none, + ), + } + def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator return hasattr(self, key) @@ -4818,6 +4960,15 @@ class LiteLLMBatch(Batch): return self.dict() +_PROVIDER_ONLY_BATCH_USAGE_FIELDS: Final = frozenset(BatchUsage.model_fields) - frozenset(Usage.model_fields) + + +def batch_chat_usage(batch: LiteLLMBatch) -> Usage | None: + if batch.usage is None: + return None + return Usage.model_validate(batch.usage.model_dump(exclude=set(_PROVIDER_ONLY_BATCH_USAGE_FIELDS))) + + class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase): # Events are already well-formed provider dicts. Validating them against the # OpenAIRealtimeEvents union makes Pydantic try every member per event, which diff --git a/tests/e2e/batches/batch_client.py b/tests/e2e/batches/batch_client.py index b02b09e557b..598593c1fcd 100644 --- a/tests/e2e/batches/batch_client.py +++ b/tests/e2e/batches/batch_client.py @@ -15,7 +15,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Final, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from e2e_metadata import step from proxy_client import ProxyClient @@ -77,6 +77,16 @@ class BatchErrorList(BaseModel): data: list[BatchErrorItem] = [] +class BatchUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None + input_tokens: int | None = None + output_tokens: int | None = None + + class BatchObject(BaseModel): id: str object: str | None = None @@ -90,6 +100,7 @@ class BatchObject(BaseModel): completion_window: str | None = None created_at: int | None = None model: str | None = None + usage: BatchUsage | None = None class BatchList(BaseModel): diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index 54bcb80ed24..b7a52da5902 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -1893,6 +1893,13 @@ class TestBatchTerminalState: f"listed-completed marker retrieved as {fetched.status!r}" ) assert fetched.output_file_id, "completed batch has no output_file_id" + usage: Final = fetched.usage + assert usage is not None and usage.input_tokens and usage.output_tokens, ( + f"completed batch has no batch token counts: {usage!r}" + ) + assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), ( + f"chat token counts disagree with the batch token counts: {usage!r}" + ) downloaded = client.proxy.transport.download( f"/v1/files/{fetched.output_file_id}/content", diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index a6d99a5ba55..c320cb36d73 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -1629,22 +1629,33 @@ class TestRetrieveBatchPricesOnlyFinalBatches: litellm_call_id="batch-call-2", function_id="f", ) - obj.custom_llm_provider = "openai" + obj.update_environment_variables( + model="gpt-5.6-luna", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) return obj @staticmethod def _batch(status: str, output_file_id: str | None): from litellm.types.utils import LiteLLMBatch - return LiteLLMBatch( - id="batch_6a9c99e185588190877d391f8b9d7f8a", - completion_window="24h", - created_at=1, - endpoint="/v1/chat/completions", - input_file_id="file-in", - object="batch", - status="validating", - output_file_id=output_file_id, + return LiteLLMBatch.model_validate( + { + "id": "batch_6a9c99e185588190877d391f8b9d7f8a", + "completion_window": "24h", + "created_at": 1, + "endpoint": "/v1/chat/completions", + "input_file_id": "file-in", + "object": "batch", + "status": "validating", + "output_file_id": output_file_id, + "usage": { + "input_tokens": 29, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 48, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 77, + }, + } ).model_copy(update={"status": status}) @pytest.mark.asyncio @@ -1658,11 +1669,12 @@ class TestRetrieveBatchPricesOnlyFinalBatches: handle_completed_batch = AsyncMock() monkeypatch.setattr(logging_module, "handle_completed_batch", handle_completed_batch) batch = self._batch(status, output_file_id) + logging_obj = self._logging_obj() - await self._logging_obj()._async_success_handler_body(result=batch, start_time=None, end_time=None) + await logging_obj._async_success_handler_body(result=batch, start_time=None, end_time=None) handle_completed_batch.assert_not_awaited() - assert "response_cost" not in batch._hidden_params + assert logging_obj.model_call_details["response_cost"] == 0.0 @pytest.mark.asyncio async def test_completed_batch_with_output_is_priced(self, monkeypatch) -> None: @@ -1689,6 +1701,80 @@ class TestRetrieveBatchPricesOnlyFinalBatches: assert batch.usage is not None assert batch.usage.total_tokens == 35 + @pytest.mark.parametrize("batch_ignore_default_logging", [True, False]) + def test_sync_logger_gets_a_free_event_for_the_cost_poller_retrieve(self, batch_ignore_default_logging: bool) -> None: + sync_events: list[object] = [] + + class SyncOnlyLogger(CustomLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: dt_object, + end_time: dt_object, + ) -> None: + sync_events.append(kwargs["response_cost"]) + + logging_obj = LitellmLogging( + model="gpt-5.6-luna", + messages=[{"role": "user", "content": "Hey"}], + stream=False, + call_type="aretrieve_batch", + start_time=time.time(), + litellm_call_id="batch-call-3", + function_id="f", + dynamic_success_callbacks=[SyncOnlyLogger()], + ) + logging_obj.update_environment_variables( + model="gpt-5.6-luna", + user=None, + optional_params={}, + litellm_params={"litellm_metadata": {"batch_ignore_default_logging": batch_ignore_default_logging}}, + custom_llm_provider="openai", + ) + + logging_obj.success_handler( + result=self._batch("completed", "file-out"), + start_time=dt_object.now(), + end_time=dt_object.now(), + for_async_request=True, + ) + + assert len(sync_events) == 1 + assert isinstance(sync_events[0], float) + assert (sync_events[0] == 0.0) is batch_ignore_default_logging + + def test_logged_usage_object_keeps_the_batch_chat_usage(self) -> None: + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + from litellm.types.utils import Usage + + aggregate: Final = Usage( + prompt_tokens=29, completion_tokens=48, total_tokens=77, reasoning_tokens=3, cache_creation_input_tokens=300 + ) + billed: Final = self._batch("completed", "file-out").model_copy(update={"usage": aggregate}) + provider_read: Final = self._batch("completed", "file-out") + now: Final = dt_object(2026, 1, 1, 12, 0, 0) + + usage_objects: Final = tuple( + payload["metadata"]["usage_object"] + for payload in ( + get_standard_logging_object_payload( + kwargs={"litellm_params": {}}, + init_response_obj=batch, + start_time=now, + end_time=now, + logging_obj=self._logging_obj(), + status="success", + ) + for batch in (billed, provider_read) + ) + if payload is not None + ) + + assert usage_objects[0] == aggregate.model_dump() + assert usage_objects[1]["prompt_tokens"] == 29 + assert "input_tokens" not in usage_objects[1] + class TestAnthropicPassthroughCustomPricing: """Verify the Anthropic pass-through handler forwards custom pricing.""" diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 86fc069f73b..4861b2687de 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -80,10 +80,11 @@ def _batch_cost_result( """Build the BatchCostUsageResult calculate_batch_cost_and_usage now returns, for mocking it in tests that only care about cost/usage/models.""" from litellm.batches.batch_utils import BatchCostUsageResult + from litellm.types.utils import Usage return BatchCostUsageResult( cost=cost, - usage=usage, + usage=Usage(**usage), models=models, successful_requests=successful_requests, failed_requests=failed_requests, @@ -3261,7 +3262,7 @@ class TestMultiPodBatchCostClaim: patch( "litellm.batches.batch_utils.calculate_batch_cost_and_usage", new_callable=AsyncMock, - return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5}, ["gpt-4"]), + return_value=_batch_cost_result(0.01, {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, ["gpt-4"]), ), patch( "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", @@ -3676,7 +3677,17 @@ class TestMultiPodBatchCostClaim: assert journal == ["fetch", "claim", "bill"] assert row.status == "complete" - assert json.loads(row.file_object)["usage"] == {"prompt_tokens": 10, "completion_tokens": 5} + assert json.loads(row.file_object)["usage"] == { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "prompt_tokens_details": None, + "completion_tokens_details": None, + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 0}, + } logging_obj.async_success_handler.assert_awaited_once() assert row.batch_processed is True diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 740334252d2..1ea445fda0d 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -69,6 +69,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + LiteLLMBatch, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, @@ -222,6 +223,64 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123 +_PROVIDER_READ_BATCH: Final = LiteLLMBatch.model_validate( + { + "id": "batch_spend_row", + "completion_window": "24h", + "created_at": 1, + "endpoint": "/v1/chat/completions", + "input_file_id": "file-in", + "object": "batch", + "status": "completed", + "output_file_id": "file-out", + "usage": { + "input_tokens": 29, + "input_tokens_details": {"cached_tokens": 8}, + "output_tokens": 46, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 75, + }, + } +) + + +@pytest.mark.parametrize( + "batch", + [ + _PROVIDER_READ_BATCH, + _PROVIDER_READ_BATCH.model_copy( + update={ + "usage": litellm.Usage( + prompt_tokens=29, + completion_tokens=46, + total_tokens=75, + prompt_tokens_details={"cached_tokens": 8}, + ) + } + ), + ], + ids=["provider_read", "poller_billed"], +) +def test_batch_spend_row_stores_the_same_usage_extras_as_a_chat_call_with_its_counts(batch: LiteLLMBatch) -> None: + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "call_type": "aretrieve_batch", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + }, + response_obj=batch, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + ) + + assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (29, 46, 75) + assert json.loads(payload["metadata"])["additional_usage_values"] == _get_additional_usage_values_for_usage( + litellm.Usage( + prompt_tokens=29, completion_tokens=46, total_tokens=75, prompt_tokens_details={"cached_tokens": 8} + ) + ) + + class _HashingCache(litellm.Cache): def __init__(self) -> None: pass diff --git a/tests/unit/types/test_types_utils.py b/tests/unit/types/test_types_utils.py index d85e48540f9..fe7b5a9470f 100644 --- a/tests/unit/types/test_types_utils.py +++ b/tests/unit/types/test_types_utils.py @@ -25,11 +25,13 @@ from litellm.types.utils import ( HiddenParams, ImageObject, ImageResponse, + LiteLLMBatch, ModelResponse, ModelResponseStream, TextCompletionResponse, all_litellm_params, is_decisions_model_mode, + Usage, text_tokens_without_nested_reasoning, ) @@ -1007,3 +1009,214 @@ def test_text_tokens_without_nested_reasoning_clamps( ) == expected_text_tokens ) + + +_BATCH_FIELDS: Final = { + "id": "batch_abc", + "object": "batch", + "endpoint": "/v1/responses", + "input_file_id": "file-abc", + "completion_window": "24h", + "status": "completed", + "created_at": 1, +} +_PROVIDER_BATCH_USAGE: Final = { + "input_tokens": 21, + "input_tokens_details": {"cached_tokens": 5}, + "output_tokens": 543, + "output_tokens_details": {"reasoning_tokens": 448}, + "total_tokens": 564, +} + + +def _batch_token_counts(batch: LiteLLMBatch) -> tuple[int, int, int, int | None, int | None]: + assert batch.usage is not None + assert batch.usage.prompt_tokens_details is not None + assert batch.usage.completion_tokens_details is not None + return ( + batch.usage.prompt_tokens, + batch.usage.completion_tokens, + batch.usage.total_tokens, + batch.usage.prompt_tokens_details.cached_tokens, + batch.usage.completion_tokens_details.reasoning_tokens, + ) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"), + pytest.param( + {"prompt_tokens": 0, "completion_tokens": 0, **_PROVIDER_BATCH_USAGE}, + id="stored_row_with_zeroed_chat_fields", + ), + ), +) +def test_batch_usage_reads_token_counts_from_the_provider_batch_usage(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + + assert _batch_token_counts(batch) == (21, 543, 564, 5, 448) + assert batch.usage is not None + assert (batch.usage.input_tokens, batch.usage.output_tokens) == (21, 543) + + +def test_batch_usage_keeps_chat_shaped_usage_as_is() -> None: + batch: Final = LiteLLMBatch.model_validate( + {**_BATCH_FIELDS, "usage": {"prompt_tokens": 28, "completion_tokens": 8, "total_tokens": 36}} + ) + + assert batch.usage is not None + assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (28, 8, 36) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param({"input_tokens": 10, "output_tokens": 5}, id="no_details_or_total"), + pytest.param( + {"input_tokens": 10, "input_tokens_details": None, "output_tokens": 5, "output_tokens_details": None}, + id="null_details", + ), + pytest.param( + { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": None}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": None}, + "total_tokens": 15, + }, + id="null_detail_counts", + ), + ), +) +def test_batch_usage_reads_token_counts_from_partial_provider_batch_usage(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + dumped_usage: Final = json.loads(batch.model_dump_json())["usage"] + + assert batch.usage is not None + assert (batch.usage.prompt_tokens, batch.usage.completion_tokens, batch.usage.total_tokens) == (10, 5, 15) + assert (dumped_usage["input_tokens"], dumped_usage["output_tokens"], dumped_usage["total_tokens"]) == (10, 5, 15) + + +@pytest.mark.parametrize( + "usage", + ( + pytest.param(_PROVIDER_BATCH_USAGE, id="provider_batch_usage"), + pytest.param( + Usage( + prompt_tokens=21, + completion_tokens=543, + total_tokens=564, + prompt_tokens_details={"cached_tokens": 5}, + completion_tokens_details={"reasoning_tokens": 448}, + ), + id="billed_chat_usage", + ), + ), +) +def test_batch_dump_carries_the_chat_and_provider_batch_usage_fields(usage: dict[str, object] | Usage) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + dumped: Final = batch.model_dump_json() + dumped_usage: Final = json.loads(dumped)["usage"] + + assert {key: dumped_usage[key] for key in _PROVIDER_BATCH_USAGE} == _PROVIDER_BATCH_USAGE + assert (dumped_usage["prompt_tokens"], dumped_usage["completion_tokens"]) == (21, 543) + assert batch.model_dump()["usage"] == dumped_usage + assert _batch_token_counts(LiteLLMBatch.model_validate_json(dumped)) == (21, 543, 564, 5, 448) + + +def test_batch_dump_keeps_cache_write_tokens_from_billed_usage() -> None: + batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy( + update={"usage": Usage(prompt_tokens=30, completion_tokens=4, total_tokens=34, cache_creation_input_tokens=7)} + ) + dumped: Final = batch.model_dump_json() + dumped_usage: Final = json.loads(dumped)["usage"] + + assert (dumped_usage["prompt_tokens"], dumped_usage["input_tokens"]) == (30, 30) + assert (dumped_usage["completion_tokens"], dumped_usage["output_tokens"]) == (4, 4) + assert dumped_usage["prompt_tokens_details"]["cache_write_tokens"] == 7 + assert None not in batch.model_dump(exclude_none=True)["usage"]["prompt_tokens_details"].values() + assert json.loads(LiteLLMBatch.model_validate_json(dumped).model_dump_json())["usage"] == dumped_usage + + +def test_batch_json_keeps_usage_that_skipped_validation() -> None: + batch: Final = LiteLLMBatch.model_validate(_BATCH_FIELDS).model_copy(update={"usage": {"prompt_tokens": 3}}) + + assert json.loads(batch.model_dump_json(warnings=False))["usage"] == {"prompt_tokens": 3} + + +def test_batch_dump_applies_nested_include_and_exclude_to_both_usage_shapes() -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": _PROVIDER_BATCH_USAGE}) + + excluded: Final = batch.model_dump(exclude={"usage": {"prompt_tokens_details", "input_tokens_details"}})["usage"] + included: Final = batch.model_dump(include={"usage": {"prompt_tokens", "input_tokens"}})["usage"] + + assert {"prompt_tokens_details", "input_tokens_details"}.isdisjoint(excluded) + assert (excluded["prompt_tokens"], excluded["input_tokens"]) == (21, 21) + assert included == {"prompt_tokens": 21, "input_tokens": 21} + + +def test_batch_dump_applies_exclude_defaults_to_the_chat_usage_keys() -> None: + batch: Final = LiteLLMBatch.model_validate( + {**_BATCH_FIELDS, "usage": Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)} + ) + + python_usage: Final = batch.model_dump(exclude_defaults=True)["usage"] + json_usage: Final = json.loads(batch.model_dump_json(exclude_defaults=True, warnings=False))["usage"] + + assert python_usage == json_usage + assert {"prompt_tokens_details", "completion_tokens_details"}.isdisjoint(python_usage) + assert (python_usage["prompt_tokens"], python_usage["input_tokens"]) == (10, 10) + + +def test_batch_dump_forwards_every_provider_usage_detail_field() -> None: + batch: Final = LiteLLMBatch.model_validate( + { + **_BATCH_FIELDS, + "usage": { + **_PROVIDER_BATCH_USAGE, + "input_tokens_details": {"cached_tokens": 5, "audio_tokens": 3}, + "output_tokens_details": {"reasoning_tokens": 448, "audio_tokens": 2}, + }, + } + ) + + python_usage: Final = batch.model_dump()["usage"] + json_usage: Final = json.loads(batch.model_dump_json(warnings=False))["usage"] + + assert python_usage == json_usage + assert python_usage["input_tokens_details"] == {"cached_tokens": 5, "audio_tokens": 3} + assert python_usage["output_tokens_details"] == {"reasoning_tokens": 448, "audio_tokens": 2} + + +@pytest.mark.parametrize( + "usage", + [ + {"input_tokens": 29, "output_tokens": None, "total_tokens": 29}, + {"input_tokens": 29, "input_tokens_details": {"cached_tokens": 8}, "total_tokens": 29}, + ], +) +def test_batch_dump_serves_unparsed_provider_usage_unchanged(usage: dict[str, object]) -> None: + batch: Final = LiteLLMBatch.model_validate({**_BATCH_FIELDS, "usage": usage}) + + python_usage: Final = batch.model_dump()["usage"] + json_usage: Final = json.loads(batch.model_dump_json(warnings=False))["usage"] + + assert python_usage == json_usage + assert {key: python_usage[key] for key in usage} == usage + + +def test_batch_dump_applies_exclude_none_to_provider_usage_detail_fields() -> None: + batch: Final = LiteLLMBatch.model_validate( + { + **_BATCH_FIELDS, + "usage": {**_PROVIDER_BATCH_USAGE, "input_tokens_details": {"cached_tokens": 5, "audio_tokens": None}}, + } + ) + + python_usage: Final = batch.model_dump(exclude_none=True)["usage"] + json_usage: Final = json.loads(batch.model_dump_json(exclude_none=True, warnings=False))["usage"] + + assert python_usage == json_usage + assert python_usage["input_tokens_details"] == {"cached_tokens": 5} + assert (python_usage["prompt_tokens"], python_usage["input_tokens"]) == (21, 21) From d25cde47924743fc78c22ee81bc26af176b4baee Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:30:07 -0700 Subject: [PATCH 16/25] chore(cost-map): add fireworks ultramarine 1 flash us and date deepseek v4 flash (#49146) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 19 +++++++++++++++++++ model_prices_and_context_window.json | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e8f6766ed25..82b1ad9bc5d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -27794,6 +27794,7 @@ }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -28242,6 +28243,7 @@ }, "fireworks_ai/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -80613,5 +80615,22 @@ "/v1/messages" ], "supports_native_streaming": true + }, + "fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": { + "cache_read_input_token_cost": 4.5e-08, + "cache_read_input_token_cost_priority": 5.625e-08, + "input_cost_per_token": 2.25e-07, + "input_cost_per_token_priority": 2.8125e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "output_cost_per_token_priority": 9.375e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": false, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e8f6766ed25..82b1ad9bc5d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -27794,6 +27794,7 @@ }, "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -28242,6 +28243,7 @@ }, "fireworks_ai/deepseek-v4-flash": { "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-08-14", "input_cost_per_token": 1.4e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, @@ -80613,5 +80615,22 @@ "/v1/messages" ], "supports_native_streaming": true + }, + "fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": { + "cache_read_input_token_cost": 4.5e-08, + "cache_read_input_token_cost_priority": 5.625e-08, + "input_cost_per_token": 2.25e-07, + "input_cost_per_token_priority": 2.8125e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "output_cost_per_token_priority": 9.375e-07, + "source": "https://docs.fireworks.ai/serverless/pricing", + "supports_function_calling": false, + "supports_response_schema": true, + "supports_tool_choice": false, + "supports_vision": true } } From 53ce41625fdd64afd26a70b77179018a0760bccc Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:30:38 -0700 Subject: [PATCH 17/25] chore(databricks): retirement dates for DeepSeek V4 Flash (0731), V4 Pro (0813) and Inkling (#49124) * chore(databricks): retirement dates for DeepSeek V4 Pro (0813) and Inkling * chore(databricks): retirement dates for DeepSeek V4 Flash (0731), V4 Pro (0813) and Inkling --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 43 ++++++++++--------- model_prices_and_context_window.json | 43 ++++++++++--------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 82b1ad9bc5d..72509bd3c69 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21435,7 +21435,7 @@ "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, "deprecation_date": "2026-10-09", - "input_cost_per_token": 2.9999900000000002e-06, + "input_cost_per_token": 2.99999e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -21444,7 +21444,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_dbu_cost_per_token": 0.000214286, "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -21557,6 +21557,7 @@ "databricks/databricks-deepseek-v4-flash-0731": { "cache_creation_input_token_cost": 1.4e-07, "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-11-05", "input_cost_per_token": 1.4e-07, "input_dbu_cost_per_token": 2e-06, "litellm_provider": "databricks", @@ -21569,7 +21570,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "output_dbu_cost_per_token": 4e-06, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21590,6 +21591,7 @@ "databricks/databricks-deepseek-v4-pro-0813": { "cache_creation_input_token_cost": 1.31999e-06, "cache_read_input_token_cost": 1.3202e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.31999e-06, "input_dbu_cost_per_token": 1.8857e-05, "litellm_provider": "databricks", @@ -21601,7 +21603,7 @@ "mode": "chat", "output_cost_per_token": 3.95997e-06, "output_dbu_cost_per_token": 5.6571e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21788,7 +21790,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22047,7 +22049,7 @@ "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 128000, @@ -22170,7 +22172,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22193,7 +22195,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22264,7 +22266,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22539,7 +22541,7 @@ "databricks/databricks-gpt-5-mini": { "cache_creation_input_token_cost": 2.4997e-07, "cache_read_input_token_cost": 2.499e-08, - "input_cost_per_token": 2.4997000000000006e-07, + "input_cost_per_token": 2.4997e-07, "input_dbu_cost_per_token": 3.571e-06, "litellm_provider": "databricks", "max_input_tokens": 272000, @@ -22548,7 +22550,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.9999700000000004e-06, + "output_cost_per_token": 1.99997e-06, "output_dbu_cost_per_token": 2.8571e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22571,7 +22573,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.9998000000000007e-07, + "output_cost_per_token": 3.9998e-07, "output_dbu_cost_per_token": 5.714000000000001e-06, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22581,7 +22583,7 @@ "databricks/databricks-gpt-oss-120b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 131072, @@ -22606,7 +22608,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.0001999999999996e-07, + "output_cost_per_token": 3.0002e-07, "output_dbu_cost_per_token": 4.285999999999999e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22639,7 +22641,7 @@ "databricks/databricks-gte-large-en": { "cache_creation_input_token_cost": 1.2999e-07, "cache_read_input_token_cost": 1.2999e-07, - "input_cost_per_token": 1.2999000000000001e-07, + "input_cost_per_token": 1.2999e-07, "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, @@ -22655,6 +22657,7 @@ "databricks/databricks-inkling": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.7003e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -22665,7 +22668,7 @@ "mode": "chat", "output_cost_per_token": 4.04999e-06, "output_dbu_cost_per_token": 5.7857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image" @@ -22730,7 +22733,7 @@ "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true @@ -22738,7 +22741,7 @@ "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -22747,7 +22750,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 4.5003000000000007e-07, + "output_cost_per_token": 4.5003e-07, "output_dbu_cost_per_token": 6.429000000000001e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22763,7 +22766,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 82b1ad9bc5d..72509bd3c69 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21435,7 +21435,7 @@ "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, "deprecation_date": "2026-10-09", - "input_cost_per_token": 2.9999900000000002e-06, + "input_cost_per_token": 2.99999e-06, "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -21444,7 +21444,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_dbu_cost_per_token": 0.000214286, "prompt_cache_min_tokens": 1024, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -21557,6 +21557,7 @@ "databricks/databricks-deepseek-v4-flash-0731": { "cache_creation_input_token_cost": 1.4e-07, "cache_read_input_token_cost": 2.8e-08, + "deprecation_date": "2026-11-05", "input_cost_per_token": 1.4e-07, "input_dbu_cost_per_token": 2e-06, "litellm_provider": "databricks", @@ -21569,7 +21570,7 @@ "mode": "chat", "output_cost_per_token": 2.8e-07, "output_dbu_cost_per_token": 4e-06, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21590,6 +21591,7 @@ "databricks/databricks-deepseek-v4-pro-0813": { "cache_creation_input_token_cost": 1.31999e-06, "cache_read_input_token_cost": 1.3202e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.31999e-06, "input_dbu_cost_per_token": 1.8857e-05, "litellm_provider": "databricks", @@ -21601,7 +21603,7 @@ "mode": "chat", "output_cost_per_token": 3.95997e-06, "output_dbu_cost_per_token": 5.6571e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text" ], @@ -21788,7 +21790,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_200k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22047,7 +22049,7 @@ "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 128000, @@ -22170,7 +22172,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22193,7 +22195,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.999990000000002e-06, + "output_cost_per_token": 9.99999e-06, "output_dbu_cost_per_token": 0.000142857, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22264,7 +22266,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000020000000002e-05, + "output_cost_per_token": 1.500002e-05, "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", @@ -22539,7 +22541,7 @@ "databricks/databricks-gpt-5-mini": { "cache_creation_input_token_cost": 2.4997e-07, "cache_read_input_token_cost": 2.499e-08, - "input_cost_per_token": 2.4997000000000006e-07, + "input_cost_per_token": 2.4997e-07, "input_dbu_cost_per_token": 3.571e-06, "litellm_provider": "databricks", "max_input_tokens": 272000, @@ -22548,7 +22550,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.9999700000000004e-06, + "output_cost_per_token": 1.99997e-06, "output_dbu_cost_per_token": 2.8571e-05, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22571,7 +22573,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.9998000000000007e-07, + "output_cost_per_token": 3.9998e-07, "output_dbu_cost_per_token": 5.714000000000001e-06, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_function_calling": true, @@ -22581,7 +22583,7 @@ "databricks/databricks-gpt-oss-120b": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 131072, @@ -22606,7 +22608,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 3.0001999999999996e-07, + "output_cost_per_token": 3.0002e-07, "output_dbu_cost_per_token": 4.285999999999999e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22639,7 +22641,7 @@ "databricks/databricks-gte-large-en": { "cache_creation_input_token_cost": 1.2999e-07, "cache_read_input_token_cost": 1.2999e-07, - "input_cost_per_token": 1.2999000000000001e-07, + "input_cost_per_token": 1.2999e-07, "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, @@ -22655,6 +22657,7 @@ "databricks/databricks-inkling": { "cache_creation_input_token_cost": 1.00002e-06, "cache_read_input_token_cost": 1.7003e-07, + "deprecation_date": "2026-10-30", "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", @@ -22665,7 +22668,7 @@ "mode": "chat", "output_cost_per_token": 4.04999e-06, "output_dbu_cost_per_token": 5.7857e-05, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image" @@ -22730,7 +22733,7 @@ "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true @@ -22738,7 +22741,7 @@ "databricks/databricks-meta-llama-3-1-8b-instruct": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 1.5001e-07, - "input_cost_per_token": 1.5000999999999998e-07, + "input_cost_per_token": 1.5001e-07, "input_dbu_cost_per_token": 2.1429999999999996e-06, "litellm_provider": "databricks", "max_input_tokens": 200000, @@ -22747,7 +22750,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 4.5003000000000007e-07, + "output_cost_per_token": 4.5003e-07, "output_dbu_cost_per_token": 6.429000000000001e-06, "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, @@ -22763,7 +22766,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5000300000000002e-06, + "output_cost_per_token": 1.50003e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true From 19e538ab11aa072b30a78877b13a02cfb1298449 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:40:34 -0700 Subject: [PATCH 18/25] fix(cost-map): set together_ai Qwen3.7-Max back to the published price (#49125) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 72509bd3c69..8f514a9dfb7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48618,13 +48618,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 7.5e-06, + "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 72509bd3c69..8f514a9dfb7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48618,13 +48618,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 2.5e-06, + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 7.5e-06, + "output_cost_per_token": 4.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, From e0778bc9dda7e38e523ef4ab481b2bcc26a6db9b Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:43:24 -0700 Subject: [PATCH 19/25] refactor(types): replace Any with proven types in 21 files (#48921) * refactor(types): replace Any with proven types in 35 files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): keep anthropic chat transformation json path untyped Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): accept null gemini vector store fields and load encryption TypedDicts on python 3.10 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(types): validate Python client user responses and type delete_user as the count it returns * fix(types): keep loose payloads working under the sweep's validators An off-schema Anthropic stream event is now skipped instead of ending the guardrail's text and stop_reason scan, and MAI image usage accepts the loose numbers it did before. New dict and list adapters hide their input from validation errors, RerankResponse indexing keeps its field types, and the delete_user test mocks the int /user/delete returns. * fix(types): make the changed DynamoAI fields ReadOnly and skip non-object A2A stream items * refactor(types): tighten round 2 parse sites and keep payloads out of errors Parse Anthropic SSE events with validate_json and ignore extra fields, validate admin team and key rows per row with hidden inputs, type rerank get and __getitem__ per key, and move the Phoenix prompt version response to a frozen BaseModel * fix(types): keep MAI image calls alive on unreadable usage counts An unreadable MAI usage count now counts as absent instead of failing a billed image call, and the provider's usage fields pass through as sent. The Anthropic stream scan warnings no longer log raw model output, and the off-schema skip has tests that fail on the old scan. * fix(types): count non-object MAI usage as absent and parse Anthropic SSE with json.loads --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/caching/caching.py | 7 +- litellm/images/utils.py | 2 +- .../arize/arize_phoenix_client.py | 11 +- litellm/integrations/opentelemetry.py | 6 +- .../a2a/chat/guardrail_translation/handler.py | 7 +- .../chat/guardrail_translation/handler.py | 64 ++++++---- .../image_generation/mai_transformation.py | 118 ++++++++++++++---- litellm/llms/bedrock/embed/embedding.py | 10 +- .../audio_transcription/transformation.py | 5 +- .../image_generation/gpt_transformation.py | 5 +- litellm/proxy/auth/oauth2_check.py | 5 +- litellm/proxy/client/cli/commands/models.py | 12 +- litellm/proxy/client/users.py | 14 ++- .../team_metadata_validator_e2e.py | 16 ++- .../vigil_guard/vigil_guard.py | 4 +- .../hooks/max_budget_per_session_limiter.py | 55 +++++--- .../internal_user_endpoints.py | 26 +++- litellm/proxy/realtime_endpoints/endpoints.py | 25 +++- .../guardrails/guardrail_hooks/dynamoai.py | 11 +- .../proxy/guardrails/guardrail_hooks/noma.py | 2 +- litellm/types/rerank.py | 33 ++++- .../integrations/arize/test_arize_phoenix.py | 17 +++ .../test_a2a_guardrail_handler.py | 4 + .../test_anthropic_guardrail_handler.py | 51 ++++++++ .../test_mai_image_generation.py | 70 +++++++++++ tests/unit/proxy/client/test_users.py | 5 +- .../test_internal_user_endpoints.py | 28 +++++ 27 files changed, 496 insertions(+), 117 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index d875ac09d40..b58f40af155 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -18,7 +18,7 @@ from contextlib import contextmanager from enum import Enum from typing import Any, Final, Literal, cast -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter import litellm from litellm._internal_context import current_service_target, service_target @@ -43,6 +43,9 @@ from .s3_cache import S3Cache RESPONSE_CACHE_TARGET: Final = "llm_response" _ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"}) _DECODED_LITERAL: Final = TypeAdapter(object) +_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter( + list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True) +) @contextmanager @@ -207,7 +210,7 @@ class Cache: if not redis_startup_nodes: _env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES") if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str): - redis_startup_nodes = json.loads(_env_cluster_nodes) + redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes)) if redis_startup_nodes: # Only pass GCP parameters if they are provided diff --git a/litellm/images/utils.py b/litellm/images/utils.py index 4e0cde24a68..d1a2dd76af0 100644 --- a/litellm/images/utils.py +++ b/litellm/images/utils.py @@ -20,7 +20,7 @@ class ImageEditRequestUtils: image_edit_optional_params: ImageEditOptionalRequestParams, drop_params: bool | None = None, additional_drop_params: Collection[str] | None = None, - ) -> dict: + ) -> dict[str, object]: """ Get optional parameters for the image edit API. diff --git a/litellm/integrations/arize/arize_phoenix_client.py b/litellm/integrations/arize/arize_phoenix_client.py index 6e8ebb99a9e..4d50389faf1 100644 --- a/litellm/integrations/arize/arize_phoenix_client.py +++ b/litellm/integrations/arize/arize_phoenix_client.py @@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix. import urllib.parse from typing import Final +from pydantic import BaseModel, ConfigDict + from litellm.llms.custom_httpx.http_handler import HTTPHandler +class _PromptVersionResponse(BaseModel): + model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True) + + data: dict[str, object] | None = None + + def _sanitize_id(identifier: str) -> str: """Reject path traversal characters and URL-encode the identifier.""" if any(c in identifier for c in ("/", "\\", "#", "?")): @@ -71,8 +79,7 @@ class ArizePhoenixClient: response: Final = self.http_handler.get(url, headers=self.headers) response.raise_for_status() - data: Final = response.json() - return data.get("data") + return _PromptVersionResponse.model_validate(response.json()).data except Exception as e: # Check if it's an HTTP error diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 0d0e309034a..a9bd93e6420 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -72,7 +72,7 @@ if TYPE_CHECKING: ) from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth - Span = _Span | Any + Span = _Span Tracer = _Tracer Context = _Context SpanExporter = _SpanExporter @@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span = None # Only set attributes if the span is still recording (not closed) # Note: parent_span is guaranteed to be not None here - if hasattr(parent_span, "set_status"): + if parent_span is not None and hasattr(parent_span, "set_status"): parent_span.set_status(Status(StatusCode.OK)) self.set_attributes(parent_span, kwargs, response_obj) # Raw-request as direct child of parent_span @@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # record error on parent span (keeps hierarchy shallow) # Only set attributes if the span is still recording (not closed) # Note: parent_otel_span is guaranteed to be not None here - if parent_otel_span.is_recording(): + if parent_otel_span is not None and parent_otel_span.is_recording(): self.set_attributes(parent_otel_span, kwargs, response_obj) parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message)) diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py index afd9ba84c96..5dc9679fe78 100644 --- a/litellm/llms/a2a/chat/guardrail_translation/handler.py +++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py @@ -14,6 +14,7 @@ import json from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Final, Optional +from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger @@ -28,6 +29,8 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class _A2ATextPart(TypedDict, total=False): """The subset of an A2A message part this handler reads text from.""" @@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation): obj = item elif isinstance(item, str): try: - obj = json.loads(item.strip()) - except (json.JSONDecodeError, TypeError): + obj = _JSON_OBJECT.validate_python(json.loads(item.strip())) + except (json.JSONDecodeError, TypeError, ValidationError): continue else: continue diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 31b0f06251d..e351542da01 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -18,9 +18,9 @@ from copy import deepcopy from dataclasses import dataclass from itertools import chain, repeat from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger @@ -349,14 +349,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ... ) -class _AnthropicSSEDelta(TypedDict, total=False): - type: ReadOnly[str] +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSETextDelta(TypedDict, total=False): + type: ReadOnly[object] text: ReadOnly[str] - stop_reason: ReadOnly[str | None] -class _AnthropicSSEEvent(TypedDict, total=False): - delta: ReadOnly[_AnthropicSSEDelta] +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSETextDeltaEvent(TypedDict, total=False): + delta: ReadOnly[_SSETextDelta] + + +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSEStopDelta(TypedDict, total=False): + stop_reason: ReadOnly[object] + + +@with_config(ConfigDict(extra="ignore", strict=True)) +class _SSEStopDeltaEvent(TypedDict, total=False): + delta: ReadOnly[_SSEStopDelta] + + +_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent) +_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent) +_SSEEventT = TypeVar("_SSEEventT") + + +def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None: + try: + return adapter.validate_python(json.loads(data_line)) + except json.JSONDecodeError: + verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type) + except ValidationError: + verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type) + return None class AnthropicMessagesHandler(BaseTranslation): @@ -1806,13 +1832,10 @@ class AnthropicMessagesHandler(BaseTranslation): # Only process content_block_delta events if event_type == "content_block_delta" and data_line: - try: - data: _AnthropicSSEEvent = json.loads(data_line) - delta: _AnthropicSSEDelta = data.get("delta", {}) - if delta.get("type") == "text_delta": - text += delta.get("text", "") - except json.JSONDecodeError: - verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) + data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line) + delta: _SSETextDelta = {} if data is None else data.get("delta", {}) + if delta.get("type") == "text_delta": + text += delta.get("text", "") except Exception as e: verbose_proxy_logger.error("Error extracting text from SSE: %s", e) @@ -1869,14 +1892,11 @@ class AnthropicMessagesHandler(BaseTranslation): # Check for message_delta event with stop_reason if event_type == "message_delta" and data_line: - try: - data: _AnthropicSSEEvent = json.loads(data_line) - delta: _AnthropicSSEDelta = data.get("delta", {}) - stop_reason: str | None = delta.get("stop_reason") - if stop_reason is not None: - return True - except json.JSONDecodeError: - verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line) + data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line) + delta: _SSEStopDelta = {} if data is None else data.get("delta", {}) + stop_reason: object = delta.get("stop_reason") + if stop_reason is not None: + return True except Exception as e: verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e) diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 8bb5d3e6b8d..421898e5765 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -1,7 +1,12 @@ -from typing import TYPE_CHECKING, Any, Final +import math +from collections.abc import Mapping +from typing import TYPE_CHECKING, Annotated, Final, TypeAlias import httpx +from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( @@ -18,6 +23,67 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float) + + +def _token_count_or_none(value: object) -> int | float | None: + if value is None: + return None + try: + count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value) + except ValidationError: + verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent") + return None + if not math.isfinite(count): + verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent") + return None + return count + + +_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)] + + +@with_config(ConfigDict(extra="allow")) +class _MAIImageUsageDetails(TypedDict, total=False): + text_tokens: ReadOnly[NotRequired[_TokenCount]] + image_tokens: ReadOnly[NotRequired[_TokenCount]] + + +@with_config(ConfigDict(extra="allow")) +class _MAIImageUsage(TypedDict, total=False): + input_tokens_details: ReadOnly[NotRequired[object]] + num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]] + num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]] + input_tokens: ReadOnly[NotRequired[_TokenCount]] + output_tokens: ReadOnly[NotRequired[_TokenCount]] + num_output_tokens: ReadOnly[NotRequired[_TokenCount]] + output_image_tokens: ReadOnly[NotRequired[_TokenCount]] + total_tokens: ReadOnly[NotRequired[_TokenCount]] + + +@with_config(ConfigDict(extra="allow")) +class _MAIImageResponse(TypedDict, total=False): + usage: ReadOnly[NotRequired[object]] + + +_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails) +_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage) +_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None: + if usage is None: + return None + try: + return _MAI_RAW_USAGE_ADAPTER.validate_python(usage) + except ValidationError: + verbose_logger.warning("MAI image usage is not an object, counting it as absent") + return None + + +_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse) + + class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): """Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5).""" @@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): return "maiimage" in model_normalized @staticmethod - def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]: + def normalize_mai_image_usage(usage: object) -> Mapping[str, object]: """Map Azure MAI usage fields to OpenAI ImageUsage schema.""" - if usage is None: + raw_usage: Final = _raw_usage_or_none(usage) + if raw_usage is None: return { "input_tokens": 0, "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, @@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): "total_tokens": 0, } - normalized_usage: Final = dict(usage) - input_tokens_details = normalized_usage.get("input_tokens_details") - if not isinstance(input_tokens_details, dict): - input_tokens_details = {} + normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage) + input_tokens_details_value: Final = normalized_usage.get("input_tokens_details") + input_tokens_details: Final[_MAIImageUsageDetails] = ( + _MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value) + if isinstance(input_tokens_details_value, dict) + else {} + ) text_tokens = normalized_usage.get("num_input_text_tokens") if text_tokens is None: @@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if total_tokens is None: total_tokens = input_tokens + output_tokens - normalized_usage.update( - { - "input_tokens": input_tokens, - "input_tokens_details": { - "image_tokens": image_tokens, - "text_tokens": text_tokens, - }, - "output_tokens": output_tokens, - "total_tokens": total_tokens, - } - ) - return normalized_usage + return { + **raw_usage, + "input_tokens": input_tokens, + "input_tokens_details": { + "image_tokens": image_tokens, + "text_tokens": text_tokens, + }, + "output_tokens": output_tokens, + "total_tokens": total_tokens, + } def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: return ["n", "size"] @@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): json_mode: bool | None = None, ) -> ImageResponse: try: - response: Final = raw_response.json() + response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json()) except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) - if "usage" in response: - response["usage"] = self.normalize_mai_image_usage(response.get("usage")) + response: Final = ( + { + **response_json, + "usage": self.normalize_mai_image_usage(response_json["usage"]), + } + if "usage" in response_json + else dict(response_json) + ) logging_obj.post_call( input=request_data.get("prompt", ""), diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 2f0ed4d0e82..f15a512077a 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Final, get_args, overload import httpx -from pydantic import TypeAdapter +from pydantic import ConfigDict, TypeAdapter import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL @@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config from .cohere_transformation import BedrockCohereEmbeddingConfig from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return response.json() + return _JSON_OBJECT.validate_python(response.json()) async def _make_async_call( self, @@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM): except httpx.TimeoutException: raise BedrockError(status_code=408, message="Timeout error occurred.") - return response.json() + return _JSON_OBJECT.validate_python(response.json()) def _transform_response( self, @@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM): # Parse response if response.status_code == 200: - return response.json() + return _JSON_OBJECT.validate_python(response.json()) else: raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}") diff --git a/litellm/llms/edenai/audio_transcription/transformation.py b/litellm/llms/edenai/audio_transcription/transformation.py index ee574a4f406..9ff530389e2 100644 --- a/litellm/llms/edenai/audio_transcription/transformation.py +++ b/litellm/llms/edenai/audio_transcription/transformation.py @@ -9,6 +9,7 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params @@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]: # mutable-ok: httpx form data """LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a @@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse: if "application/json" not in raw_response.headers.get("content-type", ""): return TranscriptionResponse(text=raw_response.text) - body: Final = raw_response.json() + body: Final = _JSON_OBJECT.validate_python(raw_response.json()) response: Final[TranscriptionResponse] = convert_to_model_response_object( response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription" ) diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index c3a826616ed..c29fe6a7d4f 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Final import httpx +from pydantic import ConfigDict, TypeAdapter from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -9,6 +10,8 @@ from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams from litellm.types.utils import ImageResponse from litellm.utils import convert_to_model_response_object +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + if TYPE_CHECKING: from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj @@ -66,7 +69,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: - response: Final = raw_response.json() + response: Final = _JSON_OBJECT.validate_python(raw_response.json()) stringified_response: Final = response ## LOGGING diff --git a/litellm/proxy/auth/oauth2_check.py b/litellm/proxy/auth/oauth2_check.py index 970f046c3d6..dc724e7afbe 100644 --- a/litellm/proxy/auth/oauth2_check.py +++ b/litellm/proxy/auth/oauth2_check.py @@ -3,6 +3,7 @@ import os from typing import Final, cast import httpx +from pydantic import ConfigDict, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.llms.custom_httpx.http_handler import ( @@ -11,6 +12,8 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +_TOKEN_INFO_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class OAuth2TokenValidationError(ValueError): """The token-info or introspection endpoint did not vouch for the token: it rejected it, reported it @@ -184,7 +187,7 @@ class Oauth2Handler: response.raise_for_status() # If we get here, the request was successful - data = response.json() + data = _TOKEN_INFO_RESPONSE.validate_python(response.json()) verbose_proxy_logger.debug( "Oauth2 token validation for token=%s, response from endpoint=%s", diff --git a/litellm/proxy/client/cli/commands/models.py b/litellm/proxy/client/cli/commands/models.py index ac7d5ee45ea..538b42a8c83 100644 --- a/litellm/proxy/client/cli/commands/models.py +++ b/litellm/proxy/client/cli/commands/models.py @@ -1,7 +1,7 @@ # stdlib imports import re from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal @@ -61,6 +61,10 @@ def _get_model_info_obj_from_yaml(model: dict[str, Any]) -> ModelYamlInfo: ) +def _str_field(model: Mapping[str, object], key: str, default: str) -> str: + return str(model.get(key, default)) + + def format_iso_datetime_str(iso_datetime_str: str | None) -> str: """Format an ISO format datetime string to human-readable date with minute resolution.""" if not iso_datetime_str: @@ -142,10 +146,10 @@ def list_models(ctx: click.Context, output_format: Literal["table", "json"]) -> created = int(created) table.add_row( - str(model.get("id", "")), - str(model.get("object", "model")), + _str_field(model, "id", ""), + _str_field(model, "object", "model"), (format_timestamp(created) if isinstance(created, int) else format_iso_datetime_str(created)), - str(model.get("owned_by", "")), + _str_field(model, "owned_by", ""), ) rich.print(table) diff --git a/litellm/proxy/client/users.py b/litellm/proxy/client/users.py index 7b4f36dbc2f..11a32896a99 100644 --- a/litellm/proxy/client/users.py +++ b/litellm/proxy/client/users.py @@ -2,9 +2,13 @@ from collections.abc import Iterable, Mapping from typing import Any, Final import requests +from pydantic import ConfigDict, TypeAdapter from .exceptions import NotFoundError, UnauthorizedError +_JSON_VALUE: Final = TypeAdapter(object, config=ConfigDict(strict=True, hide_input_in_errors=True)) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)) + class UsersManagementClient: def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30): @@ -39,7 +43,7 @@ class UsersManagementClient: if response.status_code == 404: raise NotFoundError(response.text) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) def get_user_v2(self, user_id: str | None = None) -> dict[str, object]: """Get user info v2 - lightweight, returns only user object (GET /v2/user/info)""" @@ -51,7 +55,7 @@ class UsersManagementClient: if response.status_code == 404: raise NotFoundError(response.text) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) def create_user(self, user_data: Mapping[str, object]) -> dict[str, object]: """Create a new user (POST /user/new)""" @@ -60,9 +64,9 @@ class UsersManagementClient: if response.status_code == 401: raise UnauthorizedError(response.text) response.raise_for_status() - return response.json() + return _JSON_OBJECT.validate_python(response.json()) - def delete_user(self, user_ids: list[str]) -> dict[str, object]: + def delete_user(self, user_ids: list[str]) -> object: """Delete users (POST /user/delete)""" url: Final = f"{self.base_url}/user/delete" response: Final = requests.post( @@ -71,4 +75,4 @@ class UsersManagementClient: if response.status_code == 401: raise UnauthorizedError(response.text) response.raise_for_status() - return response.json() + return _JSON_VALUE.validate_python(response.json()) diff --git a/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py b/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py index af4a59efb06..fe61f8c5b01 100644 --- a/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py +++ b/litellm/proxy/example_config_yaml/team_metadata_validator_e2e.py @@ -16,6 +16,10 @@ unknown impl value raises, which the proxy converts to the fail-closed 503. """ import os +from typing import Final + +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy.management_helpers.team_metadata_validation import ( @@ -23,12 +27,20 @@ from litellm.proxy.management_helpers.team_metadata_validation import ( TeamMetadataValidationResult, ) from litellm.types.llms.custom_http import httpxSpecialProvider -from typing import Final ALLOWED_COST_CENTERS: Final = frozenset({"CC-1001", "CC-1002"}) CLOSED_PORT_URL: Final = "http://127.0.0.1:9/validate" +@with_config(ConfigDict(extra="allow", strict=True)) +class _CostCenterReply(TypedDict): + ok: NotRequired[ReadOnly[object]] + reason: NotRequired[ReadOnly[str | None]] + + +_COST_CENTER_REPLY: Final = TypeAdapter(_CostCenterReply) + + async def _validate_allowlist(payload: TeamMetadataValidationPayload) -> TeamMetadataValidationResult: cost_center: Final = payload.metadata.get("cost_center") if cost_center is None: @@ -54,7 +66,7 @@ async def _validate_via_http(payload: TeamMetadataValidationPayload, service_url }, timeout=2.0, ) - body: Final = response.json() + body: Final = _COST_CENTER_REPLY.validate_python(response.json()) if body.get("ok") is True: return TeamMetadataValidationResult(valid=True) return TeamMetadataValidationResult( diff --git a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py index 611738ede8a..701b7bbb3c6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/vigil_guard/vigil_guard.py @@ -300,7 +300,7 @@ class VigilGuardGuardrail(CustomGuardrail): def _build_output( inputs: GenericGuardrailAPIInputs, final_texts: list[str], - final_tool_calls: Any, + final_tool_calls: _ToolCalls | None, ) -> GenericGuardrailAPIInputs: # When nothing was changed, return the input shape verbatim so the guardrail # logs "allow" rather than "mask". When a text or a tool-call argument was @@ -315,7 +315,7 @@ class VigilGuardGuardrail(CustomGuardrail): guardrailed["images"] = inputs["images"] if "tools" in inputs: guardrailed["tools"] = inputs["tools"] - if tool_calls_changed: + if tool_calls_changed and final_tool_calls is not None: guardrailed["tool_calls"] = final_tool_calls return guardrailed diff --git a/litellm/proxy/hooks/max_budget_per_session_limiter.py b/litellm/proxy/hooks/max_budget_per_session_limiter.py index cf3025d8c7b..002aeb145a5 100644 --- a/litellm/proxy/hooks/max_budget_per_session_limiter.py +++ b/litellm/proxy/hooks/max_budget_per_session_limiter.py @@ -18,6 +18,8 @@ import logging import os from typing import TYPE_CHECKING, Any, Final +from pydantic import ConfigDict, TypeAdapter + from litellm import DualCache from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger @@ -36,6 +38,14 @@ else: InternalUsageCache = Any +_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float]] = TypeAdapter( + str | bytes | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True) +) +_OPTIONAL_CACHE_AMOUNT_ADAPTER: Final[TypeAdapter[str | bytes | int | float | None]] = TypeAdapter( + str | bytes | int | float | None, config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + # Redis Lua script for atomic float increment with TTL. # INCRBYFLOAT returns the new value as a string. # Only sets EXPIRE on first call (when prior value was nil). @@ -216,9 +226,11 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): """Read current accumulated spend for a session.""" if self.internal_usage_cache.dual_cache.redis_cache is not None: try: - result = await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) - if result is not None: - return float(result) + redis_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( + await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(key=cache_key) + ) + if redis_result is not None: + return float(redis_result) return 0.0 except Exception as e: log_redis_failure( @@ -228,24 +240,28 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): e, ) - result = await self.internal_usage_cache.async_get_cache( - key=cache_key, - litellm_parent_otel_span=None, - local_only=True, + cache_result: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( + await self.internal_usage_cache.async_get_cache( + key=cache_key, + litellm_parent_otel_span=None, + local_only=True, + ) ) - if result is not None: - return float(result) + if cache_result is not None: + return float(cache_result) return 0.0 async def _increment_spend(self, cache_key: str, amount: float) -> float: """Atomically increment the session spend and return the new value.""" if self.increment_script is not None: try: - result: Final = await self.increment_script( - keys=[cache_key], - args=[str(amount), self.ttl], + script_result: Final[str | bytes | int | float] = _CACHE_AMOUNT_ADAPTER.validate_python( + await self.increment_script( + keys=[cache_key], + args=[str(amount), self.ttl], + ) ) - return float(result) + return float(script_result) except Exception as e: log_redis_failure( verbose_proxy_logger, @@ -257,12 +273,15 @@ class _PROXY_MaxBudgetPerSessionHandler(CustomLogger): return await self._in_memory_increment_spend(cache_key, amount) async def _in_memory_increment_spend(self, cache_key: str, amount: float) -> float: - current: Final = await self.internal_usage_cache.async_get_cache( - key=cache_key, - litellm_parent_otel_span=None, - local_only=True, + current: Final[str | bytes | int | float | None] = _OPTIONAL_CACHE_AMOUNT_ADAPTER.validate_python( + await self.internal_usage_cache.async_get_cache( + key=cache_key, + litellm_parent_otel_span=None, + local_only=True, + ) ) - new_value: Final = (float(current) if current is not None else 0.0) + amount + current_spend: Final = float(current) if current is not None else 0.0 + new_value: Final = current_spend + amount await self.internal_usage_cache.async_set_cache( key=cache_key, value=new_value, diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index bb901f22649..68287432492 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -22,7 +22,7 @@ from typing import Any, Final, Literal, Protocol, cast, overload import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, TypedDict import litellm @@ -124,6 +124,20 @@ _USER_MODEL_BUDGET_ADAPTER: Final = TypeAdapter(dict[str, float | BudgetConfig]) _USER_CACHE_INVALIDATION_BATCH_SIZE: Final = 50 +@with_config(ConfigDict(extra="allow", strict=True)) +class _AdminTeamAndKeyRows(TypedDict): + keys: ReadOnly[Sequence[object] | None] + teams: ReadOnly[Sequence[object] | None] + + +_ADMIN_TEAM_AND_KEY_ROWS_ADAPTER: Final = TypeAdapter( + list[_AdminTeamAndKeyRows], config=ConfigDict(hide_input_in_errors=True) +) +_ADMIN_KEY_ROW_ADAPTER: Final = TypeAdapter( + dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True) +) + + def _user_table( prisma_client: "PrismaClient | None", ) -> "TableActions[prisma_models.LiteLLM_UserTable]": @@ -1170,8 +1184,8 @@ async def user_info_v2( async def _fetch_admin_teams_and_keys_rows( prisma_client: "PrismaClient", sql_query: str -) -> Sequence[Mapping[str, Sequence[Mapping[str, object]] | None]]: - return await prisma_client.db.query_raw(sql_query) +) -> Sequence[_AdminTeamAndKeyRows]: + return _ADMIN_TEAM_AND_KEY_ROWS_ADAPTER.validate_python(await prisma_client.db.query_raw(sql_query)) async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth): @@ -1201,17 +1215,17 @@ async def _get_user_info_for_proxy_admin(user_api_key_dict: UserAPIKeyAuth): verbose_proxy_logger.debug("results_keys: %s", results) - _keys_in_db: Final[Sequence[Mapping[str, object]]] = results[0]["keys"] or [] + _keys_in_db: Final[Sequence[object]] = results[0]["keys"] or [] # cast all keys to LiteLLM_VerificationToken keys_in_db: Final = [] for key in _keys_in_db: - key_payload = dict[str, object](key) + key_payload = _ADMIN_KEY_ROW_ADAPTER.validate_python(key) if key_payload.get("models") is None: key_payload["models"] = [] keys_in_db.append(LiteLLM_VerificationToken.model_validate(key_payload)) # cast all teams to LiteLLM_TeamTable - _teams_rows: Final[Sequence[Mapping[str, object]]] = results[0]["teams"] or [] + _teams_rows: Final[Sequence[object]] = results[0]["teams"] or [] _teams_in_db: Final = sorted( (LiteLLM_TeamTable.model_validate(team) for team in _teams_rows), key=lambda x: getattr(x, "team_alias", "") or "", diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 9c0f3c4b81c..9dbf6bcc43d 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -3,11 +3,13 @@ import json import time from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status +from pydantic import ConfigDict, TypeAdapter, with_config +from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth @@ -44,6 +46,20 @@ _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" _ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +@with_config(ConfigDict(strict=True, extra="allow")) +class _RealtimeTokenPayload(TypedDict, total=False): + v: NotRequired[ReadOnly[object]] + ephemeral_key: NotRequired[ReadOnly[str]] + model_id: NotRequired[ReadOnly[str]] + user_id: NotRequired[ReadOnly[str | None]] + team_id: NotRequired[ReadOnly[str | None]] + expires_at: NotRequired[ReadOnly[object]] + session_type: NotRequired[ReadOnly[str | None]] + + +_REALTIME_TOKEN_PAYLOAD_ADAPTER: Final = TypeAdapter(_RealtimeTokenPayload) + + def _coerce_realtime_session_type(session_type: str | None) -> str: if session_type in _ALLOWED_SESSION_TYPES: return session_type @@ -201,17 +217,14 @@ def _header_value(headers: Mapping[str, str], name: str, default: str) -> str: def _decode_realtime_token_payload( decrypted_value: str, -) -> dict[str, Any] | None: +) -> _RealtimeTokenPayload | None: """ Decode realtime token payload; returns None for legacy/raw ephemeral tokens. """ try: - decoded: Final = json.loads(decrypted_value) + decoded: Final = _REALTIME_TOKEN_PAYLOAD_ADAPTER.validate_python(json.loads(decrypted_value)) except Exception: return None - - if not isinstance(decoded, dict): - return None if decoded.get("v") != _REALTIME_TOKEN_VERSION: return None if not isinstance(decoded.get("ephemeral_key"), str): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py b/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py index ae2df8069a9..70d83eeb840 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/dynamoai.py @@ -2,9 +2,10 @@ import enum from collections.abc import Mapping -from typing import Any, Literal, TypedDict +from typing import Literal, TypedDict from pydantic import Field +from typing_extensions import ReadOnly from .base import GuardrailConfigModel @@ -42,7 +43,7 @@ class PolicyApplicableTo(str, enum.Enum): class DynamoAIRequest(TypedDict, total=False): """Request structure for DynamoAI /moderation/analyze endpoint""" - messages: list[dict[str, Any]] + messages: ReadOnly[list[dict[str, object]]] textType: DynamoTextType | None policyIds: list[str] modelId: str | None @@ -58,8 +59,8 @@ class PolicyInfo(TypedDict, total=False): description: str method: PolicyMethod action: Literal["BLOCK", "WARN", "REDACT", "SANITIZE", "NONE"] - methodParams: dict[str, Any] - decisionParams: dict[str, Any] + methodParams: ReadOnly[dict[str, object]] + decisionParams: ReadOnly[dict[str, object]] applicableTo: PolicyApplicableTo created_at: str creatorId: str @@ -76,7 +77,7 @@ class AppliedPolicyDto(TypedDict, total=False): """Applied policy details from DynamoAI response""" policy: PolicyInfo - outputs: dict[str, Any] | None + outputs: ReadOnly[dict[str, object] | None] action: str | None diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py index 992815bf454..30b0fb80970 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py @@ -89,7 +89,7 @@ class NomaV2StreamingParams(BaseModel): cls, value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo ) -> object: try: - validated: Final = handler(value) + validated: Final[object] = handler(value) except ValidationError: verbose_proxy_logger.warning( "Noma v2 guardrail %s: ignoring %s=%.100r, which is not a valid value; the default applies", diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index 6d1ab85012b..ffdcf53fd7e 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -4,7 +4,8 @@ https://docs.cohere.com/reference/rerank """ -from typing import Literal +from collections.abc import Sequence +from typing import Literal, overload from pydantic import ConfigDict, PrivateAttr from typing_extensions import ReadOnly, Required, TypedDict @@ -95,11 +96,35 @@ class RerankResponse(LiteLLMBaseModel): def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params - def __getitem__(self, key): + @overload + def __getitem__(self, key: Literal["id"]) -> str | None: ... + + @overload + def __getitem__(self, key: Literal["results"]) -> list[RerankResponseResult] | None: ... + + @overload + def __getitem__(self, key: Literal["meta"]) -> RerankResponseMeta | None: ... + + @overload + def __getitem__(self, key: str) -> str | Sequence[RerankResponseResult] | RerankResponseMeta | None: ... + + def __getitem__(self, key: str) -> str | Sequence[RerankResponseResult] | RerankResponseMeta | None: return self.__dict__[key] - def get(self, key, default=None): + @overload + def get(self, key: Literal["id"], default: None = None) -> str | None: ... + + @overload + def get(self, key: Literal["results"], default: None = None) -> list[RerankResponseResult] | None: ... + + @overload + def get(self, key: Literal["meta"], default: None = None) -> RerankResponseMeta | None: ... + + @overload + def get(self, key: str, default: object = None) -> object: ... + + def get(self, key: str, default: object = None) -> object: return self.__dict__.get(key, default) - def __contains__(self, key) -> bool: + def __contains__(self, key: str) -> bool: return key in self.__dict__ diff --git a/tests/unit/integrations/arize/test_arize_phoenix.py b/tests/unit/integrations/arize/test_arize_phoenix.py index eb9e1326ced..5edf6e6d379 100644 --- a/tests/unit/integrations/arize/test_arize_phoenix.py +++ b/tests/unit/integrations/arize/test_arize_phoenix.py @@ -971,6 +971,23 @@ def test_arize_phoenix_client_get_prompt_version_rejects_traversal(): client.get_prompt_version("../../projects") +def test_arize_phoenix_client_error_does_not_echo_prompt_content(): + import httpx + import respx + + from litellm.integrations.arize.arize_phoenix_client import ArizePhoenixClient + + client = ArizePhoenixClient(api_key="test-key", api_base="https://phoenix.test") + with respx.mock: + respx.get("https://phoenix.test/v1/prompt_versions/pv-1").mock( + return_value=httpx.Response(200, json={"data": "secret system prompt text"}) + ) + with pytest.raises(Exception, match="Error fetching prompt version 'pv-1'") as exc_info: + client.get_prompt_version("pv-1") + + assert "secret system prompt text" not in str(exc_info.value) + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) diff --git a/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py b/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py index dd7e8fadcd8..01eff7f6bbf 100644 --- a/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py +++ b/tests/unit/llms/a2a/chat/guardrail_translation/test_a2a_guardrail_handler.py @@ -33,3 +33,7 @@ class TestA2AGuardrailHandlerStreamingScanKey: def test_unparseable_items_are_ignored(self): key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hi"), "not json", b"bytes"]) assert key.texts == ("hi",) + + def test_json_items_that_are_not_objects_are_ignored(self): + key = A2AGuardrailHandler().get_streaming_scan_key([_text_event("hi"), "[1, 2]", "5"]) + assert key.texts == ("hi",) diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 9e88b59adc9..141ad8a0481 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -302,6 +302,57 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: texts.append(data["delta"]["text"]) return texts + @pytest.mark.parametrize( + "off_schema_delta", + [None, "stray", {"type": "text_delta", "text": 5}, {"type": "thinking_delta", "text": None}], + ) + def test_off_schema_delta_is_skipped_without_ending_the_text_scan(self, off_schema_delta): + events = [ + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "abc"}}, + {"type": "content_block_delta", "index": 1, "delta": off_schema_delta}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "def"}}, + ] + chunk = "".join(f"event: content_block_delta\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + assert AnthropicMessagesHandler().get_streaming_string_so_far([chunk]) == "abcdef" + + def test_malformed_delta_data_is_skipped_without_ending_the_text_scan(self): + text_delta = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "abc"}} + chunk = ( + f"event: content_block_delta\ndata: {json.dumps(text_delta)}\n\n" + "event: content_block_delta\ndata: {not json\n\n" + f"event: content_block_delta\ndata: {json.dumps(text_delta)}\n\n" + ).encode() + + assert AnthropicMessagesHandler().get_streaming_string_so_far([chunk]) == "abcabc" + + def test_text_delta_with_a_lone_surrogate_escape_is_scanned(self): + event = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "pineapple \ud83d"}} + chunk = f"event: content_block_delta\ndata: {json.dumps(event)}\n\n".encode() + + assert AnthropicMessagesHandler().get_streaming_string_so_far([chunk]) == "pineapple \ud83d" + + def test_message_delta_with_a_lone_surrogate_escape_still_ends_the_stream(self): + event = {"type": "message_delta", "delta": {"stop_reason": "stop_sequence", "stop_sequence": "\ud83d"}} + chunk = f"event: message_delta\ndata: {json.dumps(event)}\n\n".encode() + + scan_key = AnthropicMessagesHandler().get_streaming_scan_key([chunk]) + + assert scan_key is not None + assert scan_key.stream_ended is True + + def test_off_schema_message_delta_does_not_hide_a_later_stop_reason(self): + events = [ + {"type": "message_delta", "delta": None}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}}, + ] + chunk = "".join(f"event: message_delta\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + scan_key = AnthropicMessagesHandler().get_streaming_scan_key([chunk]) + + assert scan_key is not None + assert scan_key.stream_ended is True + @pytest.mark.asyncio async def test_deliver_ended_stream_rewrites_writes_text_back_into_sse_chunks(self): handler = AnthropicMessagesHandler() diff --git a/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py b/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py index 27e78d35c69..e702abf196d 100644 --- a/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py +++ b/tests/unit/llms/azure_ai/image_generation/test_mai_image_generation.py @@ -386,6 +386,76 @@ class TestAzureMAIImageGeneration: assert normalized["input_tokens"] == 22 assert normalized["total_tokens"] == 22 + def test_normalize_mai_usage_accepts_a_bool_token_count(self): + normalized = AzureFoundryMAIImageGenerationConfig().normalize_mai_image_usage( + {"input_tokens": 22, "output_tokens": True} + ) + + assert normalized["output_tokens"] == 1 + assert normalized["total_tokens"] == 23 + + def test_normalize_mai_usage_reads_string_token_counts_as_numbers(self): + normalized = AzureFoundryMAIImageGenerationConfig().normalize_mai_image_usage( + {"input_tokens": "8", "output_tokens": "1024"} + ) + + assert normalized["input_tokens"] == 8 + assert normalized["total_tokens"] == 1032 + + @pytest.mark.parametrize("unreadable", ["n/a", "inf", "nan", float("inf")]) + def test_normalize_mai_usage_falls_back_past_an_unreadable_count(self, unreadable): + normalized = AzureFoundryMAIImageGenerationConfig().normalize_mai_image_usage( + {"input_tokens": 8, "output_tokens": unreadable, "num_output_tokens": 1024} + ) + + assert normalized["output_tokens"] == 1024 + assert normalized["total_tokens"] == 1032 + + @pytest.mark.parametrize("unreadable", ["n/a", {"n": 1}, [1024]]) + def test_transform_image_generation_response_keeps_the_image_when_a_count_is_unreadable(self, unreadable): + raw_response = MagicMock(spec=httpx.Response) + raw_response.json.return_value = { + "created": 1780897477, + "data": [{"b64_json": "abc123"}], + "usage": {"input_tokens": 8, "output_tokens": 1024, "total_tokens": 1032, "num_output_tokens": unreadable}, + } + + image_response = AzureFoundryMAIImageGenerationConfig().transform_image_generation_response( + model="MAI-Image-2.5", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A red fox"}, + optional_params={"width": 1024, "height": 1024}, + litellm_params={}, + encoding=None, + ) + + assert image_response.data[0].b64_json == "abc123" + assert image_response.usage.output_tokens == 1024 + assert image_response.usage.total_tokens == 1032 + + @pytest.mark.parametrize("usage", [[], "n/a", 5]) + def test_transform_image_generation_response_keeps_the_image_when_usage_is_not_an_object(self, usage): + raw_response = MagicMock(spec=httpx.Response) + raw_response.json.return_value = {"created": 1780897477, "data": [{"b64_json": "abc123"}], "usage": usage} + + image_response = AzureFoundryMAIImageGenerationConfig().transform_image_generation_response( + model="MAI-Image-2.5", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A red fox"}, + optional_params={"width": 1024, "height": 1024}, + litellm_params={}, + encoding=None, + ) + + assert image_response.data[0].b64_json == "abc123" + assert image_response.usage.input_tokens == 0 + assert image_response.usage.output_tokens == 0 + assert image_response.usage.total_tokens == 0 + def test_azure_sync_image_generation_uses_mai_response_transform(self): raw_response = MagicMock(spec=httpx.Response) raw_response.json.return_value = { diff --git a/tests/unit/proxy/client/test_users.py b/tests/unit/proxy/client/test_users.py index 5b4d89420ab..a936de77c3e 100644 --- a/tests/unit/proxy/client/test_users.py +++ b/tests/unit/proxy/client/test_users.py @@ -5,7 +5,6 @@ import pytest import requests - from litellm.proxy.client.users import ( NotFoundError, UnauthorizedError, @@ -72,9 +71,9 @@ def test_create_user_unauthorized(mock_post, client): @patch("requests.post") def test_delete_user_success(mock_post, client): mock_post.return_value.status_code = 200 - mock_post.return_value.json.return_value = {"deleted": 1} + mock_post.return_value.json.return_value = 1 result = client.delete_user(["u1"]) - assert result["deleted"] == 1 + assert result == 1 mock_post.assert_called_once() diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 78a0a765fe6..42162087c9b 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4292,6 +4292,34 @@ async def test_get_user_info_for_proxy_admin_validates_keys_and_teams(): assert returned_key["models"] == [] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "raw_rows", + [ + [{"teams": None, "keys": "sk-row-content-must-not-leak"}], + [{"teams": None, "keys": ["sk-row-content-must-not-leak"]}], + ], +) +async def test_get_user_info_for_proxy_admin_errors_do_not_echo_row_content(raw_rows): + from unittest.mock import AsyncMock, MagicMock, patch + + from pydantic import ValidationError + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _get_user_info_for_proxy_admin, + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.query_raw = AsyncMock(return_value=raw_rows) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): + with pytest.raises(ValidationError) as exc_info: + await _get_user_info_for_proxy_admin(user_api_key_dict=UserAPIKeyAuth(user_id=None)) + + assert "sk-row-content-must-not-leak" not in str(exc_info.value) + + def _object_permission_mocks(mocker, existing_object_permission_id=None): """Prisma double whose user row optionally already links a permission row.""" mock_prisma_client = mocker.MagicMock() From fd594ab0cf743fb16d8aa8e50972fc788fadd44a Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 29 Oct 2026 19:00:04 -0700 Subject: [PATCH 20/25] fix: enforce LiteAdmin MCP route permissions (#48080) * fix(admin-mcp): expose and enforce credential route permissions * fix: restore beta header and telemetry test parity * fix(admin-mcp): align permissions with registered HTTP routes * test(admin-mcp): verify encoded paths through HTTP authorization * test(admin-mcp): simplify HTTP permission regression fixtures --- .../src/providers/anthropic/beta.rs | 4 +- litellm/proxy/_types.py | 7 + litellm/proxy/admin_mcp.py | 96 ++++++++- litellm/proxy/auth/route_checks.py | 19 +- .../internal_user_endpoints.py | 27 ++- pyproject.toml | 2 +- tests/unit/proxy/test_admin_mcp.py | 186 +++++++++++++++++- tests/unit/proxy/test_component_allowlists.py | 5 +- tests/unit/proxy/test_proxy_server.py | 5 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 ++ uv.lock | 8 +- 11 files changed, 344 insertions(+), 29 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 8429c98a88a..80095cac36c 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -238,7 +238,8 @@ impl AnthropicBeta { Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 - | Self::ThinkingBindingControls20260801 => &[], + | Self::ThinkingBindingControls20260801 + | Self::ThinkingDisplayUpdates20260818 => &[], Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse], Self::Effort20251124 => &[BetaProvider::VertexAi], Self::FastMode20260201 | Self::Oauth20250420 => &[ @@ -275,7 +276,6 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi], - Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks], Self::ToolSearchTool20251019 => &[ BetaProvider::Anthropic, BetaProvider::AzureAi, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4047fa3f0ad..0a09fe4a855 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3177,11 +3177,18 @@ from litellm.types.proxy.auth.user_api_key_auth import ( ) +class AdminMCPPermissions(LiteLLMPydanticObjectBase): + model_config = ConfigDict(frozen=True) + tools: tuple[str, ...] + action_allowed: bool | None = None + + class UserInfoResponse(LiteLLMPydanticObjectBase): user_id: str | None user_info: dict | BaseModel | None keys: list teams: list + admin_mcp_permissions: AdminMCPPermissions | None = None class UserInfoV2Response(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/admin_mcp.py b/litellm/proxy/admin_mcp.py index 6371863f45c..32576ea8113 100644 --- a/litellm/proxy/admin_mcp.py +++ b/litellm/proxy/admin_mcp.py @@ -1,20 +1,21 @@ import os import re -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, Iterator, Mapping from contextlib import asynccontextmanager from contextvars import ContextVar from typing import Final -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException, Request +from fastapi.routing import APIRoute from pydantic import TypeAdapter from starlette.datastructures import Headers -from starlette.requests import Request from starlette.routing import Mount from starlette.types import ASGIApp, Receive, Scope, Send from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS -from litellm.proxy._types import SpecialHeaders +from litellm.proxy._types import AdminMCPPermissions, SpecialHeaders, UserAPIKeyAuth +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.middleware.admission_control_middleware import ADMISSION_LEASE_SCOPE_KEY _REQUEST_HEADERS: Final = frozenset( @@ -58,6 +59,91 @@ _RESERVED_KEY_HEADERS: Final = ( _SETTINGS: Final = TypeAdapter(Mapping[str, object]) _IDENTITY_MAPPINGS: Final = TypeAdapter(tuple[dict[str, object], ...] | dict[str, object] | None) _OAUTH_MAPPINGS: Final = TypeAdapter(dict[str, str]) +_ROUTE_GRANTS: Final = TypeAdapter(tuple[str, ...]) + + +def _admin_route_allowed(auth: UserAPIKeyAuth, request: Request, method: str, path: str) -> bool: + target: Final = Request({**request.scope, "method": method, "path": path, "raw_path": path.encode()}) + try: + RouteChecks.should_call_route(route=path, valid_token=auth, request=target) + except HTTPException: + return False + return True + + +def _admin_routes(request: Request) -> Iterator[tuple[tuple[str, str], APIRoute]]: + app: Final = request.app + if not isinstance(app, FastAPI): + return + for route in app.routes: + if not isinstance(route, APIRoute): + continue + for method in route.methods: + yield (method, route.path_format), route + + +def _admin_grant_paths(route: APIRoute, grant: str) -> Iterator[str]: + if route.path_regex.fullmatch(grant) is not None: + yield grant + prefix: Final = grant[:-1] if grant.endswith("*") else grant + "/" + representative: Final = re.sub(r"\{[^}]+\}", "_", route.path_format) + # A matching suffix supplies the rest of a concrete route for prefix grants, + # including grants ending inside a static segment or a path parameter. + for candidate in (prefix + representative[offset:] for offset in range(len(representative) + 1)): + if route.path_regex.fullmatch(candidate) is not None: + yield candidate + + +def _admin_grant_allowed(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute, grant: str) -> bool: + return any(_admin_route_allowed(auth, request, method, path) for path in _admin_grant_paths(route, grant)) + + +def _admin_tool_visible(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute) -> bool: + try: + RouteChecks.check_route_enabled(route.path) + except HTTPException: + return False + if _admin_route_allowed(auth, request, method, route.path): + return True + grants: Final = _ROUTE_GRANTS.validate_python(getattr(auth, "allowed_routes", None) or ()) + return bool(route.param_convertors) and any( + _admin_grant_allowed(auth, request, method, route, grant) for grant in grants if grant.startswith("/") + ) + + +def get_admin_mcp_permissions( + auth: UserAPIKeyAuth, request: Request, tool: str | None = None, path: str | None = None +) -> AdminMCPPermissions: + try: + from litellm_admin_mcp.catalog import ( # pyright: ignore[reportMissingTypeStubs] # connector has no py.typed marker + BY_NAME, + OPERATIONS, + ) + except ImportError as exc: + raise HTTPException( + status_code=503, detail="Admin MCP permissions require the admin-mcp dependency group (Python 3.12+)" + ) from exc + + routes: Final = dict(_admin_routes(request)) + allowed: Final = tuple( + operation.name + for operation in OPERATIONS + if (route := routes.get((operation.method, operation.path))) is not None + and _admin_tool_visible(auth, request, operation.method, route) + ) + if tool is None: + return AdminMCPPermissions(tools=allowed) + operation: Final = BY_NAME.get(tool) + if operation is None or tool not in allowed: + return AdminMCPPermissions(tools=allowed, action_allowed=False) + if path is None: + return AdminMCPPermissions(tools=allowed, action_allowed=True) + registered: Final = routes[(operation.method, operation.path)] + decoded_path: Final = unquote(path) + action_allowed: Final = registered.path_regex.fullmatch(decoded_path) is not None and _admin_route_allowed( + auth, request, operation.method, decoded_path + ) + return AdminMCPPermissions(tools=allowed, action_allowed=action_allowed) def _configured_key_header() -> bytes | None: diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 105acb237af..125bfab4edf 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -111,14 +111,7 @@ _TOKEN_COUNTING_ACTION: Final = "countTokens" class RouteChecks: @staticmethod - def should_call_route( - route: str, - valid_token: UserAPIKeyAuth, - request: Request | None = None, - ): - """ - Check if management route is disabled and raise exception - """ + def check_route_enabled(route: str) -> None: try: from litellm_enterprise.proxy.auth.route_checks import EnterpriseRouteChecks @@ -128,6 +121,16 @@ class RouteChecks: except Exception: pass + @staticmethod + def should_call_route( + route: str, + valid_token: UserAPIKeyAuth, + request: Request | None = None, + ): + """ + Check if management route is disabled and raise exception + """ + RouteChecks.check_route_enabled(route) # Check if Virtual Key is allowed to call the route - Applies to all Roles RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token, request=request) return True diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 68287432492..6ceab27befa 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -945,6 +945,14 @@ async def user_info( request: Request, user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + admin_mcp: bool = fastapi.Query( + default=False, description="Include the caller's current admin MCP route permissions" + ), + mcp_tool: str | None = fastapi.Query(default=None, description="Admin MCP tool to authorize for this caller"), + mcp_path: str | None = fastapi.Query( + default=None, + description="URL-encoded concrete management path used by the MCP tool, before query-string encoding", + ), ): """ [10/07/2024] @@ -965,12 +973,23 @@ async def user_info( user_id = _normalize_user_info_user_id(request=request, user_id=user_id) _enforce_user_info_access(user_id=user_id, user_api_key_dict=user_api_key_dict) + from litellm.proxy.admin_mcp import get_admin_mcp_permissions + + mcp_permissions: Final = ( + get_admin_mcp_permissions(user_api_key_dict, request, mcp_tool, mcp_path) if admin_mcp is True else None + ) + if prisma_client is None: raise Exception( "Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" ) if user_id is None and user_api_key_has_admin_view(user_api_key_dict): - return await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict) + admin_response: Final = await _get_user_info_for_proxy_admin(user_api_key_dict=user_api_key_dict) + return ( + admin_response.model_copy(update={"admin_mcp_permissions": mcp_permissions}) + if mcp_permissions is not None + else admin_response + ) elif user_id is None: user_id = user_api_key_dict.user_id ## GET USER ROW ## @@ -1009,7 +1028,11 @@ async def user_info( ), ) - return response_data + return ( + response_data.model_copy(update={"admin_mcp_permissions": mcp_permissions}) + if mcp_permissions is not None + else response_data + ) except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info(): Exception occured - %s", e) raise handle_exception_on_proxy(e) diff --git a/pyproject.toml b/pyproject.toml index 6a21fd9b26b..6e9e75a5b7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -251,7 +251,7 @@ e2e-dev = [ "mcp>=2.3.0,<3", ] admin-mcp = [ - "litellm-admin-mcp @ https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz#sha256=86b930f6706fb2da10d53d1798ee7fd14e44fb0afdef0da122cf0e6bb88fa2c3 ; python_version >= '3.12'", + "litellm-admin-mcp @ https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz#sha256=5803a20a00a18177f4591706956f31be311879543049b8b38143ad90bb1cb0a6 ; python_version >= '3.12'", ] proxy-dev = [ { include-group = "admin-mcp" }, diff --git a/tests/unit/proxy/test_admin_mcp.py b/tests/unit/proxy/test_admin_mcp.py index dc956038525..4aa0226552f 100644 --- a/tests/unit/proxy/test_admin_mcp.py +++ b/tests/unit/proxy/test_admin_mcp.py @@ -1,6 +1,7 @@ import asyncio import json import sys +from types import SimpleNamespace from typing import Final import httpx2 @@ -12,11 +13,11 @@ from starlette.testclient import TestClient from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS from litellm.proxy import proxy_server -from litellm.proxy._types import LiteLLM_UserTable, ProxyException, SpecialHeaders -from litellm.proxy.admin_mcp import admin_mcp_lifespan -from litellm.proxy.auth.user_api_key_auth import get_api_key, get_api_key_from_custom_header +from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, ProxyException, SpecialHeaders, UserAPIKeyAuth +from litellm.proxy.admin_mcp import admin_mcp_lifespan, get_admin_mcp_permissions +from litellm.proxy.auth.user_api_key_auth import get_api_key, get_api_key_from_custom_header, user_api_key_auth from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache -from litellm.proxy.management_endpoints import internal_user_endpoints +from litellm.proxy.management_endpoints import internal_user_endpoints, team_callback_endpoints from litellm.proxy.middleware.admission_control_middleware import ( AdmissionControlMiddleware, AdmissionControlSettings, @@ -66,12 +67,22 @@ def management_app(monkeypatch: pytest.MonkeyPatch) -> FastAPI: if credential not in {"admin-a", "admin-b", "admin-b-limited", "member", "viewer"}: raise HTTPException(status_code=401) user_id: Final = "admin-b" if credential == "admin-b-limited" else credential + role: Final = {"member": LitellmUserRoles.INTERNAL_USER, "viewer": LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY}.get( + user_id, LitellmUserRoles.PROXY_ADMIN + ) + auth: Final = UserAPIKeyAuth( + user_id=user_id, user_role=role, + allowed_routes=["/user/info"] if credential == "admin-b-limited" else None, + ) return { "user_id": user_id, "user_info": { "user_id": user_id, - "user_role": {"member": "internal_user", "viewer": "proxy_admin_viewer"}.get(user_id, "proxy_admin"), + "user_role": role.value, }, + "admin_mcp_permissions": get_admin_mcp_permissions( + auth, request, request.query_params.get("mcp_tool"), request.query_params.get("mcp_path") + ).model_dump(mode="json"), } @app.get("/key/list", operation_id="list_keys_key_list_get") @@ -565,3 +576,168 @@ async def test_oauth2_proxy_auth_preserves_native_identity_and_peer_trust( assert response.status_code == status, response.text if status == 200: assert "list_keys" in {tool["name"] for tool in response.json()["result"]["tools"]} + + +@pytest.fixture +def user_info_http_app(monkeypatch: pytest.MonkeyPatch, grant: str) -> FastAPI: + class EmptyDatabase: + async def query_raw(self, _: str) -> tuple[dict[str, None], ...]: + return ({"teams": None, "keys": None},) + + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=EmptyDatabase())) + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", grant]) + app: Final = FastAPI() + app.include_router(internal_user_endpoints.router) + app.include_router(team_callback_endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: auth + return app + + +@pytest.fixture +def permissions_request() -> Request: + return Request({"type": "http", "method": "GET", "path": "/user/info", "headers": [], "app": proxy_server.app}) + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize( + "routes,session,can_write", + [ + (None, False, True), + ([], False, True), + (["management_routes"], False, True), + (["info_routes"], False, False), + (["llm_api_routes"], False, False), + (None, True, True), + (["info_routes"], True, False), + ], +) +def test_admin_mcp_uses_credential_routes_without_removing_full_access( + routes: list[str] | None, session: bool, can_write: bool, permissions_request: Request +) -> None: + auth: Final = UserAPIKeyAuth( + user_id="admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + allowed_routes=routes, + is_session_token=session, + ) + permissions: Final = get_admin_mcp_permissions(auth, permissions_request, "create_key", "/key/generate") + assert permissions.action_allowed is can_write + assert ("create_key" in permissions.tools) is can_write + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize( + "team_id,grant", + [ + ("team-a", "/team/team-a/callback"), + ("team-a", "/team/team-a/*"), + ("org/team-a", "/team/org/team-a/callback"), + ("org/team-a", "/team/org/team-a/*"), + ("org/team-a", "/team/org/team-a/call*"), + ], +) +def test_admin_mcp_checks_the_concrete_resource_even_when_the_tool_is_visible( + team_id: str, grant: str, permissions_request: Request +) -> None: + auth: Final = UserAPIKeyAuth( + user_id="admin", + user_role=LitellmUserRoles.PROXY_ADMIN, + allowed_routes=["/user/info", grant], + metadata={"is_session_token": True, "user_role": "proxy_admin"}, + ) + own: Final = get_admin_mcp_permissions(auth, permissions_request, "get_team_callbacks", f"/team/{team_id}/callback") + other: Final = get_admin_mcp_permissions(auth, permissions_request, "get_team_callbacks", "/team/other/callback") + wrong_route: Final = get_admin_mcp_permissions(auth, permissions_request, "get_team_callbacks", "/key/generate") + assert "get_team_callbacks" in own.tools + assert own.action_allowed is True + assert other.action_allowed is False + assert wrong_route.action_allowed is False + + +@pytest.mark.asyncio +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize( + "grant,path,allowed", + [ + ("/team/team a/callback", "/team/team%20a/callback", True), + ("/team/org/team-a/callback", "/team/org%2Fteam-a/callback", True), + ("/team/team%2Fa/callback", "/team/team%252Fa/callback", True), + ("/team/team+a/callback", "/team/team+a/callback", True), + ("/team/team-é/callback", "/team/team-%C3%A9/callback", True), + ("/team/team/a/callback", "/team/team%252Fa/callback", False), + ("/team/team a/callback", "/team/team+a/callback", False), + ("/team/team%41/callback", "/team/team%2541/callback", True), + ], +) +async def test_user_info_decodes_mcp_path_after_query_parsing( + user_info_http_app: FastAPI, + path: str, + allowed: bool, +) -> None: + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=user_info_http_app), + base_url="http://testserver", + params={"admin_mcp": "true", "mcp_tool": "get_team_callbacks"}, + ) as client: + own: Final = await client.get("/user/info", params={"mcp_path": path}) + sibling: Final = await client.get("/user/info", params={"mcp_path": "/team/other/callback"}) + assert own.status_code == 200, own.text + assert sibling.status_code == 200, sibling.text + assert "get_team_callbacks" in own.json()["admin_mcp_permissions"]["tools"] + assert own.json()["admin_mcp_permissions"]["action_allowed"] is allowed + assert sibling.json()["admin_mcp_permissions"]["action_allowed"] is False + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +@pytest.mark.parametrize("grant", ["/te?m/*", "/[t]eam/*", "/te*m/*"]) +def test_admin_mcp_does_not_invent_glob_permissions(grant: str, permissions_request: Request) -> None: + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", grant]) + permissions: Final = get_admin_mcp_permissions( + auth, permissions_request, "get_team_callbacks", "/team/team-a/callback" + ) + assert "get_team_callbacks" not in permissions.tools + assert permissions.action_allowed is False + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +def test_admin_mcp_discovery_respects_disabled_management_routes( + permissions_request: Request, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setenv("DISABLE_ADMIN_ENDPOINTS", "true") + auth: Final = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", "/key/key-a/reset_spend"] + ) + permissions: Final = get_admin_mcp_permissions( + auth, permissions_request, "reset_key_spend", "/key/key-a/reset_spend" + ) + assert permissions.tools == () + assert permissions.action_allowed is False + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="Admin MCP requires Python 3.12+") +def test_admin_mcp_preserves_http_prefix_grants(permissions_request: Request) -> None: + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, allowed_routes=["/user/info", "/key/list"]) + own: Final = get_admin_mcp_permissions(auth, permissions_request, "reset_key_spend", "/key/list/reset_spend") + other: Final = get_admin_mcp_permissions(auth, permissions_request, "reset_key_spend", "/key/other/reset_spend") + assert "reset_key_spend" in own.tools + assert own.action_allowed is True + assert other.action_allowed is False + + +@pytest.mark.asyncio +async def test_user_info_explains_missing_admin_mcp_dependency( + permissions_request: Request, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(sys.modules, "litellm_admin_mcp.catalog", None) + auth: Final = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(ProxyException, match="admin-mcp dependency group") as exc: + await internal_user_endpoints.user_info( + request=permissions_request, + user_id=None, + user_api_key_dict=auth, + admin_mcp=True, + mcp_tool=None, + mcp_path=None, + ) + assert exc.value.code == "503" diff --git a/tests/unit/proxy/test_component_allowlists.py b/tests/unit/proxy/test_component_allowlists.py index 4c0b3ad913b..738fe9d53df 100644 --- a/tests/unit/proxy/test_component_allowlists.py +++ b/tests/unit/proxy/test_component_allowlists.py @@ -468,7 +468,10 @@ def test_admin_mcp_survives_only_management_component_lifespans( @application.get("/user/info") async def user_info() -> dict[str, object]: - return {"user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}} + return { + "user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}, + "admin_mcp_permissions": {"tools": ["list_teams"], "action_allowed": True}, + } if component_lifespan is not None: application.router.lifespan_context = partial(component_lifespan, lifespan=application.router.lifespan_context) diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index c835e1e27ea..139cfbbe58c 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -147,7 +147,10 @@ async def test_proxy_shutdown_drains_active_admin_tool_before_closing_connector( @admin_mcp_proxy.get("/user/info") async def user_info() -> dict[str, object]: - return {"user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}} + return { + "user_id": "admin", "user_info": {"user_id": "admin", "user_role": "proxy_admin"}, + "admin_mcp_permissions": {"tools": ["list_teams"], "action_allowed": True}, + } @admin_mcp_proxy.get("/team/list", operation_id="list_team_team_list_get") async def list_teams() -> dict[str, object]: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90a0597cea5..13a064517a4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25710,6 +25710,13 @@ export interface components { */ source: "admin"; }; + /** AdminMCPPermissions */ + AdminMCPPermissions: { + /** Action Allowed */ + action_allowed?: boolean | null; + /** Tools */ + tools: string[]; + }; /** * AgentCapabilities * @description Defines optional capabilities supported by an agent. @@ -49920,6 +49927,7 @@ export interface components { }; /** UserInfoResponse */ UserInfoResponse: { + admin_mcp_permissions?: components["schemas"]["AdminMCPPermissions"] | null; /** Keys */ keys: unknown[]; /** Teams */ @@ -76732,6 +76740,12 @@ export interface operations { query?: { /** @description User ID in the request parameters */ user_id?: string | null; + /** @description Include the caller's current admin MCP route permissions */ + admin_mcp?: boolean; + /** @description Admin MCP tool to authorize for this caller */ + mcp_tool?: string | null; + /** @description URL-encoded concrete management path used by the MCP tool, before query-string encoding */ + mcp_path?: string | null; }; header?: never; path?: never; diff --git a/uv.lock b/uv.lock index 87b8fc95486..8cd31eb0828 100644 --- a/uv.lock +++ b/uv.lock @@ -5334,7 +5334,7 @@ requires-dist = [ provides-extras = ["proxy", "cli", "extra-proxy", "utils", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] -admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }] +admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" }] benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, @@ -5431,7 +5431,7 @@ proxy-dev = [ { name = "azure-identity", specifier = "==1.25.2" }, { name = "backports-zstd", marker = "python_full_version < '3.14'", specifier = "==1.8.0" }, { name = "hypercorn", specifier = "==0.17.3" }, - { name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }, + { name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" }, { name = "opentelemetry-api", specifier = "==1.45.1" }, { name = "opentelemetry-exporter-otlp", specifier = "==1.45.1" }, { name = "opentelemetry-instrumentation-fastapi", specifier = "==0.66b1" }, @@ -5444,14 +5444,14 @@ proxy-dev = [ [[package]] name = "litellm-admin-mcp" version = "0.1.0" -source = { url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" } +source = { url = "https://github.com/BerriAI/liteadmin-mcp/archive/31fb096904ae05684c0779110a93b135b5601182.tar.gz" } dependencies = [ { name = "httpx2" }, { name = "jsonschema" }, { name = "mcp" }, { name = "uvicorn" }, ] -sdist = { hash = "sha256:86b930f6706fb2da10d53d1798ee7fd14e44fb0afdef0da122cf0e6bb88fa2c3" } +sdist = { hash = "sha256:5803a20a00a18177f4591706956f31be311879543049b8b38143ad90bb1cb0a6" } [package.metadata] requires-dist = [ From 7aa0393cb67dc94c1c71a58ebb192f6d34d08fbb Mon Sep 17 00:00:00 2001 From: bap1106 Date: Fri, 30 Oct 2026 09:05:19 +0700 Subject: [PATCH 21/25] feat(providers): add CLF AI Gateway as an OpenAI-compatible provider with ten models (#39324) * feat(providers): add CLF AI Gateway (clf_ai_gateway), OpenAI-compatible, 9 models OpenAI-compatible gateway at https://api.clfaigateway.dev/v1 serving open-weight models (GLM, Kimi, DeepSeek, Qwen) on Cloudflare Workers AI upstream. Follows the DeepInfra pattern: ClfAiGatewayConfig(OpenAIGPTConfig), CLF_AI_GATEWAY_API_KEY / _API_BASE, provider registered in enum, provider lists, endpoint inference, lazy-import registry, ProviderConfigManager, validate_environment. Supported params deliberately omit functions/function_call/logit_bias (gateway returns 400 for them). 9 cost-map entries (root + backup) with measured context/output limits and currently billed prices; endpoint-support entry; README row. Mocked tests only. Co-Authored-By: Claude Fable 5 * fix(clf_ai_gateway): declare each model's real reasoning_effort levels The per-level supports_*_reasoning_effort flags cannot express the sets this gateway actually takes, so all nine models advertised a wrong one: every model offered "minimal" and eight offered "none", neither of which the gateway accepts, five dropped "xhigh", which it does accept, and qwen3.8-27b offered "high", which it rejects. That set surfaces through /model_group/info and the model picker, so a user picking a level off it gets a 400 from the gateway. Declare reasoning_effort_levels instead, taken from the gateway's own GET /v1/public/models, which resolve_supported_reasoning_efforts reads first and uses whole. Declaring it also stops clf_ai_gateway/deepseek-v4-pro and clf_ai_gateway/deepseek-v4-flash inheriting effort metadata from the unprefixed deepseek entries of the same name. * test(clf_ai_gateway): name the gateway-rejected params instead of commenting them * fix(clf_ai_gateway): list the provider in the proxy Add Model form main now requires every LlmProviders value to have an entry in provider_create_fields.json (or be frozen as unlisted), so the Add Model dropdown cannot silently miss a backend provider. Add the clf_ai_gateway entry in the same shape as the SCX.ai registration: required API key, optional API base with https://api.clfaigateway.dev/v1 as placeholder, and a cost-map model as the default model placeholder. A provider test pins the entry. Co-Authored-By: Claude Opus 5 * fix(clf_ai_gateway): sync prices and reasoning_effort levels with the live catalog CLF AI Gateway raised its launch discount from 40% to 60% off list on 2026-09-26. It also re-measured every model's reasoning_effort levels against the current Workers AI behaviour. - input/output/cached per-token prices for all 9 models now match GET https://api.clfaigateway.dev/v1/public/models to the nano; qwen3.8-27b cached input also follows the upstream cached rate published on 2026-09-25 - reasoning_effort_levels: add "max" to deepseek-v4-flash, deepseek-v4-pro, glm-5.2 and glm-5.3-flash; add "none" to kimi-k2.6 (it really disables thinking there); drop "none" from glm-5.3 (upstream runs it as "max") Context windows, output limits and vision flags are unchanged and were checked against the same endpoint. Co-Authored-By: Claude Opus 5.5 * fix(clf_ai_gateway): drop a rebind-ok that no longer suppresses anything After rebasing onto main, the type-discipline gate's new LIT013 rule flags the `# rebind-ok` on the endpoint-inference `dynamic_api_key` assignment. The rebind rule no longer fires on that line, so the suppression is inert. The sibling endpoint branches carry no comment there either. Co-Authored-By: Claude Opus 5.5 * test(clf_ai_gateway): expect the re-measured reasoning_effort levels Follows the catalog sync: max on deepseek-v4-flash, deepseek-v4-pro, glm-5.2 and glm-5.3-flash, none on kimi-k2.6, and no none on glm-5.3. Co-Authored-By: Claude Opus 5.5 * test(clf_ai_gateway): move the provider tests to tests/unit/llms main moved every per-provider test from tests/test_litellm/llms// to tests/unit/llms// (DeepInfra lives at tests/unit/llms/deepinfra/). This branch still added the old path, so no CI shard claimed it. That failed assert-ci-coverage and assert-shard-coverage, and the file never ran (hence the low codecov/patch). Co-Authored-By: Claude Opus 5.5 * test(clf_ai_gateway): assert routing, billing, and dropped params instead of pinned gateway facts * fix(clf_ai_gateway): keep the type-discipline gate at its merge-base count * chore(clf_ai_gateway): regenerate provider endpoint support * ci(clf_ai_gateway): run the provider tests in the llms unit shard * fix(clf_ai_gateway): drop the retired max_tokens field from the cost map rows * fix(clf_ai_gateway): forward params the gateway accepts, drop undeclared reasoning levels, add deepseek-v4.1-flash * fix(clf_ai_gateway): drop the unreachable non-chat supported-params branch * fix(clf_ai_gateway): send a dict reasoning_effort as its level * fix(clf_ai_gateway): refuse a reasoning_effort dict without a level unless drop_params drops it --------- Co-authored-by: bap1106 Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Claude Opus 5 --- .github/workflows/test-unit.yml | 1 + README.md | 1 + litellm/__init__.py | 9 + litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 3 + .../get_llm_provider_logic.py | 9 + .../clf_ai_gateway/chat/transformation.py | 95 +++++++ ...odel_prices_and_context_window_backup.json | 266 ++++++++++++++++++ .../provider_endpoints_support_backup.json | 26 ++ .../provider_create_fields.json | 28 ++ litellm/types/utils.py | 1 + litellm/utils.py | 6 + model_prices_and_context_window.json | 266 ++++++++++++++++++ provider_endpoints_support.json | 26 ++ tests/unit/llms/clf_ai_gateway/__init__.py | 0 ...test_clf_ai_gateway_chat_transformation.py | 251 +++++++++++++++++ 16 files changed, 993 insertions(+) create mode 100644 litellm/llms/clf_ai_gateway/chat/transformation.py create mode 100644 tests/unit/llms/clf_ai_gateway/__init__.py create mode 100644 tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 9ebb9bab018..9570b4d8cc8 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -245,6 +245,7 @@ jobs: tests/unit/llms/chatgpt tests/unit/llms/clarifai tests/unit/llms/claude_code + tests/unit/llms/clf_ai_gateway tests/unit/llms/cloudflare tests/unit/llms/codestral tests/unit/llms/codex diff --git a/README.md b/README.md index 531a50e1aca..3f1f21f903e 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | | | [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | | | [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | | +| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | | | [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | | | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | | [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index 084a4c516ce..d59edcaff89 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -633,6 +633,7 @@ fireworks_ai_models: Set = set() baseten_models: Set[str] = set() fireworks_ai_embedding_models: Set = set() deepinfra_models: Set = set() +clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets perplexity_models: Set = set() watsonx_models: Set = set() gemini_models: Set = set() @@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje bedrock_converse_models.add(key) elif value.get("litellm_provider") == "deepinfra": deepinfra_models.add(key) + elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map + clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key elif value.get("litellm_provider") == "perplexity": perplexity_models.add(key) elif value.get("litellm_provider") == "watsonx": @@ -1110,6 +1113,7 @@ model_list = list( | set(ollama_models) | bedrock_models | deepinfra_models + | clf_ai_gateway_models | perplexity_models | set(maritalk_models) | runwayml_models @@ -1235,6 +1239,7 @@ def _build_models_by_provider() -> dict: "ollama": ollama_models, "ollama_chat": ollama_models, "deepinfra": deepinfra_models, + "clf_ai_gateway": clf_ai_gateway_models, "perplexity": perplexity_models, "maritalk": maritalk_models, "watsonx": watsonx_models, @@ -2111,6 +2116,9 @@ if TYPE_CHECKING: LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig, ) from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig + from .llms.clf_ai_gateway.chat.transformation import ( + ClfAiGatewayConfig as _ClfAiGatewayConfig, + ) from .llms.llamafile.chat.transformation import ( LlamafileChatConfig as _LlamafileChatConfig, ) @@ -2140,6 +2148,7 @@ if TYPE_CHECKING: IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig] LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig] DeepInfraConfig: Type[_DeepInfraConfig] + ClfAiGatewayConfig: type[_ClfAiGatewayConfig] LlamafileChatConfig: Type[_LlamafileChatConfig] LMStudioChatConfig: Type[_LMStudioChatConfig] LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 7e81602fe77..c1f484f8d64 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -245,6 +245,7 @@ LLM_CONFIG_NAMES: Final = ( "MistralEmbeddingConfig", "OpenAIImageVariationConfig", "DeepInfraConfig", + "ClfAiGatewayConfig", "DeepgramAudioTranscriptionConfig", "TopazImageVariationConfig", "OpenAITextCompletionConfig", @@ -975,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OpenAIImageVariationConfig", ), "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"), + "ClfAiGatewayConfig": ( + ".llms.clf_ai_gateway.chat.transformation", + "ClfAiGatewayConfig", + ), "DeepgramAudioTranscriptionConfig": ( ".llms.deepgram.audio_transcription.transformation", "DeepgramAudioTranscriptionConfig", diff --git a/litellm/constants.py b/litellm/constants.py index 466942d35be..e07d727973d 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -819,6 +819,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "ollama", "ollama_chat", "deepinfra", + "clf_ai_gateway", "perplexity", "mistral", "groq", @@ -1018,6 +1019,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "api.perplexity.ai", "api.endpoints.anyscale.com/v1", "api.deepinfra.com/v1/openai", + "api.clfaigateway.dev/v1", "api.mistral.ai/v1", "codestral.mistral.ai/v1/chat/completions", "codestral.mistral.ai/v1/fim/completions", @@ -1089,6 +1091,7 @@ openai_compatible_providers: Final[list[str]] = [ "deepseek", "tencent", "deepinfra", + "clf_ai_gateway", "perplexity", "xinference", "xai", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f8c8ff71f8e..38242291b71 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -292,6 +292,10 @@ def get_llm_provider( elif endpoint == "api.deepinfra.com/v1/openai": custom_llm_provider = "deepinfra" dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY") + elif endpoint == "api.clfaigateway.dev/v1": + custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch + clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY") + dynamic_api_key = api_key or clf_env_key elif endpoint == "api.mistral.ai/v1": custom_llm_provider = "mistral" dynamic_api_key = get_secret_str("MISTRAL_API_KEY") @@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, ) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key) + elif custom_llm_provider == "clf_ai_gateway": + ( + api_base, # rebind-ok: resolved base/key are what this function returns, like every branch + dynamic_api_key, # rebind-ok: see above + ) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key) elif custom_llm_provider == "empower": api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1" dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY") diff --git a/litellm/llms/clf_ai_gateway/chat/transformation.py b/litellm/llms/clf_ai_gateway/chat/transformation.py new file mode 100644 index 00000000000..25e5d0f51f7 --- /dev/null +++ b/litellm/llms/clf_ai_gateway/chat/transformation.py @@ -0,0 +1,95 @@ +from typing import Final + +import litellm +from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model +from litellm.secret_managers.main import get_secret_str + +SUPPORTED_OPENAI_PARAMS: Final = ( + "stream", + "stream_options", + "frequency_penalty", + "presence_penalty", + "logit_bias", + "logprobs", + "top_logprobs", + "max_tokens", + "max_completion_tokens", + "n", + "stop", + "temperature", + "top_p", + "seed", + "response_format", + "tools", + "tool_choice", + "parallel_tool_calls", + "user", + "store", + "service_tier", +) + + +class ClfAiGatewayConfig(OpenAIGPTConfig): + @property + def custom_llm_provider(self) -> str | None: + return "clf_ai_gateway" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: matches OpenAIGPTConfig's list return + supports_reasoning: Final = litellm.supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ) + extra: Final = ("reasoning_effort",) if supports_reasoning else () + return [*SUPPORTED_OPENAI_PARAMS, *extra] + + def map_openai_params( + self, + non_default_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature + optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature + requested: Final = non_default_params.get("reasoning_effort") + requested_mapping: Final = as_str_mapping(requested) + effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested + declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway") + drop_enabled: Final = drop_params or litellm.drop_params is True + if requested_mapping is not None and effort is None and not drop_enabled: + raise UnsupportedParamsError( + message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop " + "unsupported openai params from the call, set `litellm.drop_params = True`", + llm_provider="clf_ai_gateway", + model=model, + status_code=400, + ) + drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared + if drops_effort: + verbose_logger.warning( + "Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.", + effort, + model, + declared, + ) + others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"} + normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others + forwarded: Final = others if drops_effort else normalized + mapped: Final[dict[str, object]] = super().map_openai_params( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # OpenAIGPTConfig types its params and result as bare dicts # mutable-ok: parent returns a dict + non_default_params=forwarded, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + return mapped + + def get_openai_compatible_provider_info( + self, api_base: str | None, api_key: str | None + ) -> tuple[str | None, str | None]: + resolved_api_base: Final = ( + api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1" + ) + dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY") + return resolved_api_base, dynamic_api_key diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8f514a9dfb7..1eaac0fda24 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -78638,6 +78638,272 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "clf_ai_gateway/glm-5.3": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.3-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 1.2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.2": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-4.7-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.4e-08, + "output_cost_per_token": 1.6e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-pro": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.28e-07, + "output_cost_per_token": 1.584e-06, + "cache_read_input_token_cost": 1.7e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.76e-07, + "output_cost_per_token": 5.28e-07, + "cache_read_input_token_cost": 5e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4.1-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 4.8e-07, + "cache_read_input_token_cost": 2e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.6": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 6.4e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.7-code": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 7.6e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/qwen3.8-27b": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.28e-06, + "cache_read_input_token_cost": 2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "xhigh" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, "kyma/claude-fable-5": { "max_input_tokens": 1000000, "max_output_tokens": 128000, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 18edb2cd7dc..d33e61026df 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -827,6 +827,32 @@ "generateContent": false } }, + "clf_ai_gateway": { + "display_name": "CLF AI Gateway (`clf_ai_gateway`)", + "url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "cloudflare": { "display_name": "Cloudflare AI Workers (`cloudflare`)", "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d8ff32c3819..df3af4ad487 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -931,6 +931,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "CLF_AI_GATEWAY", + "provider_display_name": "CLF AI Gateway", + "litellm_provider": "clf_ai_gateway", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.clfaigateway.dev/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "clf_ai_gateway/glm-5.3" + }, { "provider": "CLOUDFLARE", "provider_display_name": "Cloudflare", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 181ed5ea6db..5905aaa3380 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4485,6 +4485,7 @@ class LlmProviders(str, Enum): OLLAMA = "ollama" OLLAMA_CHAT = "ollama_chat" DEEPINFRA = "deepinfra" + CLF_AI_GATEWAY = "clf_ai_gateway" PERPLEXITY = "perplexity" TYPESAFE = "typesafe" NACE = "nace" diff --git a/litellm/utils.py b/litellm/utils.py index 549590e1cf5..d951b80d49b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7859,6 +7859,11 @@ def validate_environment( keys_in_environment = True else: missing_keys.append("DEEPINFRA_API_KEY") + elif custom_llm_provider == "clf_ai_gateway": + if "CLF_AI_GATEWAY_API_KEY" in os.environ: + keys_in_environment = True # rebind-ok: the result accumulates across the provider chain + else: + missing_keys.append("CLF_AI_GATEWAY_API_KEY") elif custom_llm_provider == "featherless_ai": if "FEATHERLESS_AI_API_KEY" in os.environ: keys_in_environment = True @@ -9537,6 +9542,7 @@ class ProviderConfigManager: LlmProviders.OOBABOOGA: (lambda: litellm.OobaboogaConfig(), False), LlmProviders.OLLAMA_CHAT: (lambda: litellm.OllamaChatConfig(), False), LlmProviders.DEEPINFRA: (lambda: litellm.DeepInfraConfig(), False), + LlmProviders.CLF_AI_GATEWAY: (lambda: litellm.ClfAiGatewayConfig(), False), LlmProviders.PERPLEXITY: (lambda: litellm.PerplexityChatConfig(), False), LlmProviders.MISTRAL: (lambda: litellm.MistralConfig(), False), LlmProviders.CODESTRAL: (lambda: litellm.MistralConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8f514a9dfb7..1eaac0fda24 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -78638,6 +78638,272 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "clf_ai_gateway/glm-5.3": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.3-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 2e-07, + "cache_read_input_token_cost": 1.2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-5.2": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 5.6e-07, + "output_cost_per_token": 1.76e-06, + "cache_read_input_token_cost": 1.04e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/glm-4.7-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 2.4e-08, + "output_cost_per_token": 1.6e-07, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-pro": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 5.28e-07, + "output_cost_per_token": 1.584e-06, + "cache_read_input_token_cost": 1.7e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.76e-07, + "output_cost_per_token": 5.28e-07, + "cache_read_input_token_cost": 5e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/deepseek-v4.1-flash": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "input_cost_per_token": 1.2e-07, + "output_cost_per_token": 4.8e-07, + "cache_read_input_token_cost": 2e-09, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.6": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 6.4e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "none", + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/kimi-k2.7-code": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 3.8e-07, + "output_cost_per_token": 1.6e-06, + "cache_read_input_token_cost": 7.6e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "high" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, + "clf_ai_gateway/qwen3.8-27b": { + "metadata": { + "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." + }, + "max_input_tokens": 262144, + "max_output_tokens": 131072, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.28e-06, + "cache_read_input_token_cost": 2e-08, + "litellm_provider": "clf_ai_gateway", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "reasoning_effort_levels": [ + "low", + "medium", + "xhigh" + ], + "supports_response_schema": true, + "supports_system_messages": true, + "supports_prompt_caching": true, + "supports_vision": true, + "source": "https://clfaigateway.dev/models" + }, "kyma/claude-fable-5": { "max_input_tokens": 1000000, "max_output_tokens": 128000, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 18edb2cd7dc..d33e61026df 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -827,6 +827,32 @@ "generateContent": false } }, + "clf_ai_gateway": { + "display_name": "CLF AI Gateway (`clf_ai_gateway`)", + "url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "cloudflare": { "display_name": "Cloudflare AI Workers (`cloudflare`)", "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", diff --git a/tests/unit/llms/clf_ai_gateway/__init__.py b/tests/unit/llms/clf_ai_gateway/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py b/tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py new file mode 100644 index 00000000000..ce1a1c1b870 --- /dev/null +++ b/tests/unit/llms/clf_ai_gateway/test_clf_ai_gateway_chat_transformation.py @@ -0,0 +1,251 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.clf_ai_gateway.chat.transformation import ClfAiGatewayConfig +from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + +DEFAULT_API_BASE: Final = "https://api.clfaigateway.dev/v1" +CHAT_URL: Final = f"{DEFAULT_API_BASE}/chat/completions" +REPO_ROOT: Final = Path(litellm.__file__).parent.parent + + +def _chat_response(usage: dict[str, object]) -> dict[str, object]: + return { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "glm-5.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}], + "usage": usage, + } + + +def _clf_rows(path: Path) -> dict[str, dict[str, object]]: + rows: Final = json.loads(path.read_text(encoding="utf-8")) + return {key: value for key, value in rows.items() if value.get("litellm_provider") == "clf_ai_gateway"} + + +def test_resolves_environment_and_explicit_credentials(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLF_AI_GATEWAY_API_KEY", "fake-environment-key") + monkeypatch.delenv("CLF_AI_GATEWAY_API_BASE", raising=False) + assert get_llm_provider(model="clf_ai_gateway/glm-5.3") == ( + "glm-5.3", + "clf_ai_gateway", + "fake-environment-key", + DEFAULT_API_BASE, + ) + assert get_llm_provider( + model="clf_ai_gateway/glm-5.3", api_key="fake-explicit-key", api_base="https://gateway.example/v1" + ) == ("glm-5.3", "clf_ai_gateway", "fake-explicit-key", "https://gateway.example/v1") + + monkeypatch.setenv("CLF_AI_GATEWAY_API_BASE", "https://env-gateway.example/v1") + assert get_llm_provider(model="clf_ai_gateway/glm-5.3")[3] == "https://env-gateway.example/v1" + + +def test_api_base_alone_selects_the_provider_and_keeps_an_explicit_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLF_AI_GATEWAY_API_KEY", "fake-environment-key") + _, provider, key, _ = get_llm_provider(model="glm-5.3", api_base=DEFAULT_API_BASE) + assert (provider, key) == ("clf_ai_gateway", "fake-environment-key") + + _, provider_with_key, explicit_key, _ = get_llm_provider( + model="glm-5.3", api_base=DEFAULT_API_BASE, api_key="fake-explicit-key" + ) + assert (provider_with_key, explicit_key) == ("clf_ai_gateway", "fake-explicit-key") + + +def test_chat_forwards_reasoning_effort_and_bills_cached_tokens(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, + json=_chat_response( + { + "prompt_tokens": 100, + "completion_tokens": 5, + "total_tokens": 105, + "prompt_tokens_details": {"cached_tokens": 80}, + } + ), + ) + response: Final = litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "Say hello"}], + reasoning_effort="low", + ) + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert request.headers["authorization"] == "Bearer fake-test-key" + assert body["model"] == "glm-5.3" + assert body["reasoning_effort"] == "low" + assert response.choices[0].message.content == "hello" + rates: Final = litellm.model_cost["clf_ai_gateway/glm-5.3"] + expected: Final = ( + 20 * rates["input_cost_per_token"] + + 80 * rates["cache_read_input_token_cost"] + + 5 * rates["output_cost_per_token"] + ) + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected) + + +@pytest.mark.parametrize( + ("param", "value"), + [ + ("functions", [{"name": "lookup", "parameters": {"type": "object", "properties": {}}}]), + ("function_call", "auto"), + ], +) +def test_legacy_fields_the_gateway_rejects_are_refused_or_dropped( + local_model_cost_map: None, param: str, value: object +) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + **{param: value}, + ) + + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + drop_params=True, + **{param: value}, + ) + assert param not in json.loads(route.calls.last.request.content) + + +@pytest.mark.parametrize( + "params", + [ + {"logit_bias": {"50256": -100}}, + {"logprobs": True, "top_logprobs": 2}, + {"store": False}, + {"service_tier": "auto"}, + ], +) +def test_params_the_gateway_accepts_are_forwarded_without_drop_params( + local_model_cost_map: None, params: dict[str, object] +) -> None: + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + **params, + ) + body: Final = json.loads(route.calls.last.request.content) + assert {key: body.get(key) for key in params} == params + + +@pytest.mark.parametrize( + ("effort", "drop_params", "forwarded"), + [ + ("high", True, None), + ("high", False, "high"), + ("xhigh", True, "xhigh"), + ({"effort": "low", "summary": "auto"}, True, "low"), + ({"effort": "low", "summary": "auto"}, False, "low"), + ({"effort": "high", "summary": "auto"}, True, None), + ], +) +def test_drop_params_drops_a_reasoning_effort_level_the_model_does_not_declare( + local_model_cost_map: None, effort: str | dict[str, str], drop_params: bool, forwarded: str | None +) -> None: + assert "high" not in litellm.model_cost["clf_ai_gateway/qwen3.8-27b"]["reasoning_effort_levels"] + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + litellm.completion( + model="clf_ai_gateway/qwen3.8-27b", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort=effort, + drop_params=drop_params, + ) + assert json.loads(route.calls.last.request.content).get("reasoning_effort") == forwarded + + +def test_a_reasoning_effort_dict_without_a_level_is_refused_unless_dropped(local_model_cost_map: None) -> None: + with respx.mock() as upstream: + route: Final = upstream.post(CHAT_URL).respond( + 200, json=_chat_response({"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}) + ) + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort={"summary": "auto"}, + ) + litellm.completion( + model="clf_ai_gateway/glm-5.3", + api_key="fake-test-key", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort={"summary": "auto"}, + drop_params=True, + ) + assert route.call_count == 1 + assert "reasoning_effort" not in json.loads(route.calls.last.request.content) + + +def test_reasoning_effort_is_advertised_only_for_reasoning_models(local_model_cost_map: None) -> None: + config: Final = ClfAiGatewayConfig() + assert "reasoning_effort" in config.get_supported_openai_params(model="glm-5.3") + assert "reasoning_effort" not in config.get_supported_openai_params(model="not-in-the-cost-map") + assert litellm.get_supported_openai_params( + model="glm-5.3", custom_llm_provider="clf_ai_gateway" + ) == config.get_supported_openai_params(model="glm-5.3") + + +def test_cost_map_rows_match_the_backup_and_resolve_their_effort_levels() -> None: + rows: Final = _clf_rows(REPO_ROOT / "model_prices_and_context_window.json") + assert rows + assert rows == _clf_rows(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") + for key, row in rows.items(): + assert row["input_cost_per_token"] > 0, key + assert row["output_cost_per_token"] > 0, key + assert row["max_output_tokens"] <= row["max_input_tokens"], key + assert resolve_supported_reasoning_efforts({**row, "key": key}, deployment_is_mapped=True) == tuple( + row["reasoning_effort_levels"] + ), key + + +def test_validate_environment_reports_the_env_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CLF_AI_GATEWAY_API_KEY", raising=False) + missing: Final = litellm.validate_environment(model="clf_ai_gateway/glm-5.3") + assert missing["keys_in_environment"] is False + assert missing["missing_keys"] == ["CLF_AI_GATEWAY_API_KEY"] + + monkeypatch.setenv("CLF_AI_GATEWAY_API_KEY", "fake-environment-key") + present: Final = litellm.validate_environment(model="clf_ai_gateway/glm-5.3") + assert present == {"keys_in_environment": True, "missing_keys": []} + + +def test_add_model_form_entry_points_at_a_mapped_model_and_the_default_base(local_model_cost_map: None) -> None: + fields_path: Final = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + entries: Final = [ + entry + for entry in json.loads(fields_path.read_text(encoding="utf-8")) + if entry["litellm_provider"] == "clf_ai_gateway" + ] + assert len(entries) == 1 + assert entries[0]["default_model_placeholder"] in litellm.model_cost + fields: Final = {field["key"]: field for field in entries[0]["credential_fields"]} + assert fields["api_key"]["required"] is True + assert fields["api_base"]["required"] is False + assert fields["api_base"]["placeholder"] == DEFAULT_API_BASE From a7030a5a2cab8b1ff525e1e47f8e7bb741609d50 Mon Sep 17 00:00:00 2001 From: Gopal Kamineni <72813883+Gopal3746@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:12:30 -0400 Subject: [PATCH 22/25] fix(vertex_ai): batch Claude models in Anthropic's native row shape (#45715) * fix(vertex_ai): batch Claude models in Anthropic's native row shape Claude deployments on Vertex were filed under publishers/google/models/ when the batch input was uploaded, so create_batch asked Vertex for a Google model that doesn't exist and got a bare 404 on batchPredictionJobs. The rows were also written as Gemini GenerateContentRequests, which Vertex rejects for Claude even with the right model path Claude uploads now go under publishers/anthropic/models/ and each line becomes {"custom_id", "request": } using the existing Vertex Claude transform. Output rows in the Anthropic message shape are converted back to OpenAI batch rows (custom_id from the row, errors as failed rows), native rows are costed from Anthropic usage, and create_batch fails fast with a 400 when a Claude job targets the global location, which Vertex doesn't serve for partner model batches Fixes #45671 * test(vertex_ai): cover malformed Claude output rows and rows without custom_id * fix(vertex_ai): build Claude batch rows with chat's param mapping and surface Vertex batch create errors Claude batch rows now go through get_optional_params like Vertex Claude chat, so unsupported params, top_k, response_format and extra_body follow drop_params. Tool names Claude would rename get a 400 at upload. A refused batch create returns Vertex's own status and message. * fix(vertex_ai): price Claude batch output rows without a request echo, keep @ version ids, refuse a json_tool_call user tool, keep Vertex error headers * fix(vertex_ai): keep httpx.HTTPStatusError on Vertex batch create errors and apply deployment chat params to Claude batch rows * fix(vertex_ai): keep Vertex's status code on batch create errors through the proxy * fix(vertex_ai): keep Vertex's error body on batch create errors and require custom_id on Claude lines * fix(vertex_ai): keep the old batch create error text and Google object names A failed Vertex batch create keeps httpx's error line first and adds Vertex's body after it, and logs the status and body again. Only Claude object names keep the @ version, so Google models are stored under the same names as before --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/vertex_ai/batches/handler.py | 34 +- .../llms/vertex_ai/batches/transformation.py | 67 ++- .../llms/vertex_ai/files/transformation.py | 259 +++++++++-- tests/unit/batches/test_batch_utils.py | 30 ++ .../llms/vertex_ai/batches/test_handler.py | 47 +- .../vertex_ai/batches/test_transformation.py | 60 +++ .../test_vertex_ai_files_transformation.py | 431 +++++++++++++++++- 7 files changed, 858 insertions(+), 70 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index 7c4238e3498..58f589fea61 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -15,8 +15,10 @@ from litellm.litellm_core_utils.url_utils import ( ) from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, + MaskedHTTPStatusError, get_async_httpx_client, get_httpx_client, + mask_sensitive_info, ) from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM @@ -81,6 +83,15 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str return response.json() +def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> MaskedHTTPStatusError: + body: Final = mask_sensitive_info(error.response.text) + return MaskedHTTPStatusError( + httpx.HTTPStatusError(f"{error}\n{body}", request=error.request, response=error.response), + message=body, + text=body, + ) + + class VertexAIBatchPrediction(VertexLLM): def __init__(self, gcs_bucket_name: str, *args, **kwargs): super().__init__(*args, **kwargs) @@ -166,11 +177,19 @@ class VertexAIBatchPrediction(VertexLLM): headers=headers, ) - response: Final = sync_handler.post( - url=api_base, - headers=headers, - data=json.dumps(vertex_batch_request), - ) + try: + response: Final = sync_handler.post( + url=api_base, + headers=headers, + data=json.dumps(vertex_batch_request), + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch create failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise _vertex_batch_create_error(e) from e payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()} _json_response: Final = payload_view["payload"] @@ -266,13 +285,12 @@ class VertexAIBatchPrediction(VertexLLM): data=json.dumps(vertex_batch_request), ) except httpx.HTTPStatusError as e: - error_body: Final = e.response.text litellm.verbose_logger.error( "Vertex AI batch create failed: status=%s, body=%s", e.response.status_code, - error_body[:1000], + e.response.text[:1000], ) - raise + raise _vertex_batch_create_error(e) from e payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()} _json_response: Final = payload_view["payload"] diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index 99ef750b5fc..99d78bcdca6 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -9,14 +9,41 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.llms.vertex_ai.common_utils import ( VertexAIError, + VertexAIModelRoute, convert_vertex_datetime_to_openai_datetime, + get_vertex_ai_model_route, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest from litellm.types.llms.vertex_ai import * -from litellm.types.llms.vertex_ai import GenerateContentResponseBody +from litellm.types.llms.vertex_ai import GenerateContentResponseBody, VertexPartnerProvider from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage _NATIVE_VERTEX_RESPONSE: Final = TypeAdapter(GenerateContentResponseBody) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +VERTEX_ANTHROPIC_VERSION: Final = "vertex-2023-10-16" +_ANTHROPIC_PUBLISHER_MODEL_PATH: Final = "publishers/anthropic/models/" + + +def is_vertex_anthropic_batch_model(model: str) -> bool: + """ + Whether a Vertex batch model (bare, `vertex_ai/`-prefixed or a publisher path) is a Claude model, decided the + way Vertex chat routes Claude. Vertex batches Claude only as `publishers/anthropic/models/` with rows in + Anthropic's Messages shape + """ + bare_model: Final = model.rsplit("/", 1)[-1] + return _ANTHROPIC_PUBLISHER_MODEL_PATH in model or ( + get_vertex_ai_model_route(bare_model) is VertexAIModelRoute.PARTNER_MODELS + and VertexPartnerProvider.claude.value in bare_model + ) + + +def as_json_object(value: object) -> Mapping[str, object] | None: + return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None + + +def is_vertex_anthropic_batch_output_response(response_body: Mapping[str, object]) -> bool: + return response_body.get("type") == "message" and isinstance(response_body.get("usage"), dict) def _int_field(mapping: Mapping[str, object], key: str) -> int: @@ -43,8 +70,22 @@ def is_vertex_embedding_batch_output_response(response_body: Mapping[str, object return isinstance(response_body.get("embedding"), dict) +def is_vertex_anthropic_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool: + """ + Whether a Vertex batch output row came from a Claude (Anthropic Messages) request. Vertex echoes only the + request's top-level `custom_id` on these rows, with an Anthropic `message` response on success and an empty + response plus a `status` on failure, so a Claude input file read back is never mistaken for output + """ + if not ("custom_id" in vertex_output_row and "processed_time" in vertex_output_row): + return False + response: Final = as_json_object(vertex_output_row.get("response")) + return (response is not None and is_vertex_anthropic_batch_output_response(response)) or bool( + vertex_output_row.get("status") + ) + + def is_native_vertex_batch_output_row(row: Mapping[str, object]) -> bool: - return isinstance(row.get("request"), dict) + return isinstance(row.get("request"), dict) or is_vertex_anthropic_batch_output_row(row) class NativeVertexBatchCostCalculator(Protocol): @@ -72,6 +113,9 @@ def _native_vertex_row_usage( response_body: Mapping[str, object], calculate_usage: Callable[[GenerateContentResponseBody], Usage], ) -> Usage | None: + anthropic_usage: Final = as_json_object(response_body.get("usage")) + if is_vertex_anthropic_batch_output_response(response_body) and anthropic_usage is not None: + return VertexAIAnthropicConfig().calculate_usage(usage_object=anthropic_usage, reasoning_content=None) if "usageMetadata" not in response_body: if not is_vertex_embedding_batch_output_response(response_body): return None @@ -97,21 +141,24 @@ def native_vertex_batch_row_stats( """ Usage and cost of one native Vertex predictions.jsonl row, a `{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}, "modelVersion": ...}}` - generateContent object or a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}` - embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`). - `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` - does, else the wildcard name so explicit deployment prices still apply; a row without a response, a - generateContent row without `response.usageMetadata`, and a row whose response fails validation are + generateContent object, a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}` + embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`) or a + `{"custom_id": ..., "request": ..., "response": {"type": "message", "model": ..., "usage": {...}}}` Claude object. + `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` (Claude + rows: `model`) does, else the wildcard name so explicit deployment prices still apply; a row without a response, + a generateContent row without `response.usageMetadata`, and a row whose response fails validation are None (failed). """ - response_body: Final = row.get("response") - if not isinstance(response_body, dict): + response_body: Final = as_json_object(row.get("response")) + if response_body is None: return None usage: Final = _native_vertex_row_usage(response_body, calculate_usage) if usage is None: return None total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens) - model_version: Final = response_body.get("modelVersion") + model_version: Final = response_body.get( + "model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion" + ) deployment_model: Final = model_name if model_name and "*" not in model_name else None model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name) if model is None: diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 410f29ef88a..9159c9a4070 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._uuid import uuid +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, RESPONSE_FORMAT_TOOL_NAME from litellm.files.types import FileContentStreamingResult from litellm.files.utils import FilesAPIUtils from litellm.litellm_core_utils.cloud_storage_security import ( @@ -32,13 +33,14 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, extract_file_metadata, ) from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body @@ -47,7 +49,14 @@ from litellm.llms.base_llm.files.transformation import ( BaseFileUploadStream, LiteLLMLoggingObj, ) -from litellm.llms.vertex_ai.batches.transformation import vertex_embedding_prompt_token_count +from litellm.llms.vertex_ai.batches.transformation import ( + VERTEX_ANTHROPIC_VERSION, + as_json_object, + is_vertex_anthropic_batch_model, + is_vertex_anthropic_batch_output_response, + is_vertex_anthropic_batch_output_row, + vertex_embedding_prompt_token_count, +) from litellm.llms.vertex_ai.common_utils import ( convert_vertex_datetime_to_openai_datetime, get_vertex_ai_fine_tuned_endpoint_id, @@ -59,6 +68,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( transform_openai_input_gemini_embed_content, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.files import StreamingMediaUploadConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -79,6 +89,7 @@ from litellm.types.utils import ( ModelResponse, Usage, ) +from litellm.utils import get_non_default_completion_params, get_optional_params from ..common_utils import VertexAIError from ..vertex_llm_base import VertexBase @@ -87,6 +98,7 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63 _CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_" _VERTEX_BATCH_KEY_FIELD: Final = "key" _MANAGED_GCS_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/([^/?]+)") +_PUBLISHER_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/") _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = ( ("outputDimensionality", "output_dimensionality"), ("taskType", "task_type"), @@ -225,10 +237,14 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, object]) -> str """ Resolve the OpenAI `custom_id` for a Vertex batch output row. - Embedding rows carry it in the top-level `key` field that Vertex echoes back; - `generateContent` rows have no such field, so it is smuggled through request - labels instead (see `_set_litellm_batch_custom_id_labels`). + Claude rows carry it in their own top-level `custom_id` field and embedding rows in + the top-level `key` field that Vertex echoes back; `generateContent` rows have no + such field, so it is smuggled through request labels instead (see + `_set_litellm_batch_custom_id_labels`). """ + anthropic_custom_id: Final = vertex_output_row.get("custom_id") + if anthropic_custom_id is not None: + return str(anthropic_custom_id) key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) if key is not None: return unquote(str(key)) @@ -289,6 +305,12 @@ def _is_vertex_generate_content_batch_output_row(vertex_output_row: Mapping[str, ) +def _is_vertex_chat_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool: + return _is_vertex_generate_content_batch_output_row(vertex_output_row) or is_vertex_anthropic_batch_output_row( + vertex_output_row + ) + + def _try_parse_vertex_batch_output_row(line: bytes) -> _VertexBatchRow | None: try: row: Final = _parse_vertex_batch_output_row(line.decode("utf-8")) @@ -410,6 +432,37 @@ def _openai_batch_output_row( } +def _vertex_anthropic_batch_output_row_to_openai( + custom_id: str, + response: object, + mock_httpx_response: httpx.Response, +) -> _OpenAIBatchOutputRow: + """ + One OpenAI batch output row from a Vertex Claude row's Anthropic `message` response + """ + anthropic_response: Final = as_json_object(response) + if anthropic_response is None or not is_vertex_anthropic_batch_output_response(anthropic_response): + return _openai_batch_output_row( + custom_id=custom_id, + error_code="vertex_ai_error", + error_message="Claude batch row has no message response", + ) + try: + model_response: Final = VertexAIAnthropicConfig().transform_parsed_response( + completion_response=dict(anthropic_response), + raw_response=mock_httpx_response, + model_response=ModelResponse(), + json_mode=True, + ) + except Exception as e: # noqa: BLE001 # one malformed row must fail alone, not the whole output file + return _openai_batch_output_row( + custom_id=custom_id, + error_code="transformation_error", + error_message=f"Failed to transform response: {e}", + ) + return _openai_batch_output_row(custom_id=custom_id, body=model_response.model_dump()) + + def _split_vertex_batch_key(vertex_output_row: Mapping[str, object]) -> tuple[str, int, int]: """ Resolve `(custom_id, index within that custom_id, group size)` for a Vertex batch @@ -684,21 +737,123 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset( + ("model", "messages", "custom_llm_provider", "drop_params", "stream", "stream_options") +) +_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"} +_CLAUDE_BATCH_DEPLOYMENT_PARAMS: Final = ( + _CLAUDE_BATCH_OPENAI_PARAMS - _CLAUDE_BATCH_ROW_FIXED_PARAMS - {"extra_headers"} +) | {"drop_params"} + + +def _claude_batch_deployment_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + {key: value for key, value in litellm_params.items() if key in _CLAUDE_BATCH_DEPLOYMENT_PARAMS} + ) + + +def _names_the_response_format_tool(tool: object) -> bool: + match tool: + case {"function": {"name": str(name)}}: + return name == RESPONSE_FORMAT_TOOL_NAME + case _: + return False + + +def _openai_batch_jsonl_entry_to_vertex_anthropic_row( + custom_id: object, + messages: list[AllMessageValues], + request_params: Mapping[str, object], + model: str, + deployment_params: Mapping[str, object], +) -> Mapping[str, object]: + """ + Transforms an OpenAI chat batch entry into the Anthropic Messages row Vertex batches Claude with, + `{"custom_id": ..., "request": {"messages": [...], "anthropic_version": "vertex-2023-10-16", "max_tokens": ...}}` + """ + if custom_id is None: + raise ValueError("custom_id is required on Claude lines, since Claude output rows are matched back only by it") + config: Final = VertexAIAnthropicConfig() + bare_model: Final = model.rsplit("/", 1)[-1] + params: Final = {**deployment_params, **request_params} + drop_params: Final = drops_unsupported_params(params.get("drop_params")) + chat_params: Final = { + **get_non_default_completion_params(params), + **{key: value for key, value in params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS}, + } + if any(map(_names_the_response_format_tool, iter_items(chat_params.get("tools") or ()))): + raise ValueError( + f"tool name {RESPONSE_FORMAT_TOOL_NAME!r} is reserved for response_format on Claude, " + "and batch output cannot tell a user tool by that name apart, so rename it" + ) + optional_params: Final[Mapping[str, object]] = get_optional_params( + model=bare_model, + custom_llm_provider="vertex_ai", + drop_params=drop_params, + messages=messages, + **{ # pyright: ignore[reportArgumentType] # batch JSONL bodies are untyped; get_optional_params checks each param as chat does + key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS + }, + ) + request_state: Final[dict[str, object]] = {"drop_params": drop_params} + anthropic_request: Final[Mapping[str, object]] = config.transform_request( + model=bare_model, + messages=messages, + optional_params={ + **{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")}, + "anthropic_version": VERTEX_ANTHROPIC_VERSION, + }, + litellm_params=request_state, + headers={}, + ) + renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)) + if renamed_tools: + raise ValueError( + f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, " + "and batch output cannot map a renamed tool back, so rename these tools" + ) + extra_body: Final = as_json_object(optional_params.get("extra_body")) + request: Final[Mapping[str, object]] = ( + anthropic_request + if extra_body is None + else { + **anthropic_request, + **config.transform_extra_body( + extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={} + ), + } + ) + return {"custom_id": str(custom_id), "request": request} + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + batch_model: str | None = None, + deployment_params: Mapping[str, object] = MappingProxyType({}), ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. + `batch_model` is the deployment model the batch runs against and wins over the entry's own `body.model`, + which picks the row shape: Anthropic Messages for Claude, `GenerateContentRequest` otherwise. + jsonl body for vertex is {"request": } Example Vertex jsonl {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}} """ + entry_body: Final = openai_entry.get("body") + model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "") + is_anthropic_model: Final = is_vertex_anthropic_batch_model(model) if _is_embeddings_batch_entry(openai_entry): + if is_anthropic_model: + raise VertexAIError( + status_code=400, + message=f"Vertex AI batch embeddings are not supported for Claude models ('{model}')", + ) return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry) - if not isinstance(openai_entry.get("body"), dict): + if not isinstance(entry_body, dict): raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object") openai_request_body: Final = openai_entry["body"] chat_request_body: Final = ( @@ -706,8 +861,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", [])) + if is_anthropic_model: + return ( + _openai_batch_jsonl_entry_to_vertex_anthropic_row( + custom_id=openai_entry.get("custom_id"), + messages=chat_messages, + request_params={ + key: value for key, value in chat_request_body.items() if key not in ("model", "messages") + }, + model=model, + deployment_params=deployment_params, + ), + ) vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), + messages=chat_messages, model=chat_request_body.get("model", ""), optional_params=map_openai_to_vertex_params(chat_request_body), custom_llm_provider="vertex_ai", @@ -859,15 +1027,24 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + batch_model: str | None = None, + deployment_params: Mapping[str, object] = MappingProxyType({}), ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._batch_model = batch_model + self._deployment_params = deployment_params def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, + self._map_openai_to_vertex_params, + batch_model=self._batch_model, + deployment_params=self._deployment_params, + ) except Exception as e: raise VertexAIError( status_code=400, @@ -923,16 +1100,31 @@ class _RawFileUploadStream(BaseFileUploadStream): return _iter_raw_file_chunks(self._file_content) +def _deployment_batch_model(litellm_params: Mapping[str, object]) -> str | None: + model: Final = litellm_params.get("model") + return model.removeprefix("vertex_ai/") if isinstance(model, str) and model else None + + +def _publisher_model_path(raw_model: str) -> str: + if _PUBLISHER_MODEL_PATH_PATTERN.search(raw_model): + return raw_model + publisher: Final = "anthropic" if is_vertex_anthropic_batch_model(raw_model) else "google" + return f"publishers/{publisher}/models/{raw_model}" + + def _managed_batch_object_name(raw_model: str, *, passthrough: bool) -> str: endpoint_id: Final = get_vertex_ai_fine_tuned_endpoint_id(raw_model) - model_path: Final = ( - f"endpoints/{endpoint_id}" - if endpoint_id is not None - else (raw_model if "publishers/google/models" in raw_model else f"publishers/google/models/{raw_model}") - ) - safe_model_path: Final = sanitize_cloud_object_path(model_path, fallback="model") + model_path: Final = f"endpoints/{endpoint_id}" if endpoint_id is not None else _publisher_model_path(raw_model) prefix: Final = _PASSTHROUGH_MANAGED_GCS_PREFIX if passthrough else VERTEX_AI_MANAGED_GCS_PREFIX - return f"{prefix}{safe_model_path}/{uuid.uuid4()}" + return f"{prefix}{_safe_managed_model_path(model_path)}/{uuid.uuid4()}" + + +def _safe_managed_model_path(model_path: str) -> str: + if not is_vertex_anthropic_batch_model(model_path): + return sanitize_cloud_object_path(model_path, fallback="model") + return "@".join( + sanitize_cloud_object_path(version_part, fallback="model") for version_part in model_path.split("@") + ) class VertexAIFilesConfig(VertexBase, BaseFilesConfig): @@ -1036,10 +1228,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): fallback_filename="file", ) - def _get_configured_bucket_name(self, litellm_params: dict) -> str: + def _get_configured_bucket_name(self, litellm_params: Mapping[str, object]) -> str: + configured_bucket_name: Final = litellm_params.get("gcs_bucket_name") or litellm_params.get("bucket_name") bucket_name: Final = ( - litellm_params.get("gcs_bucket_name") - or litellm_params.get("bucket_name") + (configured_bucket_name if isinstance(configured_bucket_name, str) else None) or os.getenv("GCS_BATCH_BUCKET_NAME") or os.getenv("GCS_BUCKET_NAME") ) @@ -1053,7 +1245,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], data: CreateFileRequest, ) -> str: """ @@ -1077,11 +1269,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_deployment_batch_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1138,7 +1329,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1170,6 +1361,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params), + batch_model=_deployment_batch_model(litellm_params), + deployment_params=_claude_batch_deployment_params(litellm_params), ), content_type="application/json", ) @@ -1220,7 +1413,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException: return VertexAIError(status_code=status_code, message=error_message, headers=headers) - def _parse_gcs_uri(self, file_id: str, litellm_params: dict | None = None) -> tuple[str, str]: + def _parse_gcs_uri(self, file_id: str, litellm_params: Mapping[str, object] | None = None) -> tuple[str, str]: """ Validate a managed GCS file_id and return (bucket, url-encoded-object-path). """ @@ -1238,7 +1431,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_id: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}" @@ -1268,7 +1461,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_id: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}" @@ -1311,7 +1504,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_content_request, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: file_id: Final = file_content_request.get("file_id", "") bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) @@ -1408,7 +1601,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): headers=MappingProxyType({**headers, "content-length": str(len(transformed_content))}), ) - if not _is_vertex_generate_content_batch_output_row(first_row): + if not _is_vertex_chat_batch_output_row(first_row): return FileContentStreamingResult(stream_iterator=replayed_stream, headers=headers) return FileContentStreamingResult( @@ -1492,10 +1685,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): # first line is not valid UTF-8/JSON) raises and falls through to the # passthrough below, leaving the content untouched. first_row: Final = _parse_vertex_batch_output_row(first_line) - if not ( - _is_vertex_embeddings_batch_output_row(first_row) - or _is_vertex_generate_content_batch_output_row(first_row) - ): + if not (_is_vertex_embeddings_batch_output_row(first_row) or _is_vertex_chat_batch_output_row(first_row)): return content context: Final = _new_vertex_batch_output_row_transform_context() @@ -1560,6 +1750,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): error_message=status, ) + if is_vertex_anthropic_batch_output_row(vertex_output): + return _vertex_anthropic_batch_output_row_to_openai( + custom_id=custom_id, + response=vertex_output.get("response"), + mock_httpx_response=mock_httpx_response, + ) + # Transform successful response using existing transformation vertex_response: Final = vertex_output.get("response", {}) diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index f2bd5855d15..d154086aec9 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -2310,6 +2310,36 @@ async def test_native_vertex_rows_route_to_vertex_cost_path_without_flag(monkeyp assert {(call["model"], call["custom_llm_provider"]) for call in calls} == {("gemini-2.5-flash", "vertex_ai")} +@pytest.mark.asyncio +async def test_raw_vertex_claude_rows_are_priced_like_with_the_native_output_flag(monkeypatch): + claude_message: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "Blue."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 5}, + } + rows: Final = [ + {"custom_id": "ok", "processed_time": "2026-10-29T03:00:00Z", "response": claude_message, "status": ""}, + {"custom_id": "bad", "processed_time": "2026-10-29T03:00:00Z", "response": {}, "status": "Bad Request"}, + ] + + async def cost_with_flag(flag: bool) -> bu.BatchCostUsageResult: + monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", flag, raising=False) + return await bu.calculate_batch_cost_and_usage( + file_content_dictionary=rows, custom_llm_provider="vertex_ai", model_name="vertex_ai/claude-sonnet-4-6" + ) + + unflagged: Final = await cost_with_flag(False) + flagged: Final = await cost_with_flag(True) + + assert unflagged.cost > 0 + assert (unflagged.successful_requests, unflagged.failed_requests) == (1, 1) + assert (unflagged.cost, unflagged.usage.total_tokens) == (flagged.cost, flagged.usage.total_tokens) + + @pytest.mark.asyncio async def test_openai_shaped_vertex_rows_keep_the_generic_path_without_flag(monkeypatch): monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", False, raising=False) diff --git a/tests/unit/llms/vertex_ai/batches/test_handler.py b/tests/unit/llms/vertex_ai/batches/test_handler.py index 91a2adb9d19..2c246e065a8 100644 --- a/tests/unit/llms/vertex_ai/batches/test_handler.py +++ b/tests/unit/llms/vertex_ai/batches/test_handler.py @@ -30,6 +30,7 @@ from __future__ import annotations import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -41,9 +42,11 @@ from litellm.llms.vertex_ai.batches.handler import ( # noqa: E402 VertexAIBatchPrediction, ) from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 +from litellm.proxy.utils import handle_exception_on_proxy # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 HMOD = "litellm.llms.vertex_ai.batches.handler" +_CLIENT_ERROR_429: Final = "Client error '429 Too Many Requests' for url 'https://x/batchPredictionJobs'" TOKEN = "ya29.fake-access-token" PROJECT = "my-project" LOCATION = "us-central1" @@ -370,19 +373,19 @@ def test_create_batch_sync_endpoint_without_deployed_model_raises_400(): client.post.assert_not_called() -def test_create_batch_sync_httpstatuserror_propagates(): - """``HTTPHandler.post`` raises for non-2xx via ``raise_for_status``; the - sync create path must surface that error, not swallow it.""" +def test_create_batch_sync_httpstatuserror_carries_vertex_status_body_and_headers(): h = _make_handler() client = MagicMock() request = httpx.Request("POST", "https://x/batchPredictionJobs") - err_response = httpx.Response(status_code=500, request=request, text="boom") + err_response: Final = httpx.Response( + status_code=429, request=request, text="boom", headers={"retry-after": "30"} + ) client.post.side_effect = httpx.HTTPStatusError( - "boom", request=request, response=err_response + _CLIENT_ERROR_429, request=request, response=err_response ) with patch(f"{HMOD}.get_httpx_client", return_value=client): - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(httpx.HTTPStatusError) as exc_info: h.create_batch( _is_async=False, create_batch_data=CREATE_DATA, @@ -393,6 +396,11 @@ def test_create_batch_sync_httpstatuserror_propagates(): timeout=600.0, max_retries=None, ) + assert exc_info.value.response.status_code == 429 + assert exc_info.value.response.headers["retry-after"] == "30" + assert handle_exception_on_proxy(exc_info.value).code == "429" + assert (getattr(exc_info.value, "message", None), getattr(exc_info.value, "text", None)) == ("boom", "boom") + assert str(exc_info.value).splitlines() == [_CLIENT_ERROR_429, "boom"] def test_create_batch_input_file_id_without_model_raises_400_before_post(): @@ -800,14 +808,16 @@ def test_cancel_batch_sync_httpstatuserror_logged_and_reraised(): client.get.assert_not_called() -def test_create_batch_async_httpstatuserror_logged_and_reraised(): +def test_create_batch_async_httpstatuserror_carries_vertex_status_body_and_headers(): h = _make_handler() async_client = MagicMock() request = httpx.Request("POST", "https://x/batchPredictionJobs") - err_response = httpx.Response(status_code=500, request=request, text="boom") + err_response: Final = httpx.Response( + status_code=429, request=request, text="boom", headers={"retry-after": "30"} + ) async_client.post = AsyncMock( side_effect=httpx.HTTPStatusError( - "boom", request=request, response=err_response + _CLIENT_ERROR_429, request=request, response=err_response ) ) @@ -825,8 +835,13 @@ def test_create_batch_async_httpstatuserror_logged_and_reraised(): timeout=600.0, max_retries=None, ) - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(httpx.HTTPStatusError) as exc_info: _run(coro) + assert exc_info.value.response.status_code == 429 + assert exc_info.value.response.headers["retry-after"] == "30" + assert handle_exception_on_proxy(exc_info.value).code == "429" + assert (getattr(exc_info.value, "message", None), getattr(exc_info.value, "text", None)) == ("boom", "boom") + assert str(exc_info.value).splitlines() == [_CLIENT_ERROR_429, "boom"] def test_async_retrieve_batch_non_200_raises(): @@ -1043,15 +1058,7 @@ async def test_vertex_list_batches(monkeypatch): @pytest.mark.asyncio -async def test_vertex_async_create_batch_logs_error_body_on_http_error(): - """ - When Vertex AI returns an HTTP error (e.g. 400), _async_create_batch should - re-raise httpx.HTTPStatusError (not swallow it) and log the response body. - - Before the fix the error body was lost because AsyncHTTPHandler.post() - calls raise_for_status() internally, raising before the handler's own - status-code check could log the body. - """ +async def test_vertex_async_create_batch_httpstatuserror_carries_vertex_error_body(): from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") @@ -1081,4 +1088,4 @@ async def test_vertex_async_create_batch_logs_error_body_on_http_error(): ) assert exc_info.value.response.status_code == 400 - assert "gemini-2.0-flash" in exc_info.value.response.text + assert "Do not support publisher model gemini-2.0-flash" in str(exc_info.value) diff --git a/tests/unit/llms/vertex_ai/batches/test_transformation.py b/tests/unit/llms/vertex_ai/batches/test_transformation.py index 064845a7266..1440a57a15f 100644 --- a/tests/unit/llms/vertex_ai/batches/test_transformation.py +++ b/tests/unit/llms/vertex_ai/batches/test_transformation.py @@ -21,6 +21,8 @@ import pytest from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402 VertexAIBatchTransformation, + is_vertex_anthropic_batch_model, + native_vertex_batch_row_stats, ) from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 @@ -493,3 +495,61 @@ def test_get_gcs_uri_prefix_keeps_passthrough_segment_so_output_lands_beside_inp T._get_gcs_uri_prefix_from_file(PASSTHROUGH_INPUT_FILE) == "gs://litellm-testing-bucket/litellm-vertex-files/passthrough/publishers/google/models/gemini-2.5-flash" ) + + +CLAUDE_INPUT_FILE = ( + "gs://litellm-testing-bucket/litellm-vertex-files/publishers/anthropic/models/claude-sonnet-4-6/uuid-1" +) + + +def test_claude_batch_job_targets_the_anthropic_publisher_model(): + job: Final = T.transform_openai_batch_request_to_vertex_ai_batch_request( + {"input_file_id": CLAUDE_INPUT_FILE}, vertex_project="p", vertex_location="us-east5" + ) + assert job["model"] == "publishers/anthropic/models/claude-sonnet-4-6" + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("claude-sonnet-4-6", True), + ("vertex_ai/claude-opus-4-1@20250805", True), + ("publishers/anthropic/models/claude-haiku-4-5", True), + ("gemini-2.5-flash", False), + ("publishers/google/models/gemini-2.5-flash", False), + ], +) +def test_is_vertex_anthropic_batch_model(model: str, expected: bool): + assert is_vertex_anthropic_batch_model(model) is expected + + +def test_claude_output_row_is_costed_from_anthropic_usage_and_its_own_model(): + def cost_calculator(usage, model, custom_llm_provider=None, model_info=None, vertex_location=None) -> tuple[float, float]: + if model != "claude-sonnet-4-6": + return 0.0, 0.0 + return usage.prompt_tokens * 0.001, usage.completion_tokens * 0.002 + + stats: Final = native_vertex_batch_row_stats( + { + "custom_id": "r1", + "request": {"messages": [{"role": "user", "content": "hi"}], "anthropic_version": "vertex-2023-10-16"}, + "response": { + "id": "msg_vrtx_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 5}, + }, + }, + None, + model_info=None, + calculate_usage=lambda _: pytest.fail("Claude rows must not go through the Gemini usage parser"), + cost_calculator=cost_calculator, + ) + + assert stats is not None, "a successful Claude row must be costed, not counted as failed" + assert (stats.usage.prompt_tokens, stats.usage.completion_tokens, stats.total_tokens) == (12, 5, 17) + assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((0.012, 0.010)) + assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((12 * 0.001, 5 * 0.002)) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index cf4a7c4cbc8..98e204dd44e 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,7 +5,7 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Final from types import MappingProxyType from urllib.parse import parse_qs, urlparse @@ -16,6 +16,7 @@ from pydantic import TypeAdapter from unittest.mock import MagicMock import litellm +from litellm.llms.base_llm.files.transformation import BaseFileUploadStream from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -168,6 +169,17 @@ class TestBatchObjectNaming: object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini-2.5-flash"}}]) assert object_name.startswith("litellm-vertex-files/publishers/google/models/gemini-2.5-flash/") + @pytest.mark.parametrize( + ("model", "stored_model"), + [ + ("multimodalembedding@001", "publishers/google/models/multimodalembedding_001/"), + ("claude-haiku-4-5@20251001", "publishers/anthropic/models/claude-haiku-4-5@20251001/"), + ], + ) + def test_only_claude_models_keep_the_at_version_in_their_object_name(self, config, model, stored_model): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": model}}]) + assert object_name.startswith(f"litellm-vertex-files/{stored_model}") + def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config): """A numeric endpoint id must not be filed under publishers/google/models/gemini/, which the batch transformation later mangles into a nonexistent publisher model (LIT-6899).""" @@ -1992,6 +2004,423 @@ class TestVertexEmbeddingsBatchOutputTranslation: assert config._try_transform_vertex_batch_output_to_openai(content) == content +CLAUDE_DEPLOYMENT = {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/claude-sonnet-4-6"} + + +def _claude_chat_entry(custom_id: str | None, content: str, **body: object) -> dict[str, object]: + entry: Final[dict[str, object]] = { + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "claude-sonnet-4-6", "messages": [{"role": "user", "content": content}], **body}, + } + return entry if custom_id is None else {"custom_id": custom_id, **entry} + + +def _uploaded_rows( + config: VertexAIFilesConfig, litellm_params: Mapping[str, object], entries: Sequence[Mapping[str, object]] +) -> list: + upload: Final = config.transform_create_file_request( + model="", + create_file_data={ + "file": ("batch.jsonl", "\n".join(json.dumps(entry) for entry in entries).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={}, + litellm_params=dict(litellm_params), + ) + match upload: + case {"streaming_media_upload": {"body_stream": BaseFileUploadStream() as stream}}: + return [json.loads(line) for line in b"".join(stream.iter_bytes()).splitlines()] + case _: + pytest.fail(f"expected a streamed upload, got {upload!r}") + + +def _claude_output_row(custom_id: str, text: str, content: object = None) -> dict[str, object]: + return { + "custom_id": custom_id, + "status": "", + "processed_time": "2026-10-28T20:37:47.634+00:00", + "response": { + "id": "msg_vrtx_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": text}] if content is None else content, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 5}, + }, + } + + +class TestVertexClaudeBatch: + """Claude on Vertex batches only as `publishers/anthropic/models/` with Anthropic Messages rows + (https://github.com/BerriAI/litellm/issues/45671)""" + + @pytest.mark.parametrize("claude_model", ["claude-sonnet-4-6", "claude-haiku-4-5@20251001"]) + def test_claude_upload_creates_a_batch_job_for_the_anthropic_publisher_model(self, config, claude_model): + from litellm.llms.vertex_ai.batches.transformation import VertexAIBatchTransformation + + url: Final = config.get_complete_file_url( + api_base=None, + api_key=None, + model="", + optional_params={}, + litellm_params={**CLAUDE_DEPLOYMENT, "model": f"vertex_ai/{claude_model}"}, + data={"file": ("batch.jsonl", json.dumps(_claude_chat_entry("r1", "hi")).encode()), "purpose": "batch"}, + ) + object_name: Final = parse_qs(urlparse(url).query)["name"][0] + + job: Final = VertexAIBatchTransformation.transform_openai_batch_request_to_vertex_ai_batch_request( + {"input_file_id": f"gs://my-bucket/{object_name}"}, vertex_project="p", vertex_location="us-east5" + ) + + assert job["model"] == f"publishers/anthropic/models/{claude_model}", object_name + + def test_claude_body_model_without_deployment_is_filed_under_anthropic_publisher(self, config): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "claude-haiku-4-5"}}]) + assert object_name.startswith("litellm-vertex-files/publishers/anthropic/models/claude-haiku-4-5/") + + def test_publisher_path_model_is_not_nested_under_google(self, config): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl( + [{"body": {}}], deployment_model="vertex_ai/publishers/anthropic/models/claude-sonnet-4-6" + ) + assert object_name.startswith("litellm-vertex-files/publishers/anthropic/models/claude-sonnet-4-6/") + + def test_claude_rows_are_uploaded_in_anthropic_messages_shape(self, config): + rows: Final = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [_claude_chat_entry("request-1", "Hello!", max_tokens=50, temperature=0.2)], + ) + + assert rows == [ + { + "custom_id": "request-1", + "request": { + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}], + "max_tokens": 50, + "temperature": 0.2, + "anthropic_version": "vertex-2023-10-16", + }, + } + ] + + def test_claude_system_message_becomes_the_anthropic_system_field(self, config): + entry: Final = _claude_chat_entry( + "r1", + "unused", + max_tokens=10, + messages=[{"role": "system", "content": "be brief"}, {"role": "user", "content": "Hello!"}], + ) + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert row["request"]["system"] == [{"type": "text", "text": "be brief"}] + assert [message["role"] for message in row["request"]["messages"]] == ["user"] + + def test_claude_row_drops_an_unsupported_param_under_deployment_drop_params(self, config, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + (row,) = _uploaded_rows( + config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9", "drop_params": True}, [entry] + ) + + assert row["request"]["max_tokens"] == 10 + assert "top_p" not in row["request"] + + def test_claude_row_drops_a_param_the_deployment_lists_in_additional_drop_params(self, config, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + (row,) = _uploaded_rows( + config, + {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9", "additional_drop_params": ["top_p"]}, + [entry], + ) + + assert row["request"]["max_tokens"] == 10 + assert "top_p" not in row["request"] + + def test_claude_rows_take_deployment_params_as_defaults_the_row_overrides(self, config): + rows: Final = _uploaded_rows( + config, + {**CLAUDE_DEPLOYMENT, "max_tokens": 64, "temperature": 0.3}, + [_claude_chat_entry("r1", "hi"), _claude_chat_entry("r2", "hi", max_tokens=10)], + ) + + assert [(row["request"]["max_tokens"], row["request"]["temperature"]) for row in rows] == [(64, 0.3), (10, 0.3)] + + def test_claude_row_with_an_unsupported_param_is_refused_without_drop_params(self, config, monkeypatch): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + with pytest.raises(VertexAIError, match=r"line 1 .*top_p=0.5.*drop_params") as exc_info: + _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"}, [entry]) + assert exc_info.value.status_code == 400 + + def test_claude_deployment_wins_over_a_gemini_body_model(self, config): + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, model="gemini-2.5-flash") + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert row["request"]["anthropic_version"] == "vertex-2023-10-16" + assert "contents" not in row["request"] + + def test_gemini_deployment_still_uploads_generate_content_rows(self, config): + (row,) = _uploaded_rows( + config, + {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/gemini-2.5-flash"}, + [_claude_chat_entry("r1", "hi")], + ) + assert row["request"]["contents"] == [{"role": "user", "parts": [{"text": "hi"}]}] + assert "custom_id" not in row + + def test_claude_embeddings_batch_line_is_rejected_with_400(self): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + entry: Final = { + "custom_id": "e1", + "url": "/v1/embeddings", + "body": {"model": "claude-sonnet-4-6", "input": "hi"}, + } + with pytest.raises(VertexAIError) as exc_info: + _openai_batch_jsonl_entry_to_vertex_rows(entry, VertexAIFilesConfig()._map_openai_to_vertex_params) + assert exc_info.value.status_code == 400 + + def test_claude_output_rows_become_openai_batch_rows(self, config): + content: Final = b"\n".join( + json.dumps(row).encode() for row in (_claude_output_row("a", "first"), _claude_output_row("b", "second")) + ) + + rows: Final = [ + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ] + + assert [row["custom_id"] for row in rows] == ["a", "b"] + assert [row["error"] for row in rows] == [None, None] + bodies: Final = [row["response"]["body"] for row in rows] + assert [body["choices"][0]["message"]["content"] for body in bodies] == ["first", "second"] + assert [body["choices"][0]["finish_reason"] for body in bodies] == ["stop", "stop"] + assert [(body["usage"]["prompt_tokens"], body["usage"]["completion_tokens"]) for body in bodies] == [ + (12, 5), + (12, 5), + ] + assert {body["model"] for body in bodies} == {"claude-sonnet-4-6"} + + @pytest.mark.parametrize("failed_first", [False, True]) + def test_failed_claude_output_row_is_an_openai_error_row(self, config, failed_first): + failed_row: Final = { + "custom_id": "bad", + "status": 'Bad Request: {"type": "error", "error": {"type": "invalid_request_error", "message": "too long"}}', + "processed_time": "2026-10-28T20:37:48.413+00:00", + "response": {}, + } + rows_in: Final = (failed_row, _claude_output_row("ok", "fine")) + content: Final = b"\n".join(json.dumps(row).encode() for row in (rows_in if failed_first else rows_in[::-1])) + + rows_out: Final = { + row["custom_id"]: row + for row in ( + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ) + } + + assert rows_out["ok"]["response"]["body"]["choices"][0]["message"]["content"] == "fine" + assert rows_out["bad"]["response"] is None + assert rows_out["bad"]["error"]["code"] == "vertex_ai_error" + assert "too long" in rows_out["bad"]["error"]["message"] + + def test_claude_row_keeps_provider_only_params_and_leaves_out_litellm_ones(self, config): + (row,) = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [_claude_chat_entry("r1", "hi", max_tokens=10, top_k=5, drop_params=True, metadata={"team": "a"})], + ) + + assert row["request"]["top_k"] == 5 + assert not {"drop_params", "metadata", "extra_body"} & set(row["request"]) + + def test_claude_row_top_k_follows_drop_params_on_a_model_without_sampling_params(self, config, monkeypatch): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + deployment: Final = {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"} + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_k=5) + + with pytest.raises(VertexAIError, match=r"line 1 .*top_k") as exc_info: + _uploaded_rows(config, deployment, [entry]) + (row,) = _uploaded_rows(config, {**deployment, "drop_params": True}, [entry]) + + assert exc_info.value.status_code == 400 + assert "top_k" not in row["request"] + + @pytest.mark.parametrize("param", [{"n": 2}, {"seed": 7}, {"presence_penalty": 0.5}, {"logprobs": True}]) + def test_claude_row_param_claude_does_not_support_follows_drop_params(self, config, monkeypatch, param): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setattr(litellm, "drop_params", False) + (name,) = param + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, **param) + + with pytest.raises(VertexAIError, match=rf"line 1 .*{name}") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + (row,) = _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "drop_params": True}, [entry]) + + assert exc_info.value.status_code == 400 + assert name not in row["request"] + + def test_claude_row_refuses_a_tool_name_claude_would_rename(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + def tool(name: str) -> dict[str, object]: + return {"type": "function", "function": {"name": name, "parameters": {"type": "object", "properties": {}}}} + + renamed: Final = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool("get.weather")]) + kept: Final = _claude_chat_entry("r2", "hi", max_tokens=10, tools=[tool("get_weather")]) + + with pytest.raises(VertexAIError, match=r"line 1 .*get\.weather") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [renamed]) + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [kept]) + + assert exc_info.value.status_code == 400 + assert [t["name"] for t in row["request"]["tools"]] == ["get_weather"] + + def test_claude_row_refuses_a_user_tool_named_like_the_response_format_tool(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + tool: Final = {"type": "function", "function": {"name": "json_tool_call", "parameters": {"type": "object"}}} + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool]) + + with pytest.raises(VertexAIError, match=r"line 1 .*json_tool_call") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert exc_info.value.status_code == 400 + + def test_claude_row_leaves_out_stream(self, config): + entry: Final = _claude_chat_entry( + "r1", "hi", max_tokens=10, stream=True, stream_options={"include_usage": True} + ) + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert "stream" not in row["request"] + assert "stream_options" not in row["request"] + + def test_claude_row_merges_extra_body_into_the_request(self, config): + (row,) = _uploaded_rows( + config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10, extra_body={"top_k": 3})] + ) + + assert row["request"]["top_k"] == 3 + assert "extra_body" not in row["request"] + + def test_claude_response_format_row_round_trips_as_json_content(self, config): + schema: Final = {"type": "object", "properties": {"color": {"type": "string"}}, "required": ["color"]} + (row,) = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [ + _claude_chat_entry( + "r1", + "sky color?", + max_tokens=50, + response_format={"type": "json_schema", "json_schema": {"name": "sky", "schema": schema}}, + ) + ], + ) + assert "json_mode" not in row["request"] + (json_tool,) = row["request"]["tools"] + output_row: Final = _claude_output_row( + "r1", + "unused", + content=[{"type": "tool_use", "id": "toolu_1", "name": json_tool["name"], "input": {"color": "blue"}}], + ) + + (out,) = ( + json.loads(line) + for line in config._try_transform_vertex_batch_output_to_openai( + json.dumps(output_row).encode() + ).splitlines() + ) + + message: Final = out["response"]["body"]["choices"][0]["message"] + assert json.loads(message["content"]) == {"color": "blue"} + assert not message.get("tool_calls") + + def test_claude_output_keeps_a_user_tool_call_as_a_tool_call(self, config): + output_row: Final = _claude_output_row( + "r1", + "unused", + content=[{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}], + ) + + (out,) = ( + json.loads(line) + for line in config._try_transform_vertex_batch_output_to_openai( + json.dumps(output_row).encode() + ).splitlines() + ) + + (tool_call,) = out["response"]["body"]["choices"][0]["message"]["tool_calls"] + assert tool_call["function"]["name"] == "get_weather" + assert json.loads(tool_call["function"]["arguments"]) == {"city": "Paris"} + + def test_claude_input_file_read_back_is_returned_unchanged(self, config): + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10)]) + content: Final = json.dumps(row).encode() + + assert config._try_transform_vertex_batch_output_to_openai(content) == content + + def test_malformed_claude_message_fails_only_its_own_row(self, config): + malformed: Final = _claude_output_row("bad", "unused", content=5) + content: Final = json.dumps(_claude_output_row("ok", "fine")).encode() + b"\n" + json.dumps(malformed).encode() + + ok_row, bad_row = ( + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ) + + assert ok_row["response"]["body"]["choices"][0]["message"]["content"] == "fine" + assert bad_row["custom_id"] == "bad" + assert bad_row["response"] is None + assert bad_row["error"]["code"] == "transformation_error" + + def test_claude_line_without_custom_id_is_a_400_at_upload(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + with pytest.raises(VertexAIError, match=r"line 2 .*custom_id") as exc_info: + _uploaded_rows( + config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10), _claude_chat_entry(None, "hi")] + ) + + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_streamed_claude_output_is_transformed(self, config): + async def chunks(): + yield json.dumps(_claude_output_row("a", "first")).encode() + b"\n" + + result: Final = await config.transform_file_content_stream( + stream_iterator=chunks(), + headers={}, + request_url="https://storage.googleapis.com/storage/v1/b/my-bucket/o/x?alt=media", + logging_obj=MagicMock(), + litellm_params={}, + ) + body: Final = b"".join([chunk async for chunk in result.stream_iterator]) + + (row,) = (json.loads(line) for line in body.splitlines()) + assert row["custom_id"] == "a" + assert row["response"]["body"]["choices"][0]["message"]["content"] == "first" + + _SAMPLING_KNOBS: Final = MappingProxyType({"temperature": 0, "top_p": 0.5, "top_k": 3}) From 44cd858697c397decff308e2db3d382a26887f2f Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:19:36 -0700 Subject: [PATCH 23/25] fix(cost): bill the requested model's rates when the response names an unpriced model (#48900) * fix(cost): bill the requested model's rates when the response names an unpriced model Fireworks answers chat and /v1/messages calls with a display name such as "Qwen 3.8 Max" in the response's model field. The cost path priced that name, the Fireworks calculator fell back to its free default bucket without raising, and the call billed $0 even though the requested model has a row. When the response model resolves to no cost-map row or rule that declares rates and the requested model does, price from the requested model. A priced response model still wins, so dated snapshots keep their own rates, and an unpriced requested model such as a router alias leaves the result unchanged. * perf(cost): check the exact cost-map row before generalization rules when picking the priced model * fix(cost): apply the priced fallback to the provider response model streaming records * perf(cost): skip the served model's rule lookup when the requested model has no rates * fix(cost): price the requested model without a model arg and keep Fireworks size buckets behind the proxy When completion_cost runs with no model, the requested model now comes from the call's hidden litellm_model_name, so a display-name response still bills at the requested model's rates. A Fireworks model whose name carries a parameter size counts as priced through its size bucket. The Fireworks per-token lookup now treats a cost-map row that prices nothing, like the empty row the Router registers for an unlisted deployment, as missing, so an unlisted Fireworks model bills at its size bucket instead of $0 even when Fireworks echoes its slug. The "row prices anything" helpers move to llm_cost_calc/utils so the Fireworks lookup can reuse them without a circular import. * fix(fireworks): keep prices from a matching fallback rule for an unlisted model An unlisted Fireworks model that only a fallback generalization rule prices has no raw cost map row, so the empty-row check sent it to the size bucket and dropped the rule's rates. The Fireworks lookup now keeps the model info when the row or a matching rule declares a rate. * fix(cost): price a bare response model under the requested provider and drop the unlisted Fireworks bucket change The logging payload picks the pricing model without a provider, so a bare echo like mistral-large-latest failed its price check and the requested alias took over model_map_key. The served name is now priced under the requested model's provider, and an echo that is the requested name minus its provider prefix skips the checks. Treating the Router's empty price row as missing let unlisted Fireworks deployments bill from a name guess, which misprices dedicated deployment ids and routes some short names to another provider's rates. That change and its tests go back to the merge base, so unlisted Fireworks models keep billing $0 behind the proxy as on main. * perf(cost): read the response model's price row without building full model info An unpriced display name like Qwen 3.8 Max made the price check build full model info, and Fireworks' supported params lookup scans the whole cost map for an unknown name. The row check now resolves the cost map key through the cached model info helper, so that pick takes about 0.45 ms instead of several. A regional test covers a Bedrock call whose response names an unpriced display name with no model passed, which bills the requested model's regional row. The duplicate get_model_info wrapper goes, since its twin already does the same thing. * fix(cost): keep the served model's filled-in and size-bucket rates, and cache misses A served row priced only by a fill_missing_for_providers rule bills those rates on main, but the price check read only the raw row and billed the requested alias instead. It now merges the rule's fill-in keys through the helper model info lookup uses. A served Fireworks or Together AI id that names its parameter count keeps its size-bucket price, as on main, instead of switching to the requested row. The served model is checked first, cost map misses are cached and cleared with the model info cache, and the rules are read only for a name that resolved through one. A priced snapshot behind an Azure deployment name takes about 19 us of CPU time against 7 us on the merge base, down from about 300 us. * fix(cost): bill the requested model's listed rates over a served size-bucket guess A served Fireworks or Together AI name that is not in the cost map but names its parameter count, like a display name with a size in it, now bills at the requested model's listed rates instead of the provider's size-bucket price. The bucket price still applies when the requested model has no rates, as on main --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 90 +++-- litellm/utils.py | 23 +- .../test_litellm_logging.py | 43 +++ tests/unit/test_cost_calculator.py | 309 +++++++++++++++++- 4 files changed, 430 insertions(+), 35 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index defe1e63b86..6677b7327dc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -153,7 +153,9 @@ from litellm.utils import ( ProviderConfigManager, TextCompletionResponse, TranscriptionResponse, + cached_cost_map_key, cached_get_model_info_helper, + mapped_model_fill_missing, model_info_with_pricing_overrides, runtime_model_cost_registrations, token_counter, @@ -962,21 +964,17 @@ def select_model_name_for_cost_calc( """ 1. If custom pricing is true, return received model name 2. If base_model is set (e.g. for azure models), return that - 3. If completion response has model set return that + 3. If completion response has model set return that, unless only the requested model's cost-map row has rates 4. Check if model is passed in return that """ return_model: str | None = None custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider) - completion_response_model: str | None = None - if completion_response is not None: - if isinstance(completion_response, BaseModel): - completion_response_model = getattr(completion_response, "model", None) - elif isinstance(completion_response, dict): - completion_response_model = completion_response.get("model", None) + completion_response_model: Final = _get_response_model(completion_response) hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None) provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model") + requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name") explicit_pricing: Final = custom_pricing is True or base_model is not None configured_model: Final = model or ( completion_response_model if isinstance(completion_response_model, str) else None @@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc( else: return_model = model - elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None: - return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model + elif base_model is not None or bedrock_pricing_model is not None: + return_model = base_model if base_model is not None else bedrock_pricing_model + + elif provider_response_model is not None: + return _served_or_priced_requested_model( + provider_response_model, requested_model, custom_llm_provider, priced_region + ) elif completion_response_model is None and hidden_params is not None: if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0: return_model = hidden_params.get("model", model) if return_model is None and completion_response_model is not None: - return_model = completion_response_model + return _served_or_priced_requested_model( + completion_response_model, requested_model, custom_llm_provider, priced_region + ) if return_model is None and model is not None: return_model = model - if ( - return_model is not None - and custom_llm_provider is not None - and not _model_contains_known_llm_provider(return_model) - ): # add provider prefix if not already present, to match model_cost - provider_prefix: Final = ( - custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}" - ) - return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region) - - return return_model + if return_model is None: + return None + return _cost_map_model_name(return_model, custom_llm_provider, priced_region) _select_model_name_for_cost_calc = select_model_name_for_cost_calc +def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str: + if custom_llm_provider is None or _model_contains_known_llm_provider(model): + return model + provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}" + return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name) + + +def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool: + rule_info: Final = unmapped_model_generalization(model, custom_llm_provider) + return rule_info is not None and _cost_map_entry_prices_anything(rule_info) + + +def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool: + key: Final = cached_cost_map_key(model, custom_llm_provider) + if key is None: + return False + entry: Final = _raw_cost_map_entry(key) + if entry is None: + return _rule_declares_rates(model, custom_llm_provider) + fill_missing: Final = mapped_model_fill_missing(key, entry) + return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry}) + + +def _served_or_priced_requested_model( + served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None +) -> str: + served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region) + if requested_model is None: + return served + requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None) + if served in (requested, requested.partition("/")[2]): + return served + pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider) + if _resolves_to_rates(served, pricing_provider): + return served + return requested if _resolves_to_rates(requested, pricing_provider) else served + + def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str: """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already @@ -3516,7 +3551,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a session that should bill nothing fell through to the public rates instead. """ - resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider) + resolved: Final = _cost_map_model_info(model_name, custom_llm_provider) entries: Final = ( litellm.model_cost.get(resolved.get("key")) if resolved is not None else None, litellm.model_cost.get(model_name), @@ -3712,9 +3747,9 @@ def handle_realtime_transcription_cost_calculation( return 0.0 model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name - model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider) + model_info: Final = _cost_map_model_info(model_name, custom_llm_provider) override_info: Final = ( - _get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None + _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None ) total_cost = 0.0 @@ -3745,13 +3780,6 @@ def _get_transcription_model_name_from_results( return None -def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None: - try: - return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - except Exception: - return None - - _TranscriptionRateKey: TypeAlias = Literal[ "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token" ] diff --git a/litellm/utils.py b/litellm/utils.py index d951b80d49b..41a3fddf3a3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -279,7 +279,7 @@ from litellm.types.utils import ( ) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} -_BACKFILL_MODES: Final = frozenset({"chat", "responses"}) +_BACKFILL_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"}) # +-----------------------------------------------+ # | | @@ -6530,6 +6530,7 @@ def _invalidate_model_cost_lowercase_map() -> None: # Clear LRU caches that depend on model_cost data _cached_get_model_info.cache_clear() cached_get_model_info_helper.cache_clear() + cached_cost_map_key.cache_clear() def _rebuild_model_cost_lowercase_map() -> dict[str, str]: @@ -6755,6 +6756,14 @@ def unmapped_model_generalization(model: str, custom_llm_provider: str | None) - return generalization[1] if generalization is not None else None +def mapped_model_fill_missing(key: str, model_info: Mapping[str, object]) -> Mapping[str, object] | None: + mode: Final = model_info.get("mode", "chat") + provider: Final = model_info.get("litellm_provider", "") + if not isinstance(mode, str) or mode not in _BACKFILL_MODES or not isinstance(provider, str): + return None + return match_fill_missing_generalizations(key, provider) + + def _strip_mantle_region_prefix(model: str) -> str: from litellm.llms.bedrock_mantle.common_utils import split_mantle_region_prefix @@ -6883,6 +6892,14 @@ def cached_get_model_info_helper( _cached_get_model_info_helper = cached_get_model_info_helper +@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) +def cached_cost_map_key(model: str, custom_llm_provider: str | None) -> str | None: + try: + return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)["key"] + except Exception: + return None + + def get_provider_info(model: str, custom_llm_provider: str | None) -> ProviderSpecificModelInfo | None: ## PROVIDER-SPECIFIC INFORMATION # if custom_llm_provider == "predibase": @@ -7040,8 +7057,8 @@ def get_model_info_helper( key: str | None = lookup[0] _model_info: dict[str, Any] | None = lookup[1] - if _model_info is not None and key is not None and _model_info.get("mode", "chat") in _BACKFILL_MODES: - fill_missing: Final = match_fill_missing_generalizations(key, _model_info.get("litellm_provider", "")) + if _model_info is not None and key is not None: + fill_missing: Final = mapped_model_fill_missing(key, _model_info) if fill_missing is not None: _model_info = { **{k: v for k, v in fill_missing.items() if k not in _model_info}, diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index c320cb36d73..b3ae3bf6240 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -12320,6 +12320,49 @@ def test_get_model_cost_information_custom_pricing_uses_base_model(): assert result["model_map_key"] != "invoke_test_claude" +@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +def test_get_model_cost_information_maps_requested_model_when_response_names_an_unmapped_display_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + response: Final = ModelResponse(model="Sol Display Max", choices=[]) + response._hidden_params["provider_response_model"] = "Sol Display Max" + response._hidden_params["litellm_model_name"] = requested_model + + result: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=None, custom_pricing=False, custom_llm_provider="fireworks_ai", init_response_obj=response + ) + + assert result["model_map_key"] == requested_model + assert result["model_map_value"] is not None + assert result["model_map_value"]["input_cost_per_token"] == litellm.model_cost[requested_model]["input_cost_per_token"] + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +def test_get_model_cost_information_keeps_a_priced_response_model_that_echoes_without_its_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + served_row: Final = {"litellm_provider": "mistral", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6} + requested_row: Final = {"litellm_provider": "mistral", "mode": "chat", "input_cost_per_token": 9e-6, "output_cost_per_token": 9e-6} + monkeypatch.setitem(litellm.model_cost, "mistral/sol-snapshot-test", served_row) + monkeypatch.setitem(litellm.model_cost, "mistral/sol-alias-test", requested_row) + response: Final = ModelResponse(model="sol-snapshot-test", choices=[]) + response._hidden_params["litellm_model_name"] = "mistral/sol-alias-test" + + result: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=None, custom_pricing=False, custom_llm_provider="mistral", init_response_obj=response + ) + + assert result["model_map_key"] == "sol-snapshot-test" + assert result["model_map_value"] is not None + assert result["model_map_value"]["input_cost_per_token"] == served_row["input_cost_per_token"] + + @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") def test_standard_logging_payload_uses_deployment_when_no_base_model(): """metadata["deployment"] is used for cost-map lookup when base_model is not set.""" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 84f6dab9eff..f10baccd1cb 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -37,7 +37,7 @@ from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject -from litellm.utils import supports_prompt_caching +from litellm.utils import _invalidate_model_cost_lowercase_map, supports_prompt_caching import os import threading from unittest.mock import MagicMock, patch @@ -424,6 +424,313 @@ def test_completion_cost_prices_rule_only_base_model_from_the_response_model( assert cost == pytest.approx(usage.prompt_tokens * 0.0000111 + usage.completion_tokens * 0.0000222) +def _token_rates_cost(row: Mapping[str, object], usage: Usage) -> float: + input_rate: Final = row["input_cost_per_token"] + output_rate: Final = row["output_cost_per_token"] + assert isinstance(input_rate, float) and isinstance(output_rate, float), row + return usage.prompt_tokens * input_rate + usage.completion_tokens * output_rate + + +@pytest.mark.parametrize( + "requested_model", + ["accounts/fireworks/models/sol-display-test", "fireworks_ai/accounts/fireworks/models/sol-display-test"], +) +def test_completion_cost_bills_requested_model_when_response_names_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, requested_model: str +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "fireworks_ai/accounts/fireworks/models/sol-display-test", + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="Sol Display Max", choices=[], usage=usage), + model=requested_model, + custom_llm_provider="fireworks_ai", + ) + + assert cost == pytest.approx( + _token_rates_cost(litellm.model_cost["fireworks_ai/accounts/fireworks/models/sol-display-test"], usage) + ) + + +def test_completion_cost_bills_requested_model_when_stream_reports_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="accounts/fireworks/models/sol-display-test", choices=[], usage=usage) + response._hidden_params["provider_response_model"] = "Sol Display Max" + + cost: Final = completion_cost( + completion_response=response, model=requested_model, custom_llm_provider="fireworks_ai" + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + +def test_completion_cost_bills_requested_model_when_response_row_has_no_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-requested-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-served-snapshot-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": None}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-served-snapshot-test", choices=[], usage=usage), + model="openai/sol-requested-test", + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test"], usage)) + + +def test_completion_cost_bills_priced_response_model_over_priced_requested_model( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-requested-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "openai/sol-requested-test-2099-01-01", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-requested-test-2099-01-01", choices=[], usage=usage), + model="openai/sol-requested-test", + custom_llm_provider="openai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["openai/sol-requested-test-2099-01-01"], usage)) + + +def test_completion_cost_keeps_response_pricing_when_requested_alias_has_no_rates( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + size_bucket: Final = get_base_model_for_pricing("Sol 70B Display") + monkeypatch.setitem( + litellm.model_cost, + size_bucket, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="Sol 70B Display", choices=[], usage=usage), + model="sol-team-alias", + custom_llm_provider="fireworks_ai", + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[size_bucket], usage)) + + +def test_completion_cost_bills_requested_model_priced_only_by_a_rule_when_response_row_has_no_rates( + _local_model_cost_map: None, _priced_claude_family_generalization: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "anthropic/sol-served-snapshot-test", + {"litellm_provider": "anthropic", "mode": "chat", "input_cost_per_token": None, "output_cost_per_token": None}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="sol-served-snapshot-test", choices=[], usage=usage), + model="anthropic/claude-sol-9", + custom_llm_provider="anthropic", + ) + + assert cost == pytest.approx(usage.prompt_tokens * 0.0000321 + usage.completion_tokens * 0.0000765) + + +def test_completion_cost_bills_requested_model_from_hidden_params_when_no_model_is_passed( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-display-test" + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="Sol Display Max", choices=[], usage=usage) + response._hidden_params["litellm_model_name"] = requested_model + + cost: Final = completion_cost(completion_response=response) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + +def test_completion_cost_bills_requested_model_regional_row_when_response_names_an_unmapped_display_name( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "bedrock/sol-regional-test", + {"litellm_provider": "bedrock", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + monkeypatch.setitem( + litellm.model_cost, + "bedrock/eu-central-1/sol-regional-test", + {"litellm_provider": "bedrock", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6}, + ) + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="Sol Regional Display", choices=[], usage=usage) + response._hidden_params.update( + {"litellm_model_name": "bedrock/sol-regional-test", "region_name": "eu-central-1", "custom_llm_provider": "bedrock"} + ) + + cost: Final = completion_cost(completion_response=response) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost["bedrock/eu-central-1/sol-regional-test"], usage)) + + +def test_completion_cost_keeps_served_row_rates_filled_by_a_rule_over_the_requested_alias( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(litellm.model_cost, "sol-fill-snapshot-test", {"litellm_provider": "openai", "mode": "chat"}) + monkeypatch.setitem( + litellm.model_cost, + "sol-fill-alias-test", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + filled_rates: Final = {"input_cost_per_token": 5e-6, "output_cost_per_token": 11e-6} + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response: Final = ModelResponse(model="sol-fill-snapshot-test", choices=[], usage=usage) + response._hidden_params.update({"litellm_model_name": "openai/sol-fill-alias-test", "custom_llm_provider": "openai"}) + previous_rules: Final = list(get_fallback_generalization_rules()) + set_fallback_generalizations( + [ + { + "name": "sol-fill-snapshot-rates", + "pattern": r"^sol-fill-snapshot-test$", + "fill_missing_for_providers": ["openai"], + "model_info": filled_rates, + } + ] + ) + try: + cost: Final = completion_cost(completion_response=response) + finally: + set_fallback_generalizations(previous_rules) + + assert cost == pytest.approx(_token_rates_cost(filled_rates, usage)) + + +_SIZE_BUCKET_CASES: Final = ( + pytest.param( + "fireworks_ai", + "fireworks_ai/accounts/fireworks/models/sol-bucket-requested-test", + "Sol Bucket 405B Display", + "fireworks-ai-above-16b", + id="fireworks_ai", + ), + pytest.param( + "together_ai", + "together_ai/sol-bucket-requested-test", + "meta-llama/Sol-Bucket-8B-Test", + "together-ai-4.1b-8b", + id="together_ai", + ), +) + + +def _size_bucket_response(served_model: str, requested_model: str, custom_llm_provider: str) -> ModelResponse: + response: Final = ModelResponse(model=served_model, choices=[], usage=_SIZE_BUCKET_USAGE) + response._hidden_params.update({"litellm_model_name": requested_model, "custom_llm_provider": custom_llm_provider}) + return response + + +_SIZE_BUCKET_USAGE: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + +@pytest.mark.parametrize(("custom_llm_provider", "requested_model", "served_model", "bucket"), _SIZE_BUCKET_CASES) +def test_completion_cost_bills_requested_listed_rates_over_a_served_size_bucket( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + custom_llm_provider: str, + requested_model: str, + served_model: str, + bucket: str, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + requested_model, + { + "litellm_provider": custom_llm_provider, + "mode": "chat", + "input_cost_per_token": 3e-6, + "output_cost_per_token": 7e-6, + }, + ) + + cost: Final = completion_cost( + completion_response=_size_bucket_response(served_model, requested_model, custom_llm_provider) + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], _SIZE_BUCKET_USAGE)) + assert cost != pytest.approx(_token_rates_cost(litellm.model_cost[bucket], _SIZE_BUCKET_USAGE)) + + +@pytest.mark.parametrize(("custom_llm_provider", "requested_model", "served_model", "bucket"), _SIZE_BUCKET_CASES) +def test_completion_cost_keeps_served_size_bucket_rates_when_the_requested_model_is_unpriced( + _local_model_cost_map: None, + custom_llm_provider: str, + requested_model: str, + served_model: str, + bucket: str, +) -> None: + cost: Final = completion_cost( + completion_response=_size_bucket_response(served_model, requested_model, custom_llm_provider) + ) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[bucket], _SIZE_BUCKET_USAGE)) + + +def test_completion_cost_bills_a_requested_model_priced_after_an_unpriced_lookup( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + requested_model: Final = "fireworks_ai/accounts/fireworks/models/sol-late-priced-test" + usage: Final = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + def display_name_response() -> ModelResponse: + response: Final = ModelResponse(model="Sol Late Display", choices=[], usage=usage) + response._hidden_params.update({"litellm_model_name": requested_model, "custom_llm_provider": "fireworks_ai"}) + return response + + completion_cost(completion_response=display_name_response()) + monkeypatch.setitem( + litellm.model_cost, + requested_model, + {"litellm_provider": "fireworks_ai", "mode": "chat", "input_cost_per_token": 3e-6, "output_cost_per_token": 7e-6}, + ) + _invalidate_model_cost_lowercase_map() + + cost: Final = completion_cost(completion_response=display_name_response()) + + assert cost == pytest.approx(_token_rates_cost(litellm.model_cost[requested_model], usage)) + + def test_completion_cost_orders_candidates_without_extra_huggingface_config_requests( _local_model_cost_map: None, ) -> None: From 85af2f65c9894bd0fa3a354dc6dec96df47d060d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 29 Oct 2026 19:29:44 -0700 Subject: [PATCH 24/25] refactor(helm)!: connect both charts to a separately deployed Lens (#49169) Neither chart bundles Lens anymore. The Lens subchart, its lock entry, the vendored chart archive, the templates/lens resources and the bundled-Lens install smoke are gone. A connection-only lens block wires the gateway and backend to a Lens deployed from its own chart, and an enabled lensWorker value fails the render instead of being ignored. Supersedes #47869, which made the same change for helm/litellm only. --- .github/workflows/lens-install-smoke.yml | 152 ------ helm/litellm-helm/Chart.lock | 7 +- helm/litellm-helm/Chart.yaml | 3 - helm/litellm-helm/README.md | 4 +- helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz | Bin 9226 -> 0 bytes helm/litellm-helm/templates/_helpers.tpl | 97 +--- helm/litellm-helm/templates/ingress.yaml | 12 +- .../templates/lens/clickhouse.yaml | 3 - .../templates/lens/deployment.yaml | 3 - helm/litellm-helm/templates/lens/ingress.yaml | 3 - helm/litellm-helm/templates/lens/secrets.yaml | 3 - helm/litellm-helm/templates/lens/service.yaml | 3 - .../tests/lens_connection_tests.yaml | 128 +++++ .../tests/lens_endpoint_defaults_tests.yaml | 84 --- helm/litellm-helm/tests/lens_modes_tests.yaml | 187 ------- .../tests/lens_modes_validation_tests.yaml | 52 -- .../tests/lens_saved_secrets_tests.yaml | 59 --- .../tests/lens_service_tests.yaml | 167 ------ helm/litellm-helm/tests/lens_setup_tests.yaml | 162 ------ helm/litellm-helm/values.yaml | 69 +-- helm/litellm/Chart.lock | 6 - helm/litellm/Chart.yaml | 5 - helm/litellm/README.md | 6 +- helm/litellm/charts/lens-0.1.0-dev.0.tgz | Bin 9226 -> 0 bytes helm/litellm/templates/_helpers.tpl | 96 +--- helm/litellm/templates/ingress.yaml | 12 +- helm/litellm/templates/lens/clickhouse.yaml | 3 - helm/litellm/templates/lens/deployment.yaml | 3 - helm/litellm/templates/lens/ingress.yaml | 3 - helm/litellm/templates/lens/secrets.yaml | 3 - helm/litellm/templates/lens/service.yaml | 3 - helm/litellm/tests/component_hosts_tests.yaml | 13 +- helm/litellm/tests/lens_connection_tests.yaml | 166 ++++++ .../tests/lens_endpoint_defaults_tests.yaml | 86 ---- helm/litellm/tests/lens_modes_tests.yaml | 208 -------- .../tests/lens_modes_validation_tests.yaml | 54 -- .../tests/lens_saved_secrets_tests.yaml | 61 --- helm/litellm/tests/lens_service_tests.yaml | 196 ------- helm/litellm/tests/lens_setup_tests.yaml | 164 ------ helm/litellm/tests/lens_worker_tests.yaml | 79 --- helm/litellm/values.yaml | 69 +-- tests/e2e/migrations/lens_helm_smoke.sh | 477 ------------------ .../migrations/lens_release_qualification.sh | 284 ----------- .../test_lens_release_qualification.py | 60 --- 44 files changed, 356 insertions(+), 2899 deletions(-) delete mode 100644 .github/workflows/lens-install-smoke.yml delete mode 100644 helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz delete mode 100644 helm/litellm-helm/templates/lens/clickhouse.yaml delete mode 100644 helm/litellm-helm/templates/lens/deployment.yaml delete mode 100644 helm/litellm-helm/templates/lens/ingress.yaml delete mode 100644 helm/litellm-helm/templates/lens/secrets.yaml delete mode 100644 helm/litellm-helm/templates/lens/service.yaml create mode 100644 helm/litellm-helm/tests/lens_connection_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_modes_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_modes_validation_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_saved_secrets_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_service_tests.yaml delete mode 100644 helm/litellm-helm/tests/lens_setup_tests.yaml delete mode 100644 helm/litellm/Chart.lock delete mode 100644 helm/litellm/charts/lens-0.1.0-dev.0.tgz delete mode 100644 helm/litellm/templates/lens/clickhouse.yaml delete mode 100644 helm/litellm/templates/lens/deployment.yaml delete mode 100644 helm/litellm/templates/lens/ingress.yaml delete mode 100644 helm/litellm/templates/lens/secrets.yaml delete mode 100644 helm/litellm/templates/lens/service.yaml create mode 100644 helm/litellm/tests/lens_connection_tests.yaml delete mode 100644 helm/litellm/tests/lens_endpoint_defaults_tests.yaml delete mode 100644 helm/litellm/tests/lens_modes_tests.yaml delete mode 100644 helm/litellm/tests/lens_modes_validation_tests.yaml delete mode 100644 helm/litellm/tests/lens_saved_secrets_tests.yaml delete mode 100644 helm/litellm/tests/lens_service_tests.yaml delete mode 100644 helm/litellm/tests/lens_setup_tests.yaml delete mode 100644 helm/litellm/tests/lens_worker_tests.yaml delete mode 100644 tests/e2e/migrations/lens_helm_smoke.sh delete mode 100644 tests/e2e/migrations/lens_release_qualification.sh delete mode 100644 tests/e2e_harness/migrations/test_lens_release_qualification.py diff --git a/.github/workflows/lens-install-smoke.yml b/.github/workflows/lens-install-smoke.yml deleted file mode 100644 index 9832a69bd90..00000000000 --- a/.github/workflows/lens-install-smoke.yml +++ /dev/null @@ -1,152 +0,0 @@ -name: Lens installation smoke - -on: - workflow_dispatch: - inputs: - lens_image: - description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256 - required: true - type: string - lens_upgrade_image: - description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256 - required: true - type: string - gateway_baseline_ref: - description: Previously qualified gateway source commit, full SHA - required: true - type: string - -permissions: - contents: read - -concurrency: - group: lens-install-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build-images: - runs-on: ubuntu-latest-16-cores - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - component: gateway - dockerfile: gateway/Dockerfile - - component: backend - dockerfile: backend/Dockerfile - - component: ui - dockerfile: ui/Dockerfile - - component: migrations - dockerfile: migrations/Dockerfile - - component: monolith - dockerfile: Dockerfile - - component: gateway - dockerfile: gateway/Dockerfile - baseline: true - - component: backend - dockerfile: backend/Dockerfile - baseline: true - - component: monolith - dockerfile: Dockerfile - baseline: true - env: - COMPONENT: ${{ matrix.component }} - DOCKERFILE: ${{ matrix.dockerfile }} - SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }} - IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }} - CURRENT_REF: ${{ github.sha }} - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - ref: ${{ env.SOURCE_REF }} - - name: Verify the selected source - shell: bash -euo pipefail {0} - run: | - [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]] - test "$(git rev-parse HEAD)" = "$SOURCE_REF" - if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then - test "$SOURCE_REF" != "$CURRENT_REF" - fi - - name: Build the matching release image - run: | - docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \ - -f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" . - - name: Save the matching release image - run: | - docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \ - | gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz" - - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }} - path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz - compression-level: 0 - retention-days: 3 - if-no-files-found: error - overwrite: true - - helm-install: - needs: build-images - runs-on: ubuntu-latest-16-cores - timeout-minutes: 25 - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - with: - pattern: lens-install-*-${{ github.sha }} - merge-multiple: true - path: ${{ runner.temp }}/lens-install-images - - name: Load the matching release images - run: | - for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do - gzip -dc "$archive" | docker load - rm "$archive" - done - - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2 - with: - cosign-release: v2.4.3 - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ vars.LENS_READ_USER || github.actor }} - password: ${{ secrets.LENS_READ_TOKEN }} - - name: Verify independent Lens images before loading them - env: - LENS_IMAGE: ${{ inputs.lens_image }} - LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }} - LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }} - LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }} - shell: bash -euo pipefail {0} - run: | - [[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]] - printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub" - echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check - for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do - [[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]] - cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null - cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null - docker pull "$image" - done - docker tag "$LENS_IMAGE" lens-ci-worker:baseline - docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade - - name: Install pinned Kubernetes test tools - run: | - curl --fail --location --output "$RUNNER_TEMP/kind" \ - https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64 - echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check - chmod +x "$RUNNER_TEMP/kind" - curl --fail --location --output "$RUNNER_TEMP/kubectl" \ - https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl - echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check - chmod +x "$RUNNER_TEMP/kubectl" - curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \ - https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz - echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check - tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP" - echo "$RUNNER_TEMP" >> "$GITHUB_PATH" - echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH" - - name: Install, ingest, upgrade, and restart both charts - run: bash tests/e2e/migrations/lens_helm_smoke.sh diff --git a/helm/litellm-helm/Chart.lock b/helm/litellm-helm/Chart.lock index 220e379ff5a..1e63f4f9eca 100644 --- a/helm/litellm-helm/Chart.lock +++ b/helm/litellm-helm/Chart.lock @@ -5,8 +5,5 @@ dependencies: - name: redis repository: oci://registry-1.docker.io/bitnamicharts version: 18.19.1 -- name: lens - repository: oci://ghcr.io/berriai/charts - version: 0.1.0-dev.0 -digest: sha256:16b92c3d7fc74632e8aa11e602c136d273c774ed39b2fe970b80273e0570898b -generated: '2026-10-09T04:18:10.643228000Z' +digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009 +generated: '2026-10-30T02:13:11.505726000Z' diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml index a3a94241057..a3cb388ffc6 100644 --- a/helm/litellm-helm/Chart.yaml +++ b/helm/litellm-helm/Chart.yaml @@ -39,6 +39,3 @@ dependencies: version: "18.19.1" repository: oci://registry-1.docker.io/bitnamicharts condition: redis.enabled - - name: lens - version: 0.1.0-dev.0 - repository: oci://ghcr.io/berriai/charts diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index a3488d6114c..694cca05b52 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -256,9 +256,7 @@ itself. ## Connect Lens -`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment - -Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published +This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render ## Shutdown accounting diff --git a/helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz b/helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz deleted file mode 100644 index 79021702c2ca9a8eb8f60bfb0ece3616903c7fdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9226 zcmV+lB=y@LiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhciT3$=zP|%z*2iwlCvyImLF+VH)~wSO?_%RzLt|tZ*H#z zky{eR6u|9niO0sq#(o2cA(7p^85JtHqVxD~5ApPR zz25fbCjQ^+^{W5(HrKcQw!YQh?)NvI_cosYt+(D^-(3G2={*QEl_!&miof+fyRYit z{zD#`BwSI&cp?WaLNqx9F;fu7^MJB$D!Aeyj|Zgvyxk&%i)p}9NJ5?{$`TMVU~@W! zK=Mq4Fd#FfQaR{$r%cVVaS-x(_a_j7?H+V>+bx>1w;&|r$$%{SEt;nJ&t9+|^g0nN zf?g{E848vv{IENPL=h!u2y%_EWC2pKsR4S8Ojr`JWGV?wA_5Br7Eh+66@ zg#n>y8nY0;wiZpje(A~h0uQc#jWm!C#sF?u|NZ{+es%qC_WEDf|6@Ec8w)C~24q5G z39U&M$9mnoUg$*|!GN^et?~<8P%M#(CSkdp0L>Fl!SDj<+6?(TrWq+hu9G~!*g;xV|FgfPx zM1X`unYK@|I37YMK*<4l_eYCHbC#U(3rNgc-M3*D*w_VJ4G5O$C|#&Er3xJKI(?p7T-Qv%V zh6co-za4xZYzNzamowVm+8)4o^SkFgcplQ{5IygO{jKi-#*=U|2G|ZK&%b}(TYo;; z9&c@KY=`~t``hF3cmf;IX4L;~^Sf;gNhU!INCXp_#Y*E+ae;vJd#_mWdoQL^8XP5A z93wS3I&2Z5fBu<)RQ4AkVVVuddapOPzvnPV!rNN!A2ECvvpFluyYKy1%){MFs_2+X zw=9Ye%PA`pv0f>V=-GHlujGKNZ#qKI6;dwLfZ(hV;>&09;abn504W=3F{NrYAl()r zVSx?o80oy|FSCWIwnoJFqC zI7^}!qCKx{Ky2Y}HNzu=A1hUgb~RtJvTFLWYaX2-2&r^Dwt&P5Y!`K-HN;8h7PQHsKufl zxQ1A2Y5A7N*<8zcHa~UZ)9QRU>dP4&@8JHt(gn`*3_DL&WwzFph9VW@wh+FR%^`qAWd$<(ZX zwW7fnFx7PX99G?k;=j{*OahMrzboqM5|Bj23khK+n7Z2IiGug4XiH?tt~}<+Dd#Fz zkG)TCP(NMot(RZF5PAIYB?SEVUmffbz8IOdlpf-P7KQ`nawjj3_i_!DX&rYRk>m?;Lb z+zKI);OT(8+dVw|!}|isqFm4oaPRP7@7G^mzZveIzZveIRvJTB#Wx%<5vAOkOAP zRY}Si3^efh8psBiBbt^G)x;7Og(`LCOh2bwR#gmAN3<{5387&KQXcULw2^)aH2O_2 z1+SA3{wPUgHX#W#Q9{*)LlPGIbWnHkG_&_wI}NP3Cis_7Z`EB?g5(AO{C=y7cdm2q zx#@afjoYAB_(FTP*@Zhfbl|13(FD(5VE zvA)wSeiQ{_K0UWsI@c`dMZ*g1hMbFtB{;xt%%?luS~aiBgr;)F)g7C4)#OgMxrL$! zzuvh7rC2hRFFd-~;i+{~om(z>K9?`nd%a$7r`!0=ul#4mMK(Xzt6#p*D}Se3dQ||@ zy#bOPK_}|m@PW*9TI$J8=#H5l7QNV+aj7(+y;$!*4|+i_Sl{X9?{>^w_cu0oy7;G? zQXiz{UG8)n>1gL)oSP3Xb`ll_Kqc5X^{i~1FMozoPj?mQJRVeelcSGGWO z<}}R#supZn?0PG~V-xPr&h=l>=UDasMUck)Y7U8dJpbSG&F3}$-}c5=|KDRgHUD2h z0LK5f+-vkQFWD@@$}-DE7Om&+bH!TO_fc=ld~KKP|4rWmRk)cvS?-4>O)ob(it1gv zDu)dqY@1f^=GD2~=yLDI|5uKe?X9hiexvK9{AJSo`rh3yzGCS7V`ZDB@Wqtp6yKXE zZEzsuICo)g-<^jUI7JNnfqXd5KG75MGmAl9Nd@!D)R6mi_8jgX56_QYzua&3H%LX3 zh{ncmzohobo1YF3_Rim&9xnCfXl0)Xh)DYo5=ru4I$<*Z*3Qvge7?ceb9ivJe|UIw zep@gK_iPcX_g%Ah?qB!2`|{}E`26hkulvUj;Ag5Z@JeEZi}Wmk(>&;p@*-s-ZA{Ze(eYJJAWb$ zyyaNOX0V?*u8M@H=J^EZt?{G!jAi z-B(GI?|=7f_ovZI;UHDOs;_K3|L%9K% zr|Rk@6Xnk45V34NAV-+_S9YKm35MQ^_A?L5rebMr`T2oGu~OFH=UAe({=D?VqLiPL zc(1*=Q{}!P(S!VE_gfl8=s6fa%r_z5H zll%Aol2w3;?MtgijD~Ez#lgxJpO(KUI`xUuY~bC^H&fOr=O$KA4q2GM&%Ni8uFN4_ z<~hHC@%k|%H_&grw`A}mi2%|)o@y-mUkXSg7J8JVNmg*KJcHBU*H(0Mt{a!2${n_6 z5YGdtZ3t7NlcHO`P1+X#X)|S^Dp+bp%3r8C^F!k4uwEufPRsG-YFMCY=e|@vI|95k1tM^s^e~hOtz6}91 z3vf#uUVOz*4@f{=a&f_uX>jo!F8VxI;K0;;_(TEHE`6**?tNUkH>Dm4x0UlR zx{{Vyc1V`Y{-q78*g6=LpU}|2>Ngu&6vsb%(XxdU>7FKQqX)pLL5D za7C`K4U3p~rmZIFH>O6LC(FV8d5 zY$lkgx3bp~0E%D3YoFdSAK;m`iLTl=MEsw*O!^8dzQexc&m2H4lI;L*yLff`nLj}u zyw<_7+6Qs!6A z{@+J=zOmc5jG>F($zLFzw_0a2CNUnW2!{CLSQ0e@67qQ(g91}w_vB!Wyg5KCp>!}S z5oYI%X%Y>xUo=m^lE30UOu`ew6Y!4sv1w-k*(HZe!QtVNJ1-Hdop7n90*3$VkWc~0 zvz*!9YW>Uw;do#yjmZq+luQJllPjJH67on}>~so=S|b{YcSBHwDUzjR!UdVbA5Vjz z8wA~mhq5a{Wobu4-oZM-JbD(81Iu^Rj4GludI~0{8NMjlTR)hCcPrBbWPAgXBm&qFT;&uU> zI=00pHX4sdbdpejR;?I~rItE~$s7gSwOX%JNXV%1JkxH*x}$)+ z9xKWcq_;dF#}N^U`IIFd$q{TKwzNB13f~lz^Z4PR3Ft5(9Zvo!d6H5!`|-&KbnFG* zA+gs_$VW1!61F!ta4#X(+mYv(A2dVjhu49kFCM+RLNir(VuP| z>KU1NMfOTVMMj;D9ogsyJw`UU!tqHqO}wt=G+`6K6d^OxNK27qN|a2X3eff!pe2Sm^~B^% zVb3@cN#_lq{Xi&*IgQAe#xw~*m<}S%oY^>2AoEjH2n_|7(r+W-hNivp5iuhx#Hyir z+{8>33o*xwYpiGymT)a;8&IalnkCX9yVWvIe8mNV6mk)flwN7rrPW*_cn_N9438iq zBmG!1JpwU>GXXF%z{$XAFZp2{;C74}#fNLT227AJ6GE#70v1e@q5J{oL6e7VVhL7A z#6ep1#++EM=Z;Irp$-VDXJeivkv7%vWP89BU2KUpbfCv)_AK!vfe^8*<$p!NYPI%I zED%b5&GU9_gvRY1afD4XHPZ|#M0Vvt|K{`%`_rOf)R<3b*MHN7rI8(!o;PjLfh?^e zubaY2L6EtbSx1_+lWq}wL{sc_o7GaHHET)gVU3!P1{wWOg5h1I4V>^?b2KV} zci>EM+Rih*zHue7WE#8kqnut9Ad?x@w^5CZ!k8~p5@b zUvXjtKQcjAW6e@EqG?VQ2u5S}FNny3!KGY7Wzen>2|^LUFRdh^u~ryubq%M#2v8r4 zfhJk1IeY;(ZdEh_c=zy-ygnv7nt7i)fL;XuJtCJ>l0P#Vvk8NU(5aSl%BKK#&UmXe zG~0X#OPDgCTSSL5m{Ww~QZR*!8%y}NKZ2&-?XDy}8=98|WN!xH#R_F4=2PF!u#&Dp+*^{;<=05a zg@905gRKJWRM13=ZoP;#hXHB|T8V4Qg&MAaIP~TmDFUS%zMKIvUoCB&Kv~834Rq2h zCvzkcw@VgKo-u2oD4z0|Pp=3a^M%!3oUrUTVQFc@G2EnyGzcGKr-&$xA3GAeAxjQg zt&cXk>&D3FG~rUqnUD5(E{8R}cyOXVwmy*+a(U_gd^4o^3>pONclg~F001y}M|mz~lA7{|ocs*^v;4@DX|KJQlHGS0SU zGoc;D$1J&kXc>dr;}MXJZB!;`EIBb3w2XlmA*hx>bH9iGHj#hSBQp?}dFU-_&Nb#+al zNO`yhumF+()>0Ktic`N<<0PeJjVvNE#A?|1DsD|awpu5?;$iuTi=^ba>X`c!k<5rM z0jRs^W~R980+CY2Iwv(#}EY#fP0+N#cqUC~z$DDEfV8!(m zMS?X4%M~@2r-e{sD0vlgYKhUl&~~g@q5|eB#VDJi8-P<=MA5HD@nbbhU-Yse76eas8VB;n=&}A+_owYFd(n2 zK}^sL!C6K{Jp~NU<#xOl+fv4uNJBa^cuzL^qktS}Q?@*=k!=N686M^2Q_2%bMhz!} zjD9dyacO`JEsd5Rx1r5wu4ax>v65A(2~|Yfp^9wuz1=4PIYZtOq?5|Sf9Q-11)4D zkA)+6>FhO7KNL)Dt_6^$UjwJv}Sz_);RMJse$s)ZM3Hn{h%2rvpE>SU_2$pEkvc3 zU3PheZgtsUpvQ)VE+BAGXpRu&hnY#=M=_^yOlU+?rPs1gQF>-aHZU*A$l6w#G95C_ znB7Sk)&|qOHoPjeWZB(_$V;Y|pj0f5$t4#TmeP=j^f2eCy0VMVx!f%}E{uZ?d_H7k zg`tPs!H;@*S@0soc||lWY138fPGn>b1vd0Ia@h2Mx2+6|n-E5X<*U8~CJM|`gLiD` z6;rQM*=ns?fMqwL9S}Mjjmxl@te7owf4xod49KpSan4Mu}OA;GLPJ=PSVae2p zZHM>@f6_`4LuNI zCXBYrLQ5Vy zjT=~%MZUaBLY7Q_AQ4BEG34_(Mrx46B1Z;Raw@JzgR8-u_~3lkvaA`jS|fkw<&Snd z>CTG%o-am3(F@SPL#P){2n0QiC0{kJ_`FXGGkTX4C!o#JoPJy>ZQ>UyGAZ__YPr7v znO8`;&q}idHEW1$Y|U@O)hIQSH}1DnG}CHnLgTo6jpLhNCLr|M&HZTsIpPZplok|R zP!>zjZ5Y=+B5Fk8K!))Rs+U6R9Uq^|%o;CIoK=9*{SA4;Umh&1y0$sYsF=q++sq9?M8L ziFu+KC%;Uj3kcY**U06J#rXslQ6$Y7Ol--xgYp$s4nLcrbvk;zbaD)h)=0?HD|Dt- zPm0j|x%xZb<6CnHmo}4*ipX}(13mvQnvB&g$Riio(i7vL#x!fi3jxjM^Q;T&~RKu(?5tO_XfkfL?&g72FA2_CYFjoH8x+LpP%QA{|3)gb0LG!h7D*g~Bvq@ybR z;Q5>*bhGG{E{e#Yw4NQ3a51N`nddRg=z?)!=iXvLL*eTQm}T*HPZ|pyJzi*U7$w96 zcHG7e$oo(ke%hj*KndT&7Kz2i=IlZrUt>s}_ zt7e5918V47DTP!7%zz3>HMGdUiJR( zt&QH-`~M#0!9_MGO9)TwSqETC`PYPWNI6ZoW$CtX5_z(tV{h)w;~MTtZg;yzhcGqB z*|IEi*_&81i85rRj&@sSi)g;3vwokka|xk!c8Rjt9cx{?)7W_?&G+hkXQj$Z#8;)i z$~81pO7PC`>lhW%-;Qdw)jIz$HtnDHX3j=jJ|vR+biI33O`8`iiPl^T#Zm6pH^CxP z2=lc0z&jAV4%dfOM~GOU*KTpKoEg&f1CPm(c#vb;oRsmdy?hw$KbFqM7np&^fz-P! z@@}C18xTkv0m2;}?|OgWa%$%T<0?+CbB2VWb6Ha(71w{V<%;tW~dbfeOPvauq#+i3Z536aQ7u;U^U}XIzz`F}Ae+l^VOQQZQ zp|gj2eBEmCAN+Xwzh?;EuL*F2{=d=Ntm*&Hzuy1zIFGOY`>aD`kyu>mpMwHeM}ZjX3btapnLw0@=w&5jnuw$!y8n1S=kdi68={U68i;Xi(wg&W5kg)4hkEj0?zP5Yz0I$~n)_dC<+g1H^gh-i@|mF>hPX+U9AglB$UXd~Bb`Npr-K3XMf*1MaskvH1(J9(2Y5Puc-{k`k` z5kRu=D&g@(czki?!n(Ek9d*|a*L~>(!&}_`w)9S<(v3(>x-MpW`Wqh~9;cNHxXRi@ zq3+(%&D#KmiMv&>G$7SYhq~WEFIW$Hod^~|@3Z@@T+vvApfpDhe`O>2;N8+#C_XM_ zk0zzufFh1=c|lcc&J$<`WHaao>!sHw`Z}MJLC36~2 zxR^ua;@l}qYIaTIRAh+~nN$!pBV~enGR{gD*_sE0;Nl3R#CuNf>p=c|WHuykHVVef zyI#$S4zqV7ndVzWouXbHcQx%U=QvJtFB#Lw`N}uVDn~sC;S3 zS1&|NBpHzY;94Nusdw3%>e!tKjo*4a^7Y z<&u`Uab?E}Hi${9H1~CQ&Jb3tqTBmA@-vv52;TW9?B*CujWHWU3}BrLupW? zyHewU2f>{b20X4^&&y1y6i-LjzgDY!_G-e~yL&!6-I{LY#~yH49DG;{k-J>EOF`2V z1g2^8%cRIS$L$Bzxw0Eeyg9n1EbJ8k;FjtlB{{yb`vr>py)rwz3y-VsCm0kv{yKKE z-&JGbUmbl@m5E#MOq0IguEDA+37-~f{|hk4I=(&Tfyn&}aL84&|M31F&;MujjeAf3 zyfOZ}(O<8f|Ji^3)&KVxk01Ya3TK`-9l`^7|Lmp%d&{iRQe{ZB*2@JEFz1Vvs=0-9 zKsI}?*wWA5%`8PK_#HTEEj#E=g1+eTEe>-prfh!d?yf$;C{x`z@Im9^rP;qfAg73^ zHAK)5@#Egz&e^pndUiI7aX6(A-r!}zfanpAKn}>eQ=rjrf+={Ngz!hH;ElL;PjS^T z`QUch56FkoR9@c;KliUez8zyXXJnK<`1c}%3cYiE?BN()v;QE7%tp`Ru%9w|vaxAo zRUzl5%M}ot6KxPF^W<@7yY$6zXuEhG@@xyFH0l63l7@j@hNBFopN^d5EZ@@7k4d zl~T)6suYN1K%VTxe0uH#jIREoyP%>Qv$20*QhuZs<4HH>)17Xu+LQYSO(ryzGp_Ck zpQ|Q!y3H*-S(4oDo%NDu+?VvpmHF$rOXUtZ=;b<euJ%^Y!`q{D(aMF8~1l|CR+&^Z>K~0B!#vcmMzZ diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index e4e6cdd4435..929fc415163 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -372,10 +372,6 @@ through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP. value: {{ .Values.collector.drainTimeoutSeconds | quote }} {{- end -}} -{{- define "litellm.lensWorker.image" -}} -{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}} -{{- end -}} - {{- define "litellm.gateway.collectorSocketDir" -}} {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} @@ -402,95 +398,28 @@ shutdown drain window. {{- end }} {{- end -}} -{{- define "litellm.lensWorker.labels" -}} -{{- $labels := include "litellm.labels" . | fromYaml -}} -{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}} -{{- toYaml $labels -}} -{{- end -}} - -{{- define "litellm.lensWorker.serviceTokenSecretName" -}} -{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}} -{{- end -}} - -{{- define "litellm.lensWorker.gatewaySecretName" -}} -{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}} -{{- end -}} - -{{- define "litellm.lensWorker.bundledClickhouse" -}} -{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.publicUrl" -}} -{{- if .Values.lensWorker.publicUrl -}} -{{- .Values.lensWorker.publicUrl -}} -{{- else if eq (include "litellm.lens.mode" .) "external" -}} -{{- fail "lensWorker.publicUrl is required for external Lens" -}} -{{- else if .Values.lensWorker.ingress.enabled -}} -{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}} -{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}} -{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}} -{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}} -{{- else -}} -{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}} -{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.clickhouseName" -}} -{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}} -{{- end -}} - {{- define "litellm.lensConnectionEnv" -}} -{{- $mode := include "litellm.lens.mode" . -}} -{{- if ne $mode "disabled" }} -{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}} -{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}} +{{- with .Values.lensWorker }} +{{- if or .enabled (and .mode (ne .mode "disabled")) }} +{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }} {{- end }} -{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}} -{{- fail "lensWorker.gateway.secretName is required for external Lens" -}} -{{- end -}} +{{- end }} +{{- with .Values.lens }} +{{- if .enabled }} - name: LITELLM_LENS_URL - value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }} + value: {{ required "lens.url is required when lens.enabled is true" .url | quote }} - name: LITELLM_LENS_PUBLIC_URL - value: {{ include "litellm.lensWorker.publicUrl" . | quote }} + value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }} - name: LENS_GATEWAY_SECRET valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }} - key: {{ .Values.lensWorker.gateway.secretKey | quote }} + name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }} + key: {{ .gatewaySecret.key | quote }} - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} - key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} + name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }} + key: {{ .serviceTokenSecret.key | quote }} +{{- end }} {{- end }} {{- end -}} - -{{- define "litellm.lens.mode" -}} -{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}} -{{- if not (has $mode (list "bundled" "external" "disabled")) -}} -{{- fail "lensWorker.mode must be bundled, external or disabled" -}} -{{- end -}} -{{- $mode -}} -{{- end -}} - -{{- define "litellm.lens.render" -}} -{{- $root := .root -}} -{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}} -{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}} -{{- $_ := set $values "component" "lens-worker" -}} -{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}} -{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}} -{{- $_ := set $values.gateway "enabled" true -}} -{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}} -{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}} -{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}} -{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}} -{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}} -{{- $_ := set $values "ingestionUrl" $ingestion -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}} -{{- end -}} -{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}} -{{- end -}} diff --git a/helm/litellm-helm/templates/ingress.yaml b/helm/litellm-helm/templates/ingress.yaml index 97ebb054b04..3d6e3f4af7a 100644 --- a/helm/litellm-helm/templates/ingress.yaml +++ b/helm/litellm-helm/templates/ingress.yaml @@ -1,8 +1,6 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "litellm.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- $lensMode := include "litellm.lens.mode" . -}} -{{- $lensService := .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" $fullName) -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} @@ -46,18 +44,18 @@ spec: - host: {{ .host | quote }} http: paths: - {{- if or (eq $lensMode "bundled") (and (eq $lensMode "external") $.Values.lensWorker.externalServiceName) }} + {{- if and $.Values.lens.enabled $.Values.lens.service.name }} - path: /lens-ingest pathType: Prefix backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }} + name: {{ $.Values.lens.service.name }} port: - number: {{ $.Values.lensWorker.service.port }} + number: {{ $.Values.lens.service.port }} {{- else }} - serviceName: {{ if eq $lensMode "external" }}{{ $lensService }}{{ else }}{{ $fullName }}-lens-worker{{ end }} - servicePort: {{ $.Values.lensWorker.service.port }} + serviceName: {{ $.Values.lens.service.name }} + servicePort: {{ $.Values.lens.service.port }} {{- end }} {{- end }} {{- range .paths }} diff --git a/helm/litellm-helm/templates/lens/clickhouse.yaml b/helm/litellm-helm/templates/lens/clickhouse.yaml deleted file mode 100644 index d9daafda6ba..00000000000 --- a/helm/litellm-helm/templates/lens/clickhouse.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/deployment.yaml b/helm/litellm-helm/templates/lens/deployment.yaml deleted file mode 100644 index 01a6acb1908..00000000000 --- a/helm/litellm-helm/templates/lens/deployment.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/ingress.yaml b/helm/litellm-helm/templates/lens/ingress.yaml deleted file mode 100644 index b5994b22d92..00000000000 --- a/helm/litellm-helm/templates/lens/ingress.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/secrets.yaml b/helm/litellm-helm/templates/lens/secrets.yaml deleted file mode 100644 index 76928c26e5f..00000000000 --- a/helm/litellm-helm/templates/lens/secrets.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }} -{{- end }} diff --git a/helm/litellm-helm/templates/lens/service.yaml b/helm/litellm-helm/templates/lens/service.yaml deleted file mode 100644 index 60c7e995134..00000000000 --- a/helm/litellm-helm/templates/lens/service.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "service") }} -{{- end }} diff --git a/helm/litellm-helm/tests/lens_connection_tests.yaml b/helm/litellm-helm/tests/lens_connection_tests.yaml new file mode 100644 index 00000000000..e01e49d0ffc --- /dev/null +++ b/helm/litellm-helm/tests/lens_connection_tests.yaml @@ -0,0 +1,128 @@ +suite: lens connection +templates: + - deployment.yaml + - configmap-litellm.yaml + - ingress.yaml +tests: + - it: connects the proxy to an external Lens + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_URL + value: http://lens.lens.svc:4318 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://lens.example.com/lens-ingest + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_GATEWAY_SECRET + valueFrom: + secretKeyRef: + name: lens-gateway + key: gateway-secret + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: lens-service + key: service-token + + - it: requires lens.url when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.url is required when lens.enabled is true + + - it: requires lens.publicUrl when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.publicUrl is required when lens.enabled is true + + - it: requires lens.gatewaySecret.name when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.gatewaySecret.name is required when lens.enabled is true + + - it: requires lens.serviceTokenSecret.name when Lens is enabled + template: deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + asserts: + - failedTemplate: + errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true + + - it: routes /lens-ingest to the named Lens service ahead of the host paths + template: ingress.yaml + set: + ingress.enabled: true + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + lens.service.name: existing-lens + asserts: + - equal: + path: spec.rules[0].http.paths[0].path + value: /lens-ingest + - contains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + pathType: Prefix + backend: + service: + name: existing-lens + port: + number: 4318 + + - it: leaves /lens-ingest off the ingress without a Lens service name + template: ingress.yaml + set: + ingress.enabled: true + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - exists: + path: spec.rules[0].http.paths + - notContains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + any: true diff --git a/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml b/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml deleted file mode 100644 index 22dcc7e9f9c..00000000000 --- a/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml +++ /dev/null @@ -1,84 +0,0 @@ -suite: Lens endpoint defaults -templates: - - deployment.yaml - - configmap-litellm.yaml -set: - lensWorker.enabled: true - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - ingress.tls: - - hosts: [gateway.example] - secretName: tls -tests: - - it: derives the trace endpoint from the deployment hostname and TLS - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - it: preserves an explicitly configured public address - set: - lensWorker.publicUrl: https://custom.example/prefix - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://custom.example/prefix - - it: uses the dedicated Lens ingress when configured - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: - - hosts: [traces.example] - secretName: traces-tls - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTPS for a dedicated ALB ingress with certificate annotations - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.className: alb - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' - alb.ingress.kubernetes.io/ssl-redirect: "443" - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTP for a dedicated ingress without its own TLS - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: {} - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://traces.example - - it: uses HTTP when ingress has no TLS - set: - ingress.tls: [] - template: deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://gateway.example/lens-ingest diff --git a/helm/litellm-helm/tests/lens_modes_tests.yaml b/helm/litellm-helm/tests/lens_modes_tests.yaml deleted file mode 100644 index e746db8f83e..00000000000 --- a/helm/litellm-helm/tests/lens_modes_tests.yaml +++ /dev/null @@ -1,187 +0,0 @@ ---- -suite: Lens deployment modes -templates: -- ingress.yaml -- configmap-litellm.yaml -- deployment.yaml -- lens/deployment.yaml -- lens/service.yaml -- lens/clickhouse.yaml -- lens/secrets.yaml -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: retains the existing deployment before transferring its release ownership - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.retainResources: true - asserts: - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep -- it: connects an external deployment without installing Lens resources - templates: - - lens/deployment.yaml - - lens/service.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - hasDocuments: - count: 0 -- it: sends delegated requests to the explicitly selected external Lens - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens.other-namespace:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: external-lens-signing - key: gateway-secret -- it: disables Lens explicitly even with a retained legacy enable flag - template: lens/deployment.yaml - set: - lensWorker.mode: disabled - lensWorker.enabled: true - asserts: - - hasDocuments: - count: 0 -- it: bundles Lens through explicit mode selection - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_MODE - value: standalone -- it: uses separate credentials for external identity and service access - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.serviceTokenSecret: {name: worker-service, key: token} - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: {name: worker-service, key: token} -- it: supplies the same separate signing secret to the bundled service - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} -- it: routes explicit bundled mode with the enable flag off - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: bundled - lensWorker.enabled: false - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: gateway-lens-worker - port: {number: 4318} -- it: routes external mode to its selected service - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "existing-lens" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: {number: 4318} -- it: omits ingestion when an external service has no ingress backend - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true -- it: omits ingestion for disabled mode with the enable flag on - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] - lensWorker.mode: disabled - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true diff --git a/helm/litellm-helm/tests/lens_modes_validation_tests.yaml b/helm/litellm-helm/tests/lens_modes_validation_tests.yaml deleted file mode 100644 index eaf7d58f6ac..00000000000 --- a/helm/litellm-helm/tests/lens_modes_validation_tests.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -suite: Lens deployment mode validation -templates: -- configmap-litellm.yaml -- deployment.yaml -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: requires a separate identity signing reference for external Lens - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: worker-service - asserts: - - failedTemplate: - errorMessage: lensWorker.gateway.secretName is required for external Lens -- it: requires an external service address - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - failedTemplate: - errorMessage: lensWorker.externalUrl is required for external Lens -- it: requires the shared server credential for an external service - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - asserts: - - failedTemplate: - errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens -- it: requires an agent reachable URL for external ingestion - template: deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - lensWorker.publicUrl: '' - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required for external Lens -- it: rejects unsupported deployment modes - template: deployment.yaml - set: - lensWorker.mode: typo - asserts: - - failedTemplate: - errorMessage: lensWorker.mode must be bundled, external or disabled diff --git a/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml b/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml deleted file mode 100644 index c85f8d8c9e7..00000000000 --- a/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml +++ /dev/null @@ -1,59 +0,0 @@ -suite: Lens preserves credentials across Helm upgrades -templates: - - lens/secrets.yaml -set: - fullnameOverride: lens-test - lensWorker.enabled: true -release: - namespace: lens - name: lens-test - upgrade: true -kubernetesProvider: - scheme: - v1/Secret: - gvr: - version: v1 - resource: secrets - namespaced: true - objects: - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-gateway - namespace: lens - data: - gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk= - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-service - namespace: lens - data: - service-token: c2F2ZWQtc2VydmljZS10b2tlbg== - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-clickhouse - namespace: lens - data: - password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= -tests: - - it: preserves the separate signing credential across upgrades - documentSelector: {path: metadata.name, value: lens-test-lens-gateway} - asserts: - - equal: - path: data.gateway-secret - value: c2F2ZWQtc2lnbmluZy1rZXk= - - notExists: {path: data.service-token} - - it: reuses the service credential instead of breaking running services - documentIndex: 0 - asserts: - - equal: - path: data.service-token - value: c2F2ZWQtc2VydmljZS10b2tlbg== - - it: reuses the database password instead of locking out stored traces - documentIndex: 1 - asserts: - - equal: - path: data.password - value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= diff --git a/helm/litellm-helm/tests/lens_service_tests.yaml b/helm/litellm-helm/tests/lens_service_tests.yaml deleted file mode 100644 index 1beabca3064..00000000000 --- a/helm/litellm-helm/tests/lens_service_tests.yaml +++ /dev/null @@ -1,167 +0,0 @@ -suite: Lens service isolation and ingestion routing -templates: -- configmap-litellm.yaml -- deployment.yaml -- ingress.yaml -- lens/ingress.yaml -- lens/service.yaml -- lens/deployment.yaml -tests: -- it: connects deployment.yaml to the shared Lens service - template: deployment.yaml - set: &id001 - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens-test-lens-worker:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-test-lens-gateway - key: gateway-secret -- it: routes uploads directly to Lens instead of the gateway - template: ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - ingress.enabled: true - ingress.hosts: - - host: gateway.example - paths: - - path: / - pathType: Prefix - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - number: 4318 -- it: keeps internal routes out of a dedicated ingestion hostname - template: lens/ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - asserts: - - equal: - path: spec.rules[0].http.paths - value: - - path: /v1/ - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - name: otlp -- it: maps the Lens service to the ingestion listener - template: lens/service.yaml - set: *id001 - asserts: - - equal: - path: spec.selector - value: - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: lens-worker - - equal: - path: spec.ports - value: - - name: otlp - port: 4318 - targetPort: otlp -- it: gives only Lens the ClickHouse secret - template: lens/deployment.yaml - set: *id001 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_URL - valueFrom: - secretKeyRef: - name: lens-storage - key: url -- it: requires an agent reachable ingestion URL - template: deployment.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: '' - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname -- it: omits Lens connection settings when disabled in deployment.yaml - template: deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - any: true -- it: preserves an existing ClickHouse database and retention - template: lens/deployment.yaml - set: - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.clickhouseDatabase: existing_traces - lensWorker.retentionDays: 45 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_DATABASE - value: existing_traces - - contains: - path: spec.template.spec.containers[0].env - content: - name: AGENT_TRACING_RETENTION_DAYS - value: '45' -- it: keeps Lens pods outside the gateway autoscaling selector - template: lens/deployment.yaml - set: &id002 - nameOverride: inference - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/name"] - value: inference-lens-worker -- it: preserves the existing gateway deployment selector - template: deployment.yaml - set: *id002 - asserts: - - equal: - path: spec.selector.matchLabels["app.kubernetes.io/name"] - value: inference diff --git a/helm/litellm-helm/tests/lens_setup_tests.yaml b/helm/litellm-helm/tests/lens_setup_tests.yaml deleted file mode 100644 index 20cd8590153..00000000000 --- a/helm/litellm-helm/tests/lens_setup_tests.yaml +++ /dev/null @@ -1,162 +0,0 @@ -suite: Lens managed setup -set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example -release: - namespace: lens - name: lens-test -templates: - - lens/deployment.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml -tests: - - it: generates private login, gateway and storage credentials - template: lens/secrets.yaml - asserts: - - hasDocuments: - count: 4 - - matchRegex: - path: data.service-token - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 0 - - matchRegex: - path: data.password - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 1 - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - - it: connects Lens to its private bundled storage - template: lens/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: lens-test-lens-clickhouse - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: stores traces on a persistent volume with the chosen storage class - template: lens/clickhouse.yaml - documentIndex: 1 - set: - lensWorker.clickhouse.storage: 40Gi - lensWorker.clickhouse.storageClassName: fast - asserts: - - equal: - path: spec.volumeClaimTemplates[0].spec - value: - accessModes: [ReadWriteOnce] - storageClassName: fast - resources: - requests: - storage: 40Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: keeps an existing external database instead of creating a new one - template: lens/clickhouse.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - asserts: - - hasDocuments: - count: 0 - - it: leaves supplied secrets under their existing manager - template: lens/secrets.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - lensWorker.serviceTokenSecret.name: external-service - lensWorker.adminTokenSecret.name: external-admin - lensWorker.gateway.secretName: external-signing - asserts: - - hasDocuments: - count: 0 - - it: creates no credentials or database when Lens is disabled - templates: - - lens/secrets.yaml - - lens/clickhouse.yaml - set: - lensWorker.enabled: false - asserts: - - hasDocuments: - count: 0 - - it: requires an external database when bundled storage is explicitly disabled - template: lens/deployment.yaml - set: - lensWorker.clickhouse.enabled: false - asserts: - - failedTemplate: - errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled - - it: keeps storage names valid and uses the same name for the connection - set: - fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - asserts: - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 0 - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 0 - - equal: - path: spec.serviceName - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/secrets.yaml - documentIndex: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/deployment.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - key: password - template: lens/deployment.yaml - - - it: allows cold storage startup and inherits registry credentials - template: lens/clickhouse.yaml - documentIndex: 1 - set: - imagePullSecrets: [{name: registry-auth}] - asserts: - - equal: - path: spec.template.spec.imagePullSecrets - value: [{name: registry-auth}] - - equal: - path: spec.template.spec.containers[0].startupProbe - value: - httpGet: {path: /ping, port: http} - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 60 diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 8450c5f2840..f51f8bdd173 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -668,70 +668,15 @@ serviceMonitor: # - test-namespace lens: - library: true - -lensWorker: - retainResources: false - mode: "" - externalUrl: "" - externalServiceName: "" - gateway: - secretName: "" - secretKey: gateway-secret - standaloneUrl: "" - adminTokenSecret: - name: "" - key: admin-token - extraEnv: [] - extraEnvFrom: [] enabled: false - replicaCount: 1 - image: - repository: ghcr.io/berriai/lens - tag: "0.1.0-dev.0" - digest: "" - pullPolicy: IfNotPresent - tokenSecret: + url: "" + publicUrl: "" + service: name: "" - key: token + port: 4318 + gatewaySecret: + name: "" + key: gateway-secret serviceTokenSecret: name: "" key: service-token - clickhouse: - enabled: true - image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e - storage: 20Gi - storageClassName: null - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 2Gi - clickhouseDatabase: litellm - retentionDays: 14 - clickhouseSecret: - name: "" - key: url - publicUrl: "" - service: - port: 4318 - annotations: {} - ingress: - enabled: false - path: /v1/ - className: "" - host: "" - annotations: {} - tls: [] - url: "" - tmpSizeLimit: 1Gi - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 2Gi - nodeSelector: {} - tolerations: [] - affinity: {} diff --git a/helm/litellm/Chart.lock b/helm/litellm/Chart.lock deleted file mode 100644 index d9f261935ca..00000000000 --- a/helm/litellm/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: lens - repository: oci://ghcr.io/berriai/charts - version: 0.1.0-dev.0 -digest: sha256:6b244c878890f10a093a297d70c9e68b2c3bda00fbbb56ab287e7ecee1d7bbcf -generated: '2026-10-09T04:18:10.642706000Z' diff --git a/helm/litellm/Chart.yaml b/helm/litellm/Chart.yaml index 36780b808dc..e67f5790c7e 100644 --- a/helm/litellm/Chart.yaml +++ b/helm/litellm/Chart.yaml @@ -6,8 +6,3 @@ version: 0.1.0 appVersion: "0.1.0" annotations: org.opencontainers.image.source: "https://github.com/BerriAI/litellm" - -dependencies: - - name: lens - version: 0.1.0-dev.0 - repository: oci://ghcr.io/berriai/charts diff --git a/helm/litellm/README.md b/helm/litellm/README.md index 271052dd09f..97b94754402 100644 --- a/helm/litellm/README.md +++ b/helm/litellm/README.md @@ -1,10 +1,8 @@ # LiteLLM componentized chart -This chart deploys separate gateway, backend and UI services. Configure Lens through `lensWorker.mode`: `disabled` keeps it off, `bundled` installs the Lens dependency, and `external` connects an existing Lens service +This chart deploys separate gateway, backend and UI services -Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment - -Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for the complete values and verification flow. It also links the GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published +This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render For an ingress with separate component hostnames, set all three `ingress.componentHosts` values, leave `ingress.host` and `ingress.extraPaths` empty, configure `ui.backendUrl` as the backend's public origin and `backend.gatewayUrl` as the gateway's public origin, and set `ui.publicUrl` to the browser-facing UI origin. The UI Deployment receives `LITELLM_BACKEND_URL` from `ui.backendUrl` only when `ingress.componentHosts` is configured. diff --git a/helm/litellm/charts/lens-0.1.0-dev.0.tgz b/helm/litellm/charts/lens-0.1.0-dev.0.tgz deleted file mode 100644 index 79021702c2ca9a8eb8f60bfb0ece3616903c7fdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9226 zcmV+lB=y@LiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhciT3$=zP|%z*2iwlCvyImLF+VH)~wSO?_%RzLt|tZ*H#z zky{eR6u|9niO0sq#(o2cA(7p^85JtHqVxD~5ApPR zz25fbCjQ^+^{W5(HrKcQw!YQh?)NvI_cosYt+(D^-(3G2={*QEl_!&miof+fyRYit z{zD#`BwSI&cp?WaLNqx9F;fu7^MJB$D!Aeyj|Zgvyxk&%i)p}9NJ5?{$`TMVU~@W! zK=Mq4Fd#FfQaR{$r%cVVaS-x(_a_j7?H+V>+bx>1w;&|r$$%{SEt;nJ&t9+|^g0nN zf?g{E848vv{IENPL=h!u2y%_EWC2pKsR4S8Ojr`JWGV?wA_5Br7Eh+66@ zg#n>y8nY0;wiZpje(A~h0uQc#jWm!C#sF?u|NZ{+es%qC_WEDf|6@Ec8w)C~24q5G z39U&M$9mnoUg$*|!GN^et?~<8P%M#(CSkdp0L>Fl!SDj<+6?(TrWq+hu9G~!*g;xV|FgfPx zM1X`unYK@|I37YMK*<4l_eYCHbC#U(3rNgc-M3*D*w_VJ4G5O$C|#&Er3xJKI(?p7T-Qv%V zh6co-za4xZYzNzamowVm+8)4o^SkFgcplQ{5IygO{jKi-#*=U|2G|ZK&%b}(TYo;; z9&c@KY=`~t``hF3cmf;IX4L;~^Sf;gNhU!INCXp_#Y*E+ae;vJd#_mWdoQL^8XP5A z93wS3I&2Z5fBu<)RQ4AkVVVuddapOPzvnPV!rNN!A2ECvvpFluyYKy1%){MFs_2+X zw=9Ye%PA`pv0f>V=-GHlujGKNZ#qKI6;dwLfZ(hV;>&09;abn504W=3F{NrYAl()r zVSx?o80oy|FSCWIwnoJFqC zI7^}!qCKx{Ky2Y}HNzu=A1hUgb~RtJvTFLWYaX2-2&r^Dwt&P5Y!`K-HN;8h7PQHsKufl zxQ1A2Y5A7N*<8zcHa~UZ)9QRU>dP4&@8JHt(gn`*3_DL&WwzFph9VW@wh+FR%^`qAWd$<(ZX zwW7fnFx7PX99G?k;=j{*OahMrzboqM5|Bj23khK+n7Z2IiGug4XiH?tt~}<+Dd#Fz zkG)TCP(NMot(RZF5PAIYB?SEVUmffbz8IOdlpf-P7KQ`nawjj3_i_!DX&rYRk>m?;Lb z+zKI);OT(8+dVw|!}|isqFm4oaPRP7@7G^mzZveIzZveIRvJTB#Wx%<5vAOkOAP zRY}Si3^efh8psBiBbt^G)x;7Og(`LCOh2bwR#gmAN3<{5387&KQXcULw2^)aH2O_2 z1+SA3{wPUgHX#W#Q9{*)LlPGIbWnHkG_&_wI}NP3Cis_7Z`EB?g5(AO{C=y7cdm2q zx#@afjoYAB_(FTP*@Zhfbl|13(FD(5VE zvA)wSeiQ{_K0UWsI@c`dMZ*g1hMbFtB{;xt%%?luS~aiBgr;)F)g7C4)#OgMxrL$! zzuvh7rC2hRFFd-~;i+{~om(z>K9?`nd%a$7r`!0=ul#4mMK(Xzt6#p*D}Se3dQ||@ zy#bOPK_}|m@PW*9TI$J8=#H5l7QNV+aj7(+y;$!*4|+i_Sl{X9?{>^w_cu0oy7;G? zQXiz{UG8)n>1gL)oSP3Xb`ll_Kqc5X^{i~1FMozoPj?mQJRVeelcSGGWO z<}}R#supZn?0PG~V-xPr&h=l>=UDasMUck)Y7U8dJpbSG&F3}$-}c5=|KDRgHUD2h z0LK5f+-vkQFWD@@$}-DE7Om&+bH!TO_fc=ld~KKP|4rWmRk)cvS?-4>O)ob(it1gv zDu)dqY@1f^=GD2~=yLDI|5uKe?X9hiexvK9{AJSo`rh3yzGCS7V`ZDB@Wqtp6yKXE zZEzsuICo)g-<^jUI7JNnfqXd5KG75MGmAl9Nd@!D)R6mi_8jgX56_QYzua&3H%LX3 zh{ncmzohobo1YF3_Rim&9xnCfXl0)Xh)DYo5=ru4I$<*Z*3Qvge7?ceb9ivJe|UIw zep@gK_iPcX_g%Ah?qB!2`|{}E`26hkulvUj;Ag5Z@JeEZi}Wmk(>&;p@*-s-ZA{Ze(eYJJAWb$ zyyaNOX0V?*u8M@H=J^EZt?{G!jAi z-B(GI?|=7f_ovZI;UHDOs;_K3|L%9K% zr|Rk@6Xnk45V34NAV-+_S9YKm35MQ^_A?L5rebMr`T2oGu~OFH=UAe({=D?VqLiPL zc(1*=Q{}!P(S!VE_gfl8=s6fa%r_z5H zll%Aol2w3;?MtgijD~Ez#lgxJpO(KUI`xUuY~bC^H&fOr=O$KA4q2GM&%Ni8uFN4_ z<~hHC@%k|%H_&grw`A}mi2%|)o@y-mUkXSg7J8JVNmg*KJcHBU*H(0Mt{a!2${n_6 z5YGdtZ3t7NlcHO`P1+X#X)|S^Dp+bp%3r8C^F!k4uwEufPRsG-YFMCY=e|@vI|95k1tM^s^e~hOtz6}91 z3vf#uUVOz*4@f{=a&f_uX>jo!F8VxI;K0;;_(TEHE`6**?tNUkH>Dm4x0UlR zx{{Vyc1V`Y{-q78*g6=LpU}|2>Ngu&6vsb%(XxdU>7FKQqX)pLL5D za7C`K4U3p~rmZIFH>O6LC(FV8d5 zY$lkgx3bp~0E%D3YoFdSAK;m`iLTl=MEsw*O!^8dzQexc&m2H4lI;L*yLff`nLj}u zyw<_7+6Qs!6A z{@+J=zOmc5jG>F($zLFzw_0a2CNUnW2!{CLSQ0e@67qQ(g91}w_vB!Wyg5KCp>!}S z5oYI%X%Y>xUo=m^lE30UOu`ew6Y!4sv1w-k*(HZe!QtVNJ1-Hdop7n90*3$VkWc~0 zvz*!9YW>Uw;do#yjmZq+luQJllPjJH67on}>~so=S|b{YcSBHwDUzjR!UdVbA5Vjz z8wA~mhq5a{Wobu4-oZM-JbD(81Iu^Rj4GludI~0{8NMjlTR)hCcPrBbWPAgXBm&qFT;&uU> zI=00pHX4sdbdpejR;?I~rItE~$s7gSwOX%JNXV%1JkxH*x}$)+ z9xKWcq_;dF#}N^U`IIFd$q{TKwzNB13f~lz^Z4PR3Ft5(9Zvo!d6H5!`|-&KbnFG* zA+gs_$VW1!61F!ta4#X(+mYv(A2dVjhu49kFCM+RLNir(VuP| z>KU1NMfOTVMMj;D9ogsyJw`UU!tqHqO}wt=G+`6K6d^OxNK27qN|a2X3eff!pe2Sm^~B^% zVb3@cN#_lq{Xi&*IgQAe#xw~*m<}S%oY^>2AoEjH2n_|7(r+W-hNivp5iuhx#Hyir z+{8>33o*xwYpiGymT)a;8&IalnkCX9yVWvIe8mNV6mk)flwN7rrPW*_cn_N9438iq zBmG!1JpwU>GXXF%z{$XAFZp2{;C74}#fNLT227AJ6GE#70v1e@q5J{oL6e7VVhL7A z#6ep1#++EM=Z;Irp$-VDXJeivkv7%vWP89BU2KUpbfCv)_AK!vfe^8*<$p!NYPI%I zED%b5&GU9_gvRY1afD4XHPZ|#M0Vvt|K{`%`_rOf)R<3b*MHN7rI8(!o;PjLfh?^e zubaY2L6EtbSx1_+lWq}wL{sc_o7GaHHET)gVU3!P1{wWOg5h1I4V>^?b2KV} zci>EM+Rih*zHue7WE#8kqnut9Ad?x@w^5CZ!k8~p5@b zUvXjtKQcjAW6e@EqG?VQ2u5S}FNny3!KGY7Wzen>2|^LUFRdh^u~ryubq%M#2v8r4 zfhJk1IeY;(ZdEh_c=zy-ygnv7nt7i)fL;XuJtCJ>l0P#Vvk8NU(5aSl%BKK#&UmXe zG~0X#OPDgCTSSL5m{Ww~QZR*!8%y}NKZ2&-?XDy}8=98|WN!xH#R_F4=2PF!u#&Dp+*^{;<=05a zg@905gRKJWRM13=ZoP;#hXHB|T8V4Qg&MAaIP~TmDFUS%zMKIvUoCB&Kv~834Rq2h zCvzkcw@VgKo-u2oD4z0|Pp=3a^M%!3oUrUTVQFc@G2EnyGzcGKr-&$xA3GAeAxjQg zt&cXk>&D3FG~rUqnUD5(E{8R}cyOXVwmy*+a(U_gd^4o^3>pONclg~F001y}M|mz~lA7{|ocs*^v;4@DX|KJQlHGS0SU zGoc;D$1J&kXc>dr;}MXJZB!;`EIBb3w2XlmA*hx>bH9iGHj#hSBQp?}dFU-_&Nb#+al zNO`yhumF+()>0Ktic`N<<0PeJjVvNE#A?|1DsD|awpu5?;$iuTi=^ba>X`c!k<5rM z0jRs^W~R980+CY2Iwv(#}EY#fP0+N#cqUC~z$DDEfV8!(m zMS?X4%M~@2r-e{sD0vlgYKhUl&~~g@q5|eB#VDJi8-P<=MA5HD@nbbhU-Yse76eas8VB;n=&}A+_owYFd(n2 zK}^sL!C6K{Jp~NU<#xOl+fv4uNJBa^cuzL^qktS}Q?@*=k!=N686M^2Q_2%bMhz!} zjD9dyacO`JEsd5Rx1r5wu4ax>v65A(2~|Yfp^9wuz1=4PIYZtOq?5|Sf9Q-11)4D zkA)+6>FhO7KNL)Dt_6^$UjwJv}Sz_);RMJse$s)ZM3Hn{h%2rvpE>SU_2$pEkvc3 zU3PheZgtsUpvQ)VE+BAGXpRu&hnY#=M=_^yOlU+?rPs1gQF>-aHZU*A$l6w#G95C_ znB7Sk)&|qOHoPjeWZB(_$V;Y|pj0f5$t4#TmeP=j^f2eCy0VMVx!f%}E{uZ?d_H7k zg`tPs!H;@*S@0soc||lWY138fPGn>b1vd0Ia@h2Mx2+6|n-E5X<*U8~CJM|`gLiD` z6;rQM*=ns?fMqwL9S}Mjjmxl@te7owf4xod49KpSan4Mu}OA;GLPJ=PSVae2p zZHM>@f6_`4LuNI zCXBYrLQ5Vy zjT=~%MZUaBLY7Q_AQ4BEG34_(Mrx46B1Z;Raw@JzgR8-u_~3lkvaA`jS|fkw<&Snd z>CTG%o-am3(F@SPL#P){2n0QiC0{kJ_`FXGGkTX4C!o#JoPJy>ZQ>UyGAZ__YPr7v znO8`;&q}idHEW1$Y|U@O)hIQSH}1DnG}CHnLgTo6jpLhNCLr|M&HZTsIpPZplok|R zP!>zjZ5Y=+B5Fk8K!))Rs+U6R9Uq^|%o;CIoK=9*{SA4;Umh&1y0$sYsF=q++sq9?M8L ziFu+KC%;Uj3kcY**U06J#rXslQ6$Y7Ol--xgYp$s4nLcrbvk;zbaD)h)=0?HD|Dt- zPm0j|x%xZb<6CnHmo}4*ipX}(13mvQnvB&g$Riio(i7vL#x!fi3jxjM^Q;T&~RKu(?5tO_XfkfL?&g72FA2_CYFjoH8x+LpP%QA{|3)gb0LG!h7D*g~Bvq@ybR z;Q5>*bhGG{E{e#Yw4NQ3a51N`nddRg=z?)!=iXvLL*eTQm}T*HPZ|pyJzi*U7$w96 zcHG7e$oo(ke%hj*KndT&7Kz2i=IlZrUt>s}_ zt7e5918V47DTP!7%zz3>HMGdUiJR( zt&QH-`~M#0!9_MGO9)TwSqETC`PYPWNI6ZoW$CtX5_z(tV{h)w;~MTtZg;yzhcGqB z*|IEi*_&81i85rRj&@sSi)g;3vwokka|xk!c8Rjt9cx{?)7W_?&G+hkXQj$Z#8;)i z$~81pO7PC`>lhW%-;Qdw)jIz$HtnDHX3j=jJ|vR+biI33O`8`iiPl^T#Zm6pH^CxP z2=lc0z&jAV4%dfOM~GOU*KTpKoEg&f1CPm(c#vb;oRsmdy?hw$KbFqM7np&^fz-P! z@@}C18xTkv0m2;}?|OgWa%$%T<0?+CbB2VWb6Ha(71w{V<%;tW~dbfeOPvauq#+i3Z536aQ7u;U^U}XIzz`F}Ae+l^VOQQZQ zp|gj2eBEmCAN+Xwzh?;EuL*F2{=d=Ntm*&Hzuy1zIFGOY`>aD`kyu>mpMwHeM}ZjX3btapnLw0@=w&5jnuw$!y8n1S=kdi68={U68i;Xi(wg&W5kg)4hkEj0?zP5Yz0I$~n)_dC<+g1H^gh-i@|mF>hPX+U9AglB$UXd~Bb`Npr-K3XMf*1MaskvH1(J9(2Y5Puc-{k`k` z5kRu=D&g@(czki?!n(Ek9d*|a*L~>(!&}_`w)9S<(v3(>x-MpW`Wqh~9;cNHxXRi@ zq3+(%&D#KmiMv&>G$7SYhq~WEFIW$Hod^~|@3Z@@T+vvApfpDhe`O>2;N8+#C_XM_ zk0zzufFh1=c|lcc&J$<`WHaao>!sHw`Z}MJLC36~2 zxR^ua;@l}qYIaTIRAh+~nN$!pBV~enGR{gD*_sE0;Nl3R#CuNf>p=c|WHuykHVVef zyI#$S4zqV7ndVzWouXbHcQx%U=QvJtFB#Lw`N}uVDn~sC;S3 zS1&|NBpHzY;94Nusdw3%>e!tKjo*4a^7Y z<&u`Uab?E}Hi${9H1~CQ&Jb3tqTBmA@-vv52;TW9?B*CujWHWU3}BrLupW? zyHewU2f>{b20X4^&&y1y6i-LjzgDY!_G-e~yL&!6-I{LY#~yH49DG;{k-J>EOF`2V z1g2^8%cRIS$L$Bzxw0Eeyg9n1EbJ8k;FjtlB{{yb`vr>py)rwz3y-VsCm0kv{yKKE z-&JGbUmbl@m5E#MOq0IguEDA+37-~f{|hk4I=(&Tfyn&}aL84&|M31F&;MujjeAf3 zyfOZ}(O<8f|Ji^3)&KVxk01Ya3TK`-9l`^7|Lmp%d&{iRQe{ZB*2@JEFz1Vvs=0-9 zKsI}?*wWA5%`8PK_#HTEEj#E=g1+eTEe>-prfh!d?yf$;C{x`z@Im9^rP;qfAg73^ zHAK)5@#Egz&e^pndUiI7aX6(A-r!}zfanpAKn}>eQ=rjrf+={Ngz!hH;ElL;PjS^T z`QUch56FkoR9@c;KliUez8zyXXJnK<`1c}%3cYiE?BN()v;QE7%tp`Ru%9w|vaxAo zRUzl5%M}ot6KxPF^W<@7yY$6zXuEhG@@xyFH0l63l7@j@hNBFopN^d5EZ@@7k4d zl~T)6suYN1K%VTxe0uH#jIREoyP%>Qv$20*QhuZs<4HH>)17Xu+LQYSO(ryzGp_Ck zpQ|Q!y3H*-S(4oDo%NDu+?VvpmHF$rOXUtZ=;b<euJ%^Y!`q{D(aMF8~1l|CR+&^Z>K~0B!#vcmMzZ diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 8cb2fbf441d..18e938564e9 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -604,10 +604,6 @@ Directory of the collector's unix socket, shared by the gateway and collector containers through an emptyDir. Empty when the sidecar is off or gateway.collector.address is a tcp://127.0.0.1: address. */}} -{{- define "litellm.lensWorker.image" -}} -{{- include "lens.image" (dict "Values" .Values.lensWorker "Chart" .Subcharts.lens.Chart) -}} -{{- end -}} - {{- define "litellm.gateway.collectorSocketDir" -}} {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} @@ -635,93 +631,27 @@ shutdown drain window. {{- end -}} {{- define "litellm.lensConnectionEnv" -}} -{{- $mode := include "litellm.lens.mode" . -}} -{{- if ne $mode "disabled" }} -{{- if and (eq $mode "external") (not .Values.lensWorker.serviceTokenSecret.name) -}} -{{- fail "lensWorker.serviceTokenSecret.name is required for external Lens" -}} +{{- with .Values.lensWorker }} +{{- if or .enabled (and .mode (ne .mode "disabled")) }} +{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }} {{- end }} -{{- if and (eq $mode "external") (not .Values.lensWorker.gateway.secretName) -}} -{{- fail "lensWorker.gateway.secretName is required for external Lens" -}} -{{- end -}} +{{- end }} +{{- with .Values.lens }} +{{- if .enabled }} - name: LITELLM_LENS_URL - value: {{ if eq $mode "external" }}{{ required "lensWorker.externalUrl is required for external Lens" .Values.lensWorker.externalUrl | quote }}{{ else }}{{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }}{{ end }} + value: {{ required "lens.url is required when lens.enabled is true" .url | quote }} - name: LITELLM_LENS_PUBLIC_URL - value: {{ include "litellm.lensWorker.publicUrl" . | quote }} + value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }} - name: LENS_GATEWAY_SECRET valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.gatewaySecretName" . | quote }} - key: {{ .Values.lensWorker.gateway.secretKey | quote }} + name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }} + key: {{ .gatewaySecret.key | quote }} - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} - key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} + name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }} + key: {{ .serviceTokenSecret.key | quote }} +{{- end }} {{- end }} {{- end -}} - -{{- define "litellm.lensWorker.labels" -}} -{{- $labels := include "litellm.commonLabels" . | fromYaml -}} -{{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}} -{{- toYaml $labels -}} -{{- end -}} - -{{- define "litellm.lensWorker.serviceTokenSecretName" -}} -{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}} -{{- end -}} - -{{- define "litellm.lensWorker.gatewaySecretName" -}} -{{- .Values.lensWorker.gateway.secretName | default (printf "%s-lens-gateway" (include "litellm.fullname" . | trunc 50 | trimSuffix "-")) -}} -{{- end -}} - -{{- define "litellm.lensWorker.bundledClickhouse" -}} -{{- if and (eq (include "litellm.lens.mode" .) "bundled") .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.publicUrl" -}} -{{- if .Values.lensWorker.publicUrl -}} -{{- .Values.lensWorker.publicUrl -}} -{{- else if eq (include "litellm.lens.mode" .) "external" -}} -{{- fail "lensWorker.publicUrl is required for external Lens" -}} -{{- else if .Values.lensWorker.ingress.enabled -}} -{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}} -{{- else if and .Values.ingress.enabled .Values.ingress.host -}} -{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} -{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}} -{{- else -}} -{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}} -{{- end -}} -{{- end -}} - -{{- define "litellm.lensWorker.clickhouseName" -}} -{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}} -{{- end -}} - -{{- define "litellm.lens.mode" -}} -{{- $mode := .Values.lensWorker.mode | default (ternary "bundled" "disabled" .Values.lensWorker.enabled) -}} -{{- if not (has $mode (list "bundled" "external" "disabled")) -}} -{{- fail "lensWorker.mode must be bundled, external or disabled" -}} -{{- end -}} -{{- $mode -}} -{{- end -}} - -{{- define "litellm.lens.render" -}} -{{- $root := .root -}} -{{- $values := mergeOverwrite (deepCopy $root.Subcharts.lens.Values) (deepCopy $root.Values.lensWorker) -}} -{{- $_ := set $values "fullnameOverride" (printf "%s-lens-worker" (include "litellm.fullname" $root)) -}} -{{- $_ := set $values "component" "lens-worker" -}} -{{- $_ := set $values "nameOverride" (printf "%s-lens-worker" (include "litellm.name" $root | trunc 51 | trimSuffix "-")) -}} -{{- $_ := set $values "imagePullSecrets" $root.Values.imagePullSecrets -}} -{{- $_ := set $values.gateway "enabled" true -}} -{{- $_ := set $values.gateway "generatedName" (include "litellm.lensWorker.gatewaySecretName" $root) -}} -{{- $_ := set $values.clickhouse "nameOverride" (include "litellm.lensWorker.clickhouseName" $root) -}} -{{- $_ := set $values.serviceTokenSecret "generatedName" (include "litellm.lensWorker.serviceTokenSecretName" $root) -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default $root.Subcharts.lens.Values.publicUrl) -}} -{{- if and (eq .resource "deployment") (or $root.Values.lensWorker.publicUrl $root.Values.lensWorker.ingress.enabled $root.Values.ingress.enabled) -}} -{{- $ingestion := include "litellm.lensWorker.publicUrl" $root -}} -{{- $_ := set $values "ingestionUrl" $ingestion -}} -{{- $_ := set $values "publicUrl" ($root.Values.lensWorker.standaloneUrl | default (trimSuffix "/lens-ingest" $ingestion)) -}} -{{- end -}} -{{- include (printf "lens.%s" .resource) (dict "Values" $values "Release" $root.Release "Chart" $root.Subcharts.lens.Chart "Capabilities" $root.Capabilities) -}} -{{- end -}} diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index e5dcfd28316..9f778e78087 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -166,14 +166,14 @@ spec: - host: {{ $componentHosts.gateway | quote }} http: paths: - {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }} + {{- if and .Values.lens.enabled .Values.lens.service.name }} - path: /lens-ingest pathType: Prefix backend: service: - name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }} + name: {{ .Values.lens.service.name }} port: - number: {{ .Values.lensWorker.service.port }} + number: {{ .Values.lens.service.port }} {{- end }} - path: / pathType: Prefix @@ -249,15 +249,15 @@ spec: port: number: {{ $gatewayPort }} {{- end }} - {{- if or (eq (include "litellm.lens.mode" .) "bundled") (and (eq (include "litellm.lens.mode" .) "external") .Values.lensWorker.externalServiceName) }} + {{- if and .Values.lens.enabled .Values.lens.service.name }} {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }} - path: /lens-ingest pathType: Prefix backend: service: - name: {{ .Values.lensWorker.externalServiceName | default (printf "%s-lens-worker" (include "litellm.fullname" .)) }} + name: {{ .Values.lens.service.name }} port: - number: {{ .Values.lensWorker.service.port }} + number: {{ .Values.lens.service.port }} {{- end }} {{- /* --- Operator-supplied extra paths (ingress.extraPaths) --- diff --git a/helm/litellm/templates/lens/clickhouse.yaml b/helm/litellm/templates/lens/clickhouse.yaml deleted file mode 100644 index d9daafda6ba..00000000000 --- a/helm/litellm/templates/lens/clickhouse.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "clickhouse") }} -{{- end }} diff --git a/helm/litellm/templates/lens/deployment.yaml b/helm/litellm/templates/lens/deployment.yaml deleted file mode 100644 index 01a6acb1908..00000000000 --- a/helm/litellm/templates/lens/deployment.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "deployment") }} -{{- end }} diff --git a/helm/litellm/templates/lens/ingress.yaml b/helm/litellm/templates/lens/ingress.yaml deleted file mode 100644 index b5994b22d92..00000000000 --- a/helm/litellm/templates/lens/ingress.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "ingress") }} -{{- end }} diff --git a/helm/litellm/templates/lens/secrets.yaml b/helm/litellm/templates/lens/secrets.yaml deleted file mode 100644 index 76928c26e5f..00000000000 --- a/helm/litellm/templates/lens/secrets.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "secrets") }} -{{- end }} diff --git a/helm/litellm/templates/lens/service.yaml b/helm/litellm/templates/lens/service.yaml deleted file mode 100644 index 60c7e995134..00000000000 --- a/helm/litellm/templates/lens/service.yaml +++ /dev/null @@ -1,3 +0,0 @@ -{{- if eq (include "litellm.lens.mode" .) "bundled" }} -{{ include "litellm.lens.render" (dict "root" . "resource" "service") }} -{{- end }} diff --git a/helm/litellm/tests/component_hosts_tests.yaml b/helm/litellm/tests/component_hosts_tests.yaml index f9c9ce6fad2..b55b3b694a0 100644 --- a/helm/litellm/tests/component_hosts_tests.yaml +++ b/helm/litellm/tests/component_hosts_tests.yaml @@ -15,7 +15,6 @@ tests: ingress.componentHosts.ui: ui.example.com ingress.componentHosts.backend: backend.example.com ingress.componentHosts.gateway: gateway.example.com - lensWorker.mode: disabled ingress.tls: - hosts: - ui.example.com @@ -83,12 +82,12 @@ tests: ui.backendUrl: https://backend.example.com ui.publicUrl: https://ui.example.com backend.gatewayUrl: https://gateway.example.com - lensWorker.mode: external - lensWorker.externalServiceName: existing-lens - lensWorker.externalUrl: https://lens.example.com - lensWorker.publicUrl: https://lens.example.com - lensWorker.serviceTokenSecret.name: existing-lens-signing - lensWorker.gateway.secretName: existing-lens-signing + lens.enabled: true + lens.service.name: existing-lens + lens.url: http://existing-lens:4318 + lens.publicUrl: https://gateway.example.com/lens-ingest + lens.serviceTokenSecret.name: existing-lens-signing + lens.gatewaySecret.name: existing-lens-signing asserts: - equal: path: spec.rules[2].http.paths[0].path diff --git a/helm/litellm/tests/lens_connection_tests.yaml b/helm/litellm/tests/lens_connection_tests.yaml new file mode 100644 index 00000000000..2b7c32d50cc --- /dev/null +++ b/helm/litellm/tests/lens_connection_tests.yaml @@ -0,0 +1,166 @@ +suite: lens connection +templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - gateway/configmap.yaml + - ingress.yaml +values: + - ./values/required.yaml +tests: + - it: connects the gateway to an external Lens + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_URL + value: http://lens.lens.svc:4318 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://lens.example.com/lens-ingest + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_GATEWAY_SECRET + valueFrom: + secretKeyRef: + name: lens-gateway + key: gateway-secret + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: lens-service + key: service-token + + - it: connects the backend to an external Lens + template: backend/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_URL + value: http://lens.lens.svc:4318 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://lens.example.com/lens-ingest + - contains: + path: spec.template.spec.containers[0].env + content: + name: LENS_GATEWAY_SECRET + valueFrom: + secretKeyRef: + name: lens-gateway + key: gateway-secret + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: lens-service + key: service-token + + - it: requires lens.url when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.url is required when lens.enabled is true + + - it: requires lens.publicUrl when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.publicUrl is required when lens.enabled is true + + - it: requires lens.gatewaySecret.name when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.serviceTokenSecret.name: lens-service + asserts: + - failedTemplate: + errorMessage: lens.gatewaySecret.name is required when lens.enabled is true + + - it: requires lens.serviceTokenSecret.name when Lens is enabled + template: gateway/deployment.yaml + set: + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + asserts: + - failedTemplate: + errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true + + - it: routes /lens-ingest to the named Lens service on the single ingress host + template: ingress.yaml + set: + ingress.enabled: true + ingress.host: litellm.example.com + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + lens.service.name: existing-lens + asserts: + - contains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + pathType: Prefix + backend: + service: + name: existing-lens + port: + number: 4318 + + - it: leaves /lens-ingest off the ingress without a Lens service name + template: ingress.yaml + set: + ingress.enabled: true + ingress.host: litellm.example.com + lens.enabled: true + lens.url: http://lens.lens.svc:4318 + lens.publicUrl: https://lens.example.com/lens-ingest + lens.gatewaySecret.name: lens-gateway + lens.serviceTokenSecret.name: lens-service + asserts: + - exists: + path: spec.rules[0].http.paths + - notContains: + path: spec.rules[0].http.paths + content: + path: /lens-ingest + any: true diff --git a/helm/litellm/tests/lens_endpoint_defaults_tests.yaml b/helm/litellm/tests/lens_endpoint_defaults_tests.yaml deleted file mode 100644 index 2d98dcf471e..00000000000 --- a/helm/litellm/tests/lens_endpoint_defaults_tests.yaml +++ /dev/null @@ -1,86 +0,0 @@ -suite: Lens endpoint defaults -templates: - - gateway/deployment.yaml - - gateway/configmap.yaml -set: - lensWorker.enabled: true - ingress.enabled: true - ingress.host: gateway.example - ingress.tls: - - hosts: [gateway.example] - secretName: tls -tests: - - it: derives the trace endpoint from the deployment hostname and TLS - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - it: preserves an explicitly configured public address - set: - lensWorker.publicUrl: https://custom.example/prefix - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://custom.example/prefix - - it: uses the dedicated Lens ingress when configured - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: - - hosts: [traces.example] - secretName: traces-tls - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTPS for a dedicated ALB ingress with certificate annotations - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.className: alb - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' - alb.ingress.kubernetes.io/ssl-redirect: "443" - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://traces.example - - it: uses HTTP for a dedicated ingress without its own TLS - set: - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - lensWorker.ingress.tls: [] - lensWorker.ingress.annotations: {} - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://traces.example - - it: uses HTTP when ingress has no TLS - set: - ingress.tls: [] - template: gateway/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: http://gateway.example/lens-ingest -values: - - ./values/required.yaml diff --git a/helm/litellm/tests/lens_modes_tests.yaml b/helm/litellm/tests/lens_modes_tests.yaml deleted file mode 100644 index 3166a88e094..00000000000 --- a/helm/litellm/tests/lens_modes_tests.yaml +++ /dev/null @@ -1,208 +0,0 @@ ---- -suite: Lens deployment modes -templates: -- gateway/configmap.yaml -- backend/deployment.yaml -- lens/deployment.yaml -- lens/service.yaml -- lens/clickhouse.yaml -- lens/secrets.yaml -- ingress.yaml -values: -- "./values/required.yaml" -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: retains the existing deployment before transferring its release ownership - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.retainResources: true - asserts: - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep -- it: keeps the ingestion endpoint on an independently deployed Lens service - template: ingress.yaml - set: - ingress.enabled: true - lensWorker.mode: external - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: existing-lens - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: {number: 4318} -- it: connects an external deployment without installing Lens resources - templates: - - lens/deployment.yaml - - lens/service.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - hasDocuments: - count: 0 -- it: sends delegated requests to the explicitly selected external Lens - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens.other-namespace:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: external-lens-signing - key: gateway-secret -- it: disables Lens explicitly even with a retained legacy enable flag - template: lens/deployment.yaml - set: - lensWorker.mode: disabled - lensWorker.enabled: true - asserts: - - hasDocuments: - count: 0 -- it: bundles Lens through explicit mode selection - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - asserts: - - hasDocuments: - count: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_MODE - value: standalone -- it: uses separate credentials for external identity and service access - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.serviceTokenSecret: {name: worker-service, key: token} - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: {name: worker-service, key: token} -- it: supplies the same separate signing secret to the bundled service - template: lens/deployment.yaml - set: - lensWorker.mode: bundled - lensWorker.gateway: {secretName: delegated-identity, secretKey: signature} - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: delegated-identity, key: signature} -- it: routes explicit bundled mode with the enable flag off - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: bundled - lensWorker.enabled: false - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: gateway-lens-worker - port: {number: 4318} -- it: routes external mode to its selected service - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "existing-lens" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: {number: 4318} -- it: omits ingestion when an external service has no ingress backend - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: external - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true -- it: omits ingestion for disabled mode with the enable flag on - template: ingress.yaml - set: - fullnameOverride: gateway - ingress.enabled: true - ingress.host: gateway.example - lensWorker.mode: disabled - lensWorker.enabled: true - lensWorker.externalUrl: http://existing-lens:4318 - lensWorker.externalServiceName: "" - lensWorker.serviceTokenSecret.name: worker-service - lensWorker.gateway.secretName: external-lens-signing - asserts: - - notContains: - path: spec.rules[0].http.paths - content: {path: /lens-ingest} - any: true diff --git a/helm/litellm/tests/lens_modes_validation_tests.yaml b/helm/litellm/tests/lens_modes_validation_tests.yaml deleted file mode 100644 index 6380727893d..00000000000 --- a/helm/litellm/tests/lens_modes_validation_tests.yaml +++ /dev/null @@ -1,54 +0,0 @@ ---- -suite: Lens deployment mode validation -templates: -- gateway/configmap.yaml -- backend/deployment.yaml -values: -- "./values/required.yaml" -set: - lensWorker.publicUrl: https://traces.example -tests: -- it: requires a separate identity signing reference for external Lens - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: worker-service - asserts: - - failedTemplate: - errorMessage: lensWorker.gateway.secretName is required for external Lens -- it: requires an external service address - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - asserts: - - failedTemplate: - errorMessage: lensWorker.externalUrl is required for external Lens -- it: requires the shared server credential for an external service - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - asserts: - - failedTemplate: - errorMessage: lensWorker.serviceTokenSecret.name is required for external Lens -- it: requires an agent reachable URL for external ingestion - template: backend/deployment.yaml - set: - lensWorker.mode: external - lensWorker.externalUrl: http://lens.other-namespace:4318 - lensWorker.serviceTokenSecret.name: external-lens-signing - lensWorker.gateway.secretName: external-lens-signing - lensWorker.publicUrl: '' - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required for external Lens -- it: rejects unsupported deployment modes - template: backend/deployment.yaml - set: - lensWorker.mode: typo - asserts: - - failedTemplate: - errorMessage: lensWorker.mode must be bundled, external or disabled diff --git a/helm/litellm/tests/lens_saved_secrets_tests.yaml b/helm/litellm/tests/lens_saved_secrets_tests.yaml deleted file mode 100644 index a6e190ec15b..00000000000 --- a/helm/litellm/tests/lens_saved_secrets_tests.yaml +++ /dev/null @@ -1,61 +0,0 @@ -suite: Lens preserves credentials across Helm upgrades -templates: - - lens/secrets.yaml -set: - fullnameOverride: lens-test - lensWorker.enabled: true -release: - namespace: lens - name: lens-test - upgrade: true -kubernetesProvider: - scheme: - v1/Secret: - gvr: - version: v1 - resource: secrets - namespaced: true - objects: - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-gateway - namespace: lens - data: - gateway-secret: c2F2ZWQtc2lnbmluZy1rZXk= - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-service - namespace: lens - data: - service-token: c2F2ZWQtc2VydmljZS10b2tlbg== - - apiVersion: v1 - kind: Secret - metadata: - name: lens-test-lens-clickhouse - namespace: lens - data: - password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= -tests: - - it: preserves the separate signing credential across upgrades - documentSelector: {path: metadata.name, value: lens-test-lens-gateway} - asserts: - - equal: - path: data.gateway-secret - value: c2F2ZWQtc2lnbmluZy1rZXk= - - notExists: {path: data.service-token} - - it: reuses the service credential instead of breaking running services - documentIndex: 0 - asserts: - - equal: - path: data.service-token - value: c2F2ZWQtc2VydmljZS10b2tlbg== - - it: reuses the database password instead of locking out stored traces - documentIndex: 1 - asserts: - - equal: - path: data.password - value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= -values: - - ./values/required.yaml diff --git a/helm/litellm/tests/lens_service_tests.yaml b/helm/litellm/tests/lens_service_tests.yaml deleted file mode 100644 index 544549e364d..00000000000 --- a/helm/litellm/tests/lens_service_tests.yaml +++ /dev/null @@ -1,196 +0,0 @@ -suite: Lens service isolation and ingestion routing -templates: -- gateway/configmap.yaml -- gateway/deployment.yaml -- backend/deployment.yaml -- ingress.yaml -- lens/ingress.yaml -- lens/service.yaml -- lens/deployment.yaml -tests: -- it: connects gateway/deployment.yaml to the shared Lens service - template: gateway/deployment.yaml - set: &id001 - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens-test-lens-worker:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-test-lens-gateway - key: gateway-secret -- it: connects backend/deployment.yaml to the shared Lens service - template: backend/deployment.yaml - set: *id001 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens-test-lens-worker:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://gateway.example/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-test-lens-gateway - key: gateway-secret -- it: routes uploads directly to Lens instead of the gateway - template: ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - ingress.enabled: true - ingress.host: gateway.example - asserts: - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - number: 4318 -- it: keeps internal routes out of a dedicated ingestion hostname - template: lens/ingress.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://gateway.example/lens-ingest - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.ingress.enabled: true - lensWorker.ingress.host: traces.example - asserts: - - equal: - path: spec.rules[0].http.paths - value: - - path: /v1/ - pathType: Prefix - backend: - service: - name: lens-test-lens-worker - port: - name: otlp -- it: maps the Lens service to the ingestion listener - template: lens/service.yaml - set: *id001 - asserts: - - equal: - path: spec.selector - value: - app.kubernetes.io/instance: RELEASE-NAME - app.kubernetes.io/component: lens-worker - - equal: - path: spec.ports - value: - - name: otlp - port: 4318 - targetPort: otlp -- it: gives only Lens the ClickHouse secret - template: lens/deployment.yaml - set: *id001 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_URL - valueFrom: - secretKeyRef: - name: lens-storage - key: url -- it: requires an agent reachable ingestion URL - template: gateway/deployment.yaml - set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: '' - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - failedTemplate: - errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname -- it: omits Lens connection settings when disabled in gateway/deployment.yaml - template: gateway/deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - any: true -- it: omits Lens connection settings when disabled in backend/deployment.yaml - template: backend/deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - any: true -- it: preserves an existing ClickHouse database and retention - template: lens/deployment.yaml - set: - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - lensWorker.clickhouseDatabase: existing_traces - lensWorker.retentionDays: 45 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_DATABASE - value: existing_traces - - contains: - path: spec.template.spec.containers[0].env - content: - name: AGENT_TRACING_RETENTION_DAYS - value: '45' -- it: keeps Lens pods outside the gateway autoscaling selector - template: lens/deployment.yaml - set: &id002 - nameOverride: inference - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage - asserts: - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/name"] - value: inference-lens-worker -- it: preserves the existing gateway deployment selector - template: gateway/deployment.yaml - set: *id002 - asserts: - - equal: - path: spec.selector.matchLabels["app.kubernetes.io/name"] - value: inference -values: -- ./values/required.yaml diff --git a/helm/litellm/tests/lens_setup_tests.yaml b/helm/litellm/tests/lens_setup_tests.yaml deleted file mode 100644 index d61c7cad55b..00000000000 --- a/helm/litellm/tests/lens_setup_tests.yaml +++ /dev/null @@ -1,164 +0,0 @@ -suite: Lens managed setup -set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example -release: - namespace: lens - name: lens-test -templates: - - lens/deployment.yaml - - lens/clickhouse.yaml - - lens/secrets.yaml -tests: - - it: generates private login, gateway and storage credentials - template: lens/secrets.yaml - asserts: - - hasDocuments: - count: 4 - - matchRegex: - path: data.service-token - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 0 - - matchRegex: - path: data.password - pattern: '^[A-Za-z0-9+/]{86}==$' - documentIndex: 1 - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep - - it: connects Lens to its private bundled storage - template: lens/deployment.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: lens-test-lens-clickhouse - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: stores traces on a persistent volume with the chosen storage class - template: lens/clickhouse.yaml - documentIndex: 1 - set: - lensWorker.clickhouse.storage: 40Gi - lensWorker.clickhouse.storageClassName: fast - asserts: - - equal: - path: spec.volumeClaimTemplates[0].spec - value: - accessModes: [ReadWriteOnce] - storageClassName: fast - resources: - requests: - storage: 40Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: lens-test-lens-clickhouse - key: password - - it: keeps an existing external database instead of creating a new one - template: lens/clickhouse.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - asserts: - - hasDocuments: - count: 0 - - it: leaves supplied secrets under their existing manager - template: lens/secrets.yaml - set: - lensWorker.clickhouseSecret.name: external-clickhouse - lensWorker.serviceTokenSecret.name: external-service - lensWorker.adminTokenSecret.name: external-admin - lensWorker.gateway.secretName: external-signing - asserts: - - hasDocuments: - count: 0 - - it: creates no credentials or database when Lens is disabled - templates: - - lens/secrets.yaml - - lens/clickhouse.yaml - set: - lensWorker.enabled: false - asserts: - - hasDocuments: - count: 0 - - it: requires an external database when bundled storage is explicitly disabled - template: lens/deployment.yaml - set: - lensWorker.clickhouse.enabled: false - asserts: - - failedTemplate: - errorMessage: Lens clickhouseSecret.name is required when bundled storage is disabled - - it: keeps storage names valid and uses the same name for the connection - set: - fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - asserts: - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 0 - - matchRegex: - path: metadata.name - pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 0 - - equal: - path: spec.serviceName - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/clickhouse.yaml - documentIndex: 1 - - equal: - path: metadata.name - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/secrets.yaml - documentIndex: 1 - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_HOST - value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - template: lens/deployment.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: CLICKHOUSE_PASSWORD - valueFrom: - secretKeyRef: - name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse - key: password - template: lens/deployment.yaml - - it: allows cold storage startup and inherits registry credentials - template: lens/clickhouse.yaml - documentIndex: 1 - set: - imagePullSecrets: [{name: registry-auth}] - asserts: - - equal: - path: spec.template.spec.imagePullSecrets - value: [{name: registry-auth}] - - equal: - path: spec.template.spec.containers[0].startupProbe - value: - httpGet: {path: /ping, port: http} - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 60 - -values: - - ./values/required.yaml diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml deleted file mode 100644 index 614db9d6b2b..00000000000 --- a/helm/litellm/tests/lens_worker_tests.yaml +++ /dev/null @@ -1,79 +0,0 @@ -suite: Independent Lens release and credentials -templates: - - lens/deployment.yaml -values: - - ./values/required.yaml -set: - fullnameOverride: lens-test - lensWorker.enabled: true - lensWorker.publicUrl: https://traces.example - lensWorker.serviceTokenSecret.name: lens-service - lensWorker.clickhouseSecret.name: lens-storage -tests: - - it: selects a Lens release independently of the gateway - chart: - appVersion: 9.8.7 - set: - backend.image.tag: gateway-9.8.7 - lensWorker.image.tag: 2.3.4 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/lens:2.3.4 - - it: retains the pinned Lens chart default when only the gateway changes - chart: - appVersion: 9.8.7 - set: - backend.image.tag: gateway-9.8.7 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/lens:0.1.0-dev.0 - - it: selects development publications explicitly - set: - lensWorker.image.repository: ghcr.io/berriai/lens-dev - lensWorker.image.tag: sha-0123456789abcdef - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/lens-dev:sha-0123456789abcdef - - it: preserves a verified registry mirror and digest - set: - lensWorker.image.repository: registry.example/lens - lensWorker.image.tag: replaced-label - lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: registry.example/lens@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - - it: rejects a malformed digest - set: - lensWorker.image.digest: sha256:invalid - asserts: - - failedTemplate: - errorMessage: Lens image.digest must be sha256 followed by 64 lowercase hex characters - - it: keeps the legacy enablement switch working - set: - lensWorker.enabled: false - asserts: - - hasDocuments: {count: 0} - - it: keeps isolated local analysis with delegated gateway access - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: {name: LENS_MODE, value: standalone} - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: {name: lens-test-lens-gateway, key: gateway-secret} - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - equal: - path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem - value: true - - equal: - path: spec.template.spec.volumes[0].emptyDir - value: {medium: Memory, sizeLimit: 1Gi} diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index aa6ee7a6056..92a48153c5a 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -654,70 +654,15 @@ ui: topologySpreadConstraints: [] lens: - library: true - -lensWorker: - retainResources: false - mode: "" - externalUrl: "" - externalServiceName: "" - gateway: - secretName: "" - secretKey: gateway-secret - standaloneUrl: "" - adminTokenSecret: - name: "" - key: admin-token - extraEnv: [] - extraEnvFrom: [] enabled: false - replicaCount: 1 - image: - repository: ghcr.io/berriai/lens - tag: "0.1.0-dev.0" - digest: "" - pullPolicy: IfNotPresent - tokenSecret: + url: "" + publicUrl: "" + service: name: "" - key: token + port: 4318 + gatewaySecret: + name: "" + key: gateway-secret serviceTokenSecret: name: "" key: service-token - clickhouse: - enabled: true - image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e - storage: 20Gi - storageClassName: null - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 2Gi - clickhouseDatabase: litellm - retentionDays: 14 - clickhouseSecret: - name: "" - key: url - publicUrl: "" - service: - port: 4318 - annotations: {} - ingress: - enabled: false - path: /v1/ - className: "" - host: "" - annotations: {} - tls: [] - url: "" - tmpSizeLimit: 1Gi - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 2Gi - nodeSelector: {} - tolerations: [] - affinity: {} diff --git a/tests/e2e/migrations/lens_helm_smoke.sh b/tests/e2e/migrations/lens_helm_smoke.sh deleted file mode 100644 index 19b55be75e1..00000000000 --- a/tests/e2e/migrations/lens_helm_smoke.sh +++ /dev/null @@ -1,477 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -qualification_mode=${LENS_QUALIFICATION_MODE:-smoke} -chart_selection=${LENS_HELM_CHART:-all} -case "$qualification_mode" in smoke|boundaries|release) ;; *) printf 'LENS_QUALIFICATION_MODE must be smoke, boundaries or release\n' >&2; exit 2 ;; esac -case "$chart_selection" in all|litellm-helm|litellm) ;; *) printf 'LENS_HELM_CHART must be all, litellm-helm or litellm\n' >&2; exit 2 ;; esac -if [[ "$qualification_mode" == release ]]; then - : "${LENS_RELEASE_PROVIDER_API_KEY:?A separately authorized provider credential is required}" - : "${LENS_RELEASE_PROVIDER_API_BASE:?Set the authorized provider API base}" - : "${LENS_RELEASE_PROVIDER_MODEL:?Set the provider model explicitly}" - [[ "$LENS_RELEASE_PROVIDER_API_BASE" == https://* ]] -fi -qa_dir=$(mktemp -d) -cluster="lens-install-$(openssl rand -hex 6)" -cluster_owned=false -forward_pids=() -cleanup() { - local status=$? - if declare -F qualification_finish > /dev/null; then qualification_finish "$status" || true; fi - if (( status != 0 )); then - for log in "$qa_dir"/*-forward.log; do - if [[ -f "$log" ]]; then cat "$log" >&2; fi - done - if [[ -n "${namespace:-}" ]]; then diagnose || true; fi - fi - for pid in "${forward_pids[@]}"; do kill "$pid" 2>/dev/null || true; done - if [[ "$cluster_owned" == true ]]; then kind delete cluster --name "$cluster" || true; fi - rm -rf "$qa_dir" - return "$status" -} -trap cleanup EXIT -umask 077 -export KUBECONFIG="$qa_dir/kubeconfig" -for component in gateway backend monolith; do - baseline_id=$(docker image inspect "lens-ci-$component:v0.0.0-lens-ci-baseline" --format '{{.Id}}') - current_id=$(docker image inspect "lens-ci-$component:v0.0.0-lens-ci" --format '{{.Id}}') - test "$baseline_id" != "$current_id" -done -cluster_owned=true -kind create cluster --name "$cluster" \ - --image kindest/node:v1.32.2@sha256:f226345927d7e348497136874b6d207e0b32cc52154ad8323129352923a3142f \ - --wait 120s -for component in gateway backend ui migrations monolith; do - kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci" -done -for component in gateway backend monolith; do - kind load docker-image --name "$cluster" "lens-ci-$component:v0.0.0-lens-ci-baseline" -done -test "$(docker image inspect lens-ci-worker:baseline --format '{{.Id}}')" != \ - "$(docker image inspect lens-ci-worker:upgrade --format '{{.Id}}')" -kind load docker-image --name "$cluster" lens-ci-worker:baseline lens-ci-worker:upgrade -test -f helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz - -api() { - curl --fail-with-body --silent --show-error --max-time 20 \ - -H "Authorization: Bearer $master_key" -H 'Content-Type: application/json' \ - "http://127.0.0.1:14418$1" "${@:2}" -} - -saved_trace() { - for attempt in $(seq 1 30); do - if api "/v1/traces/$trace_id" > "$qa_dir/saved.json" && \ - jq -e --arg span "$span_id" 'any(.spans[]; .span_id == $span)' "$qa_dir/saved.json" > /dev/null; then - return 0 - fi - sleep 2 - done - return 1 -} - -diagnose() { - kubectl -n "$namespace" get pods - kubectl -n "$namespace" get services,endpoints - kubectl -n "$namespace" get events --sort-by=.lastTimestamp | tail -30 - if [[ "$qualification_mode" == smoke ]]; then - kubectl -n "$namespace" logs --all-containers -l app.kubernetes.io/instance=lens --tail=50 || true - fi - return 1 -} - -forward() { - local service=$1 local_port=$2 remote_port=$3 - local log="$qa_dir/$service-forward.log" - kubectl -n "$namespace" port-forward --address 127.0.0.1 --pod-running-timeout=30s \ - "service/$service" "$local_port:$remote_port" > "$log" 2>&1 & - local pid=$! - forward_pids+=("$pid") - for attempt in $(seq 1 150); do - if ! kill -0 "$pid" 2>/dev/null; then - cat "$log" >&2 - return 1 - fi - if grep -q "^Forwarding from 127\\.0\\.0\\.1:$local_port ->" "$log"; then return 0; fi - sleep 0.2 - done - cat "$log" >&2 - return 1 -} - -stop_forwards() { - for pid in "${forward_pids[@]}"; do - kill "$pid" 2>/dev/null || true - wait "$pid" 2>/dev/null || true - done - forward_pids=() -} - -forward_bundled() { - stop_forwards - qualification_forward_control - forward lens-lens-worker 14419 4318 -} - -deployment_pods() { - local deployment selector deployment_uid replica_sets attempt - local pods="$qa_dir/$1-pods.json" snapshot="$qa_dir/$1-snapshot.json" - printf '%s: wait for deployment/%s before recording pod identities\n' "$namespace" "$1" >&2 - kubectl -n "$namespace" rollout status "deployment/$1" --timeout=180s >&2 || return - deployment=$(kubectl -n "$namespace" get deployment "$1" -o json) || return - selector=$(jq -r '.spec.selector.matchLabels | to_entries | map("\(.key)=\(.value)") | join(",")' <<< "$deployment") - deployment_uid=$(jq -er .metadata.uid <<< "$deployment") - test -n "$selector" - replica_sets=$(kubectl -n "$namespace" get replicasets -l "$selector" -o json \ - | jq -ce --arg uid "$deployment_uid" '[.items[] - | select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "Deployment" and .uid == $uid)) - | .metadata.uid] | if length > 0 then . else error("Deployment ReplicaSet identity is missing") end') || return - for attempt in $(seq 1 30); do - kubectl -n "$namespace" get pods -l "$selector" -o json > "$pods" || return - if jq -S -e --argjson replica_sets "$replica_sets" '[.items[] - | select(any(.metadata.ownerReferences[]?; .controller == true and .kind == "ReplicaSet" - and (.uid as $owner | $replica_sets | index($owner) != null))) - | select(.metadata.deletionTimestamp == null) - | {uid:.metadata.uid, images:.spec.containers | map({name,image}), - containers:(.status.containerStatuses // []) | map({name,imageID,restartCount,ready})}] | sort_by(.uid) - | if length > 0 and all(.[]; .containers | length > 0) - and all(.[].containers[]; .imageID != null and .imageID != "" and .ready) - then . else error("Ready pod runtime identity is missing") end' \ - "$pods" > "$snapshot" 2> "$qa_dir/$1-snapshot-error.log"; then - cat "$snapshot" - return 0 - fi - sleep 1 - done - printf '%s: deployment/%s pod identities did not converge\n' "$namespace" "$1" >&2 - jq '[.items[] | {name:.metadata.name, uid:.metadata.uid, deleting:.metadata.deletionTimestamp, - phase:.status.phase, containers:[.status.containerStatuses[]? | {name,imageID,restartCount,ready}]}]' \ - "$pods" >&2 - return 1 -} - -gateway_pods() { - local component=$1 tag=$2 deployment="lens-$1" - if [[ "$component" == monolith ]]; then deployment=lens; fi - deployment_pods "$deployment" \ - | jq -S -e --arg expected "lens-ci-$component:$tag" ' - if all(.[]; any(.images[]; .image == $expected)) then - [.[] as $pod | $pod.images[] | select(.image == $expected) | . as $requested - | $pod.containers[] | select(.name == $requested.name) - | {uid:$pod.uid, image:$requested.image, name, imageID, restartCount, ready}] - else error("Gateway pod does not use the expected image") end' -} - -migration_job() { - kubectl -n "$namespace" get job lens-migrations -o json \ - | jq -e --arg image "$1" ' - select(.metadata.annotations["helm.sh/hook"] == "pre-install,pre-upgrade") - | select(.metadata.annotations["helm.sh/hook-delete-policy"] == "before-hook-creation") - | select(.metadata.annotations["argocd.argoproj.io/hook"] == null) - | select(.status.succeeded == 1 and .spec.template.spec.containers[0].image == $image) - | .metadata.uid | select(type == "string" and length > 0)' -} - -source tests/e2e/migrations/lens_release_qualification.sh -qualification_initialize -if [[ "$qualification_mode" != smoke && "$chart_selection" != litellm ]]; then - qualification_bundled_postgres -fi -for chart in litellm-helm litellm; do - if [[ "$chart_selection" != all && "$chart_selection" != "$chart" ]]; then continue; fi - namespace="lens-$chart" - kubectl create namespace "$namespace" - master_key="sk-$(openssl rand -hex 24)" - kubectl -n "$namespace" create secret generic lens-secrets \ - --from-literal="master-key=$master_key" \ - --from-literal="service-token=$(openssl rand -hex 32)" \ - --from-literal="gateway-secret=$(openssl rand -hex 32)" \ - --from-literal="url=http://clickhouse:8123" \ - --from-literal=username=litellm --from-literal=password=isolated-helm-test - kubectl -n "$namespace" apply -f - <<'YAML' -apiVersion: apps/v1 -kind: Deployment -metadata: {name: postgres} -spec: - selector: {matchLabels: {app: postgres}} - template: - metadata: {labels: {app: postgres}} - spec: - containers: - - name: postgres - image: postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea - env: - - {name: POSTGRES_DB, value: litellm} - - {name: POSTGRES_USER, value: litellm} - - {name: POSTGRES_PASSWORD, value: isolated-helm-test} - readinessProbe: - exec: {command: [pg_isready, -U, litellm, -d, litellm]} ---- -apiVersion: v1 -kind: Service -metadata: {name: postgres} -spec: - selector: {app: postgres} - ports: [{port: 5432}] ---- -apiVersion: apps/v1 -kind: Deployment -metadata: {name: clickhouse} -spec: - selector: {matchLabels: {app: clickhouse}} - template: - metadata: {labels: {app: clickhouse}} - spec: - containers: - - name: clickhouse - image: clickhouse/clickhouse-server:26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e - env: [{name: CLICKHOUSE_SKIP_USER_SETUP, value: "1"}] - volumeMounts: - - {name: keeper, mountPath: /etc/clickhouse-server/config.d/lens-keeper.xml, subPath: lens-keeper.xml} - readinessProbe: - httpGet: {path: /ping, port: 8123} - volumes: - - name: keeper - configMap: {name: clickhouse-keeper} ---- -apiVersion: v1 -kind: ConfigMap -metadata: {name: clickhouse-keeper} -data: - lens-keeper.xml: | - - - 91811 - /var/lib/clickhouse/coordination/log - /var/lib/clickhouse/coordination/snapshots - 10000true - 1127.0.0.19234 - - 127.0.0.19181 - /lens/keeper-map - ---- -apiVersion: v1 -kind: Service -metadata: {name: clickhouse} -spec: - selector: {app: clickhouse} - ports: [{port: 8123}] -YAML - kubectl -n "$namespace" rollout status deployment/postgres --timeout=180s - kubectl -n "$namespace" rollout status deployment/clickhouse --timeout=180s - cat > "$qa_dir/common.yaml" <<'YAML' -fullnameOverride: lens -migrationJob: - ttlSecondsAfterFinished: 3600 - hooks: - helm: {enabled: true} - argocd: {enabled: false} -lensWorker: - enabled: true - image: {repository: lens-ci-worker, tag: baseline, pullPolicy: Never} - serviceTokenSecret: {name: lens-secrets, key: service-token} - gateway: {secretName: lens-secrets, secretKey: gateway-secret} - clickhouseSecret: {name: lens-secrets, key: url} - clickhouseDatabase: existing_traces - retentionDays: 45 - publicUrl: http://127.0.0.1:14419 -YAML - if [[ "$chart" == litellm-helm ]]; then - control=lens - control_port=4000 - cat > "$qa_dir/chart.yaml" <<'YAML' -image: {repository: lens-ci-monolith, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never} -masterkeySecretName: lens-secrets -masterkeySecretKey: master-key -envVars: {STORE_MODEL_IN_DB: "True"} -db: - deployStandalone: false - useExisting: true - endpoint: postgres - secret: {name: lens-secrets, usernameKey: username, passwordKey: password} -redis: {enabled: false} -proxy_config: - model_list: [] - general_settings: - master_key: os.environ/PROXY_MASTER_KEY - store_model_in_db: true - tracing: {enabled: true, store: {type: lens}} -YAML - else - control=lens-backend - control_port=4001 - cat > "$qa_dir/chart.yaml" <<'YAML' -masterKey: {secretName: lens-secrets, secretKey: master-key} -database: - writer: - host: postgres - dbname: litellm - passwordSecret: {name: lens-secrets, usernameKey: username, passwordKey: password} -migrationJob: - image: {repository: lens-ci-migrations, tag: v0.0.0-lens-ci, pullPolicy: Never} -gateway: - image: {repository: lens-ci-gateway, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never} - numWorkers: 1 - extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}] - hpa: {enabled: false} - resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}} - config: - create: true - proxy_config: - model_list: [] - general_settings: - store_model_in_db: true - tracing: {enabled: true, store: {type: lens}} -backend: - extraEnv: [{name: STORE_MODEL_IN_DB, value: "True"}] - image: {repository: lens-ci-backend, tag: v0.0.0-lens-ci-baseline, pullPolicy: Never} - hpa: {enabled: false} - resources: {requests: {cpu: 100m, memory: 512Mi}, limits: {memory: 2Gi}} -ui: - image: {repository: lens-ci-ui, tag: v0.0.0-lens-ci, pullPolicy: Never} - hpa: {enabled: false} -YAML - fi - install=(helm upgrade --install lens "helm/$chart" -n "$namespace" \ - -f "$qa_dir/common.yaml" -f "$qa_dir/chart.yaml" --wait --wait-for-jobs --timeout 8m) - qualification_provider_values - "${install[@]}" || diagnose - forward_bundled - for attempt in $(seq 1 30); do - if api /lens/service > "$qa_dir/status.json" && jq -e '.connected and .status.storage_ready' "$qa_dir/status.json"; then break; fi - sleep 1 - done - jq -e '.connected and .status.storage_ready' "$qa_dir/status.json" - qualification_setup - api /lens/tracing/keys -d '{"name":"Helm smoke"}' > "$qa_dir/key.json" - tracing_key=$(jq -r .key "$qa_dir/key.json") - trace_id=$(openssl rand -hex 16) - span_id=$(openssl rand -hex 8) - jq -n --arg trace "$trace_id" --arg span "$span_id" --arg at "$(date +%s)000000000" \ - '{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Helm trace",kind:1, - startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/trace.json" - curl --fail-with-body --silent --show-error --retry 10 --retry-all-errors --retry-delay 1 \ - -H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \ - -d "@$qa_dir/trace.json" http://127.0.0.1:14419/v1/traces - saved_trace - qualification_read_boundaries baseline - kubectl -n "$namespace" exec deployment/clickhouse -- clickhouse-client --query \ - "SELECT count() FROM existing_traces.otel_traces WHERE TraceId = '$trace_id'" | grep -qx 1 - "${install[@]}" || diagnose - forward_bundled - saved_trace - qualification_read_boundaries reapplied - kubectl -n "$namespace" get deployments -l app.kubernetes.io/instance=lens -o json \ - | jq -S '[.items[] | select(.metadata.name != "lens-lens-worker") | {name:.metadata.name,template:.spec.template}] | sort_by(.name)' \ - > "$qa_dir/gateway-before.json" - "${install[@]}" --set lensWorker.image.tag=upgrade || diagnose - kubectl -n "$namespace" get deployments -l app.kubernetes.io/instance=lens -o json \ - | jq -S '[.items[] | select(.metadata.name != "lens-lens-worker") | {name:.metadata.name,template:.spec.template}] | sort_by(.name)' \ - > "$qa_dir/gateway-after.json" - cmp "$qa_dir/gateway-before.json" "$qa_dir/gateway-after.json" - forward_bundled - saved_trace - qualification_read_boundaries lens-upgrade - "${install[@]}" || diagnose - forward_bundled - saved_trace - qualification_read_boundaries lens-rollback - kubectl -n "$namespace" get deployment lens-lens-worker -o json \ - | jq -S .spec.template > "$qa_dir/lens-before.json" - deployment_pods lens-lens-worker > "$qa_dir/lens-pods-before.json" - gateway_components=(monolith) - gateway_upgrade=(--set image.tag=v0.0.0-lens-ci) - migration_baseline=lens-ci-monolith:v0.0.0-lens-ci-baseline - migration_current=lens-ci-monolith:v0.0.0-lens-ci - if [[ "$chart" == litellm ]]; then - gateway_components=(gateway backend) - gateway_upgrade=(--set gateway.image.tag=v0.0.0-lens-ci --set backend.image.tag=v0.0.0-lens-ci) - migration_baseline=lens-ci-migrations:v0.0.0-lens-ci - migration_current=$migration_baseline - fi - migration_before=$(migration_job "$migration_baseline") - for component in "${gateway_components[@]}"; do - gateway_pods "$component" v0.0.0-lens-ci-baseline > "$qa_dir/$component-before.json" - done - install+=("${gateway_upgrade[@]}") - "${install[@]}" || diagnose - migration_after=$(migration_job "$migration_current") - test "$migration_before" != "$migration_after" - kubectl -n "$namespace" get deployment lens-lens-worker -o json \ - | jq -S .spec.template > "$qa_dir/lens-after.json" - cmp "$qa_dir/lens-before.json" "$qa_dir/lens-after.json" - deployment_pods lens-lens-worker > "$qa_dir/lens-pods-after.json" - cmp "$qa_dir/lens-pods-before.json" "$qa_dir/lens-pods-after.json" - for component in "${gateway_components[@]}"; do - gateway_pods "$component" v0.0.0-lens-ci > "$qa_dir/$component-after.json" - before_id=$(jq -ce '[.[].imageID] | unique | if length == 1 then .[0] else error("Mixed gateway images") end' \ - "$qa_dir/$component-before.json") - after_id=$(jq -ce '[.[].imageID] | unique | if length == 1 then .[0] else error("Mixed gateway images") end' \ - "$qa_dir/$component-after.json") - test "$before_id" != "$after_id" - done - forward_bundled - saved_trace - qualification_read_boundaries gateway-upgrade - if [[ "$qualification_mode" != smoke ]]; then - install+=(--set lensWorker.image.tag=upgrade) - "${install[@]}" || diagnose - forward_bundled - qualification_read_boundaries both-candidates - fi - stop_forwards - kubectl -n "$namespace" rollout restart "deployment/$control" deployment/lens-lens-worker - kubectl -n "$namespace" rollout status "deployment/$control" --timeout=180s - kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s - qualification_forward_control - saved_trace - qualification_outage - printf '%s: fresh install, ingestion, Lens upgrade and rollback, independent gateway image upgrade, and restart passed\n' "$chart" - stop_forwards - helm upgrade --install external-lens helm/litellm-helm/charts/lens-0.1.0-dev.0.tgz \ - -n "$namespace" --wait --timeout 5m \ - --set fullnameOverride=external-lens \ - --set image.repository=lens-ci-worker --set image.tag=upgrade --set image.pullPolicy=Never \ - --set adminTokenSecret.name=lens-secrets --set adminTokenSecret.key=master-key \ - --set gateway.enabled=true --set gateway.secretName=lens-secrets \ - --set gateway.secretKey=gateway-secret --set serviceTokenSecret.name=lens-secrets \ - --set clickhouseSecret.name=lens-secrets --set clickhouseDatabase=existing_traces \ - --set publicUrl=http://127.0.0.1:14419 - "${install[@]}" --set lensWorker.mode=external \ - --set lensWorker.externalUrl=http://external-lens:4318 || diagnose - test -z "$(kubectl -n "$namespace" get deployment lens-lens-worker --ignore-not-found -o name)" - qualification_forward_control - forward external-lens 14419 4318 - api /lens/service | jq -e '.configured and .connected and .status.storage_ready' - saved_trace - qualification_read_boundaries external-lens - trace_id=$(openssl rand -hex 16) - span_id=$(openssl rand -hex 8) - jq --arg trace "$trace_id" --arg span "$span_id" \ - '.resourceSpans[0].scopeSpans[0].spans[0] |= (.traceId=$trace | .spanId=$span)' \ - "$qa_dir/trace.json" > "$qa_dir/external-trace.json" - curl --fail-with-body --silent --show-error --max-time 20 \ - -H "Authorization: Bearer $tracing_key" -H 'Content-Type: application/json' \ - -d "@$qa_dir/external-trace.json" http://127.0.0.1:14419/v1/traces - saved_trace - kubectl -n "$namespace" get deployment external-lens -o json \ - | jq -S '{uid:.metadata.uid,template:.spec.template}' > "$qa_dir/external-before.json" - stop_forwards - tracing_setting=proxy_config.general_settings.tracing.enabled - if [[ "$chart" == litellm ]]; then - tracing_setting=gateway.config.proxy_config.general_settings.tracing.enabled - fi - "${install[@]}" --set lensWorker.mode=disabled --set "$tracing_setting=false" || diagnose - test -z "$(kubectl -n "$namespace" get deployment lens-lens-worker --ignore-not-found -o name)" - qualification_forward_control - api /lens/service | jq -e '.configured == false and .connected == false' - test "$(curl --silent --output /dev/null --write-out '%{http_code}' --max-time 20 \ - -H "Authorization: Bearer $master_key" http://127.0.0.1:14418/lens/datasets)" = 503 - kubectl -n "$namespace" get deployment external-lens -o json \ - | jq -S '{uid:.metadata.uid,template:.spec.template}' > "$qa_dir/external-after.json" - cmp "$qa_dir/external-before.json" "$qa_dir/external-after.json" - printf '%s: external Lens trace reads/writes and disabled gateway behavior passed\n' "$chart" - qualification_record chart-lifecycle passed - stop_forwards - kubectl delete namespace "$namespace" --wait=true -done diff --git a/tests/e2e/migrations/lens_release_qualification.sh b/tests/e2e/migrations/lens_release_qualification.sh deleted file mode 100644 index 86c6055f33d..00000000000 --- a/tests/e2e/migrations/lens_release_qualification.sh +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash - -qualification_initialize() { - qualification_events="$qa_dir/qualification-events.jsonl" - : > "$qualification_events" - qualification_started=$(date -u +%FT%TZ) - qualification_output=${LENS_QUALIFICATION_RESULTS_DIR:-} - if [[ -n "$qualification_output" ]]; then - mkdir -p "$qualification_output" - test ! -e "$qualification_output/host-$chart_selection-$qualification_mode.json" - fi -} - -qualification_record() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - jq -nc --arg chart "${chart:-bootstrap}" --arg case "$1" --arg result "$2" \ - --arg at "$(date -u +%FT%TZ)" '{chart:$chart,case:$case,result:$result,at:$at}' >> "$qualification_events" -} - -qualification_finish() { - if [[ "$qualification_mode" == smoke || -z "$qualification_output" ]]; then return; fi - jq -n --slurpfile checks "$qualification_events" --arg mode "$qualification_mode" \ - --arg selection "$chart_selection" --arg started "$qualification_started" \ - --arg finished "$(date -u +%FT%TZ)" --arg source "$(git rev-parse HEAD)" --argjson exit_code "$1" \ - '{qualification:"isolated candidate host behavior",mode:$mode,chart_selection:$selection, - started_at:$started,finished_at:$finished,harness_source:$source,exit_code:$exit_code, - passed:($exit_code == 0),paid_provider_qualified:($mode == "release" and $exit_code == 0), - signed_release_qualified:false,production_deployed:false,checks:$checks}' \ - > "$qualification_output/host-$chart_selection-$qualification_mode.json" -} - -qualification_request() { - local credential=$1 method=$2 route=$3 expected=$4 label=$5 input=${6:-} - local port=14418 - if [[ "$chart" == litellm && "$route" == /v1/chat/completions ]]; then port=14420; fi - local response="$qa_dir/release-response.json" status - printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "$credential" > "$qa_dir/release-headers" - local arguments=(--silent --show-error --max-time 90 --request "$method" \ - --header "@$qa_dir/release-headers" --output "$response" --dump-header "$qa_dir/release-response.headers" \ - --write-out '%{http_code}') - if [[ -n "$input" ]]; then arguments+=(--data-binary "@$input"); fi - status=$(curl "${arguments[@]}" "http://127.0.0.1:$port$route") - if [[ "$status" != "$expected" ]]; then - qualification_record "$label" "failed-http-$status-expected-$expected" - printf '%s: expected HTTP %s, received %s\n' "$label" "$expected" "$status" >&2 - return 1 - fi - qualification_record "$label" "http-$status" -} - -qualification_forward_control() { - forward "$control" 14418 "$control_port" - if [[ "$qualification_mode" == release && "$chart" == litellm ]]; then - forward lens-gateway 14420 4000 - fi -} - -qualification_provider_values() { - if [[ "$qualification_mode" != release ]]; then return; fi - printf '%s' "$LENS_RELEASE_PROVIDER_API_KEY" > "$qa_dir/provider-key" - kubectl -n "$namespace" create secret generic lens-release-provider \ - --from-file="LENS_RELEASE_PROVIDER_API_KEY=$qa_dir/provider-key" > /dev/null - if [[ "$chart" == litellm-helm ]]; then - printf 'environmentSecrets: [lens-release-provider]\n' > "$qa_dir/provider.yaml" - else - cat > "$qa_dir/provider.yaml" <<'YAML' -gateway: - extraEnv: - - {name: STORE_MODEL_IN_DB, value: "True"} - - name: LENS_RELEASE_PROVIDER_API_KEY - valueFrom: {secretKeyRef: {name: lens-release-provider, key: LENS_RELEASE_PROVIDER_API_KEY}} -backend: - extraEnv: - - {name: STORE_MODEL_IN_DB, value: "True"} - - name: LENS_RELEASE_PROVIDER_API_KEY - valueFrom: {secretKeyRef: {name: lens-release-provider, key: LENS_RELEASE_PROVIDER_API_KEY}} -YAML - fi - install+=(-f "$qa_dir/provider.yaml") -} - -qualification_setup() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - local tenant user team credential trace span attempt status - for tenant in a b; do - user="lens-release-$tenant-$(openssl rand -hex 8)" - team="lens-release-team-$tenant-$(openssl rand -hex 8)" - jq -n --arg user "$user" '{user_id:$user,user_email:($user+"@example.test"),user_role:"internal_user",auto_create_key:false}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /user/new 200 "create-user-$tenant" "$qa_dir/request.json" - jq -e --arg user "$user" '.user_id == $user' "$qa_dir/release-response.json" > /dev/null - jq -n --arg user "$user" --arg team "$team" '{team_id:$team,team_alias:$team,members_with_roles:[{user_id:$user,role:"admin"}]}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /team/new 200 "create-team-$tenant" "$qa_dir/request.json" - jq -e --arg team "$team" '.team_id == $team' "$qa_dir/release-response.json" > /dev/null - jq -n --arg user "$user" --arg team "$team" '{user_id:$user,team_id:$team,key_alias:($team+"-isolation")}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/generate 200 "create-key-$tenant" "$qa_dir/request.json" - credential=$(jq -er '.key | select(type == "string" and length > 0)' "$qa_dir/release-response.json") - jq -n --arg user "$user" --arg team "$team" --arg key "$credential" '{user:$user,team:$team,key:$key}' > "$qa_dir/tenant-$tenant.json" - jq -n --arg team "$team" '{name:"Release tenant trace",team_id:$team}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /lens/tracing/keys 200 "create-ingestion-key-$tenant" "$qa_dir/request.json" - cp "$qa_dir/release-response.json" "$qa_dir/ingestion-$tenant.json" - jq -e --arg team "$team" '.record.tenant.team_id == $team' "$qa_dir/ingestion-$tenant.json" > /dev/null - credential=$(jq -er .key "$qa_dir/ingestion-$tenant.json") - trace=$(openssl rand -hex 16) - span=$(openssl rand -hex 8) - jq -n --arg trace "$trace" --arg span "$span" --arg at "$(date +%s)000000000" \ - '{resourceSpans:[{scopeSpans:[{spans:[{traceId:$trace,spanId:$span,name:"Lens release tenant",kind:1,startTimeUnixNano:$at,endTimeUnixNano:$at,status:{code:1}}]}]}]}' > "$qa_dir/tenant-$tenant-trace.json" - printf '%s' "$trace" > "$qa_dir/tenant-$tenant-trace-id" - printf 'Authorization: Bearer %s\nContent-Type: application/json\n' "$credential" > "$qa_dir/ingestion-headers" - for attempt in $(seq 1 30); do - status=$(curl --silent --show-error --max-time 20 --output "$qa_dir/ingestion-response.json" --write-out '%{http_code}' \ - --header "@$qa_dir/ingestion-headers" --data-binary "@$qa_dir/tenant-$tenant-trace.json" http://127.0.0.1:14419/v1/traces) - if [[ "$status" == 200 ]]; then break; fi - test "$status" = 401 - sleep 1 - done - test "$status" = 200 - qualification_record "ingest-team-$tenant" http-200 - for attempt in $(seq 1 30); do - if api "/v1/traces/$trace" > "$qa_dir/tenant-$tenant-stored.json" 2> /dev/null; then break; fi - sleep 1 - done - jq -e --arg trace "$trace" '.summary.trace_id == $trace' "$qa_dir/tenant-$tenant-stored.json" > /dev/null - done - jq -n --arg user "$(jq -r .user "$qa_dir/tenant-a.json")" --arg team "$(jq -r .team "$qa_dir/tenant-a.json")" \ - '{user_id:$user,team_id:$team,key_alias:"Lens release revocation"}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/generate 200 create-revocation-key "$qa_dir/request.json" - cp "$qa_dir/release-response.json" "$qa_dir/revoked-key.json" - credential=$(jq -er .key "$qa_dir/revoked-key.json") - qualification_request "$credential" GET /lens/service 200 key-valid-before-revocation - jq -n --arg key "$credential" '{keys:[$key]}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/delete 200 revoke-gateway-key "$qa_dir/request.json" - qualification_request "$credential" GET /lens/service 401 revoked-key-denied - if [[ "$qualification_mode" == release ]]; then - qualification_model="lens-release-$(openssl rand -hex 8)" - jq -n --arg model "$qualification_model" --arg provider_model "$LENS_RELEASE_PROVIDER_MODEL" \ - --arg base "$LENS_RELEASE_PROVIDER_API_BASE" \ - '{model_name:$model,litellm_params:{model:$provider_model,api_base:$base,api_key:"os.environ/LENS_RELEASE_PROVIDER_API_KEY"}}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /model/new 200 register-provider-model "$qa_dir/request.json" - jq -e '.model_info.id | type == "string" and length > 0' "$qa_dir/release-response.json" > /dev/null - qualification_paid_calls=0 - qualification_total_cost=0 - fi -} - -qualification_read_boundaries() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - local phase=$1 tenant other credential own foreign - for tenant in a b; do - other=a - if [[ "$tenant" == a ]]; then other=b; fi - credential=$(jq -r .key "$qa_dir/tenant-$tenant.json") - own=$(cat "$qa_dir/tenant-$tenant-trace-id") - foreign=$(cat "$qa_dir/tenant-$other-trace-id") - qualification_request "$credential" GET "/v1/traces/$own" 200 "$phase-$tenant-own-trace" - jq -e --arg trace "$own" '.summary.trace_id == $trace' "$qa_dir/release-response.json" > /dev/null - qualification_request "$credential" GET "/v1/traces/$foreign" 404 "$phase-$tenant-other-team-denied" - qualification_request "$credential" GET "/v1/traces/$foreign?all_teams=1" 404 "$phase-$tenant-query-cannot-expand-scope" - jq -n --arg sql "SELECT TraceId AS trace_id FROM otel_traces WHERE TraceId IN ('$own','$foreign') ORDER BY TraceId" '{sql:$sql}' > "$qa_dir/request.json" - qualification_request "$credential" POST /v1/traces/query 200 "$phase-$tenant-scoped-sql" "$qa_dir/request.json" - jq -e --arg trace "$own" '[.data[].trace_id] == [$trace]' "$qa_dir/release-response.json" > /dev/null - qualification_request "$credential" GET /lens/signals 403 "$phase-$tenant-admin-surface-denied" - done - qualification_request "$(jq -r .key "$qa_dir/revoked-key.json")" GET /lens/service 401 "$phase-revocation-retained" - qualification_request invalid-release-key GET "/v1/traces/$own" 401 "$phase-invalid-key-denied" - qualification_record "$phase-auth-and-retained-data" passed -} - -qualification_inference() { - if [[ "$qualification_mode" != release ]]; then return; fi - local phase=$1 credential response_id call_id cost attempt - test "$qualification_paid_calls" -lt 3 - credential=$(jq -r .key "$qa_dir/tenant-a.json") - jq -n --arg model "$qualification_model" --arg marker "$(openssl rand -hex 16)" \ - '{model:$model,messages:[{role:"user",content:("Reply with the single word ready. Test marker "+$marker)}],max_completion_tokens:64}' > "$qa_dir/request.json" - qualification_request "$credential" POST /v1/chat/completions 200 "$phase-real-inference" "$qa_dir/request.json" - qualification_paid_calls=$((qualification_paid_calls + 1)) - cp "$qa_dir/release-response.json" "$qa_dir/completion.json" - jq -e '.choices[0].message.content | type == "string" and length > 0' "$qa_dir/completion.json" > /dev/null - jq -e '.usage.prompt_tokens > 0 and .usage.completion_tokens > 0 and .usage.total_tokens == (.usage.prompt_tokens + .usage.completion_tokens)' "$qa_dir/completion.json" > /dev/null - response_id=$(jq -er .id "$qa_dir/completion.json") - call_id=$(awk 'tolower($1)=="x-litellm-call-id:" {gsub("\r", "", $2); print $2}' "$qa_dir/release-response.headers") - cost=$(awk 'tolower($1)=="x-litellm-response-cost:" {gsub("\r", "", $2); print $2}' "$qa_dir/release-response.headers") - [[ "$call_id" =~ ^[A-Za-z0-9_.:-]+$ ]] - jq -en --arg cost "$cost" '($cost|tonumber) > 0' > /dev/null - for attempt in $(seq 1 60); do - qualification_request "$master_key" GET "/spend/logs?request_id=$call_id" 200 "$phase-billing-read" - if jq -e --arg id "$response_id" '[.[] | select(.request_id == $id)] | length == 1' "$qa_dir/release-response.json" > /dev/null; then break; fi - sleep 2 - done - jq -e --arg id "$response_id" --arg cost "$cost" --arg team "$(jq -r .team "$qa_dir/tenant-a.json")" \ - --slurpfile completion "$qa_dir/completion.json" \ - '[.[] | select(.request_id == $id)] | length == 1 and (.[0] | .team_id == $team - and .prompt_tokens == $completion[0].usage.prompt_tokens - and .completion_tokens == $completion[0].usage.completion_tokens - and .total_tokens == $completion[0].usage.total_tokens - and ((.spend - ($cost|tonumber)) | fabs) < 0.00000001)' "$qa_dir/release-response.json" > /dev/null - qualification_total_cost=$(jq -en --arg total "$qualification_total_cost" --arg cost "$cost" '($total|tonumber)+($cost|tonumber)') - jq -en --arg total "$qualification_total_cost" '($total|tonumber) <= 0.10' > /dev/null - qualification_record "$phase-billing-matches-usage-and-team" passed - jq -nc --arg chart "$chart" --arg phase "$phase" --arg response_id "$response_id" \ - --arg call_id "$call_id" --argjson cost "$cost" '{chart:$chart,case:($phase+"-paid-receipt"),result:"passed",response_id:$response_id,request_id:$call_id,cost_usd:$cost}' >> "$qualification_events" -} - -qualification_outage() { - if [[ "$qualification_mode" == smoke ]]; then return; fi - local replicas selector - qualification_read_boundaries before-outage - qualification_inference before-outage - deployment_pods "$control" > "$qa_dir/outage-gateway-before.json" - replicas=$(kubectl -n "$namespace" get deployment lens-lens-worker -o jsonpath='{.spec.replicas}') - test "$replicas" = 1 - selector=$(kubectl -n "$namespace" get deployment lens-lens-worker -o json | jq -r '.spec.selector.matchLabels|to_entries|map("\(.key)=\(.value)")|join(",")') - kubectl -n "$namespace" scale deployment/lens-lens-worker --replicas=0 - kubectl -n "$namespace" wait --for=delete pods -l "$selector" --timeout=90s - qualification_request "$master_key" GET /lens/service 200 outage-service-status - jq -e '.configured == true and .connected == false' "$qa_dir/release-response.json" > /dev/null - qualification_request "$master_key" GET /lens/signals 503 outage-lens-unavailable - qualification_request invalid-release-key GET /lens/signals 401 outage-invalid-key-still-denied - qualification_inference during-outage - kubectl -n "$namespace" scale deployment/lens-lens-worker --replicas="$replicas" - kubectl -n "$namespace" rollout status deployment/lens-lens-worker --timeout=180s - forward_bundled - saved_trace - qualification_read_boundaries after-outage - qualification_inference after-outage - deployment_pods "$control" > "$qa_dir/outage-gateway-after.json" - cmp "$qa_dir/outage-gateway-before.json" "$qa_dir/outage-gateway-after.json" - qualification_record lens-outage-without-gateway-restart passed -} - -qualification_bundled_postgres() { - local chart=bundled-postgresql release=lens-bootstrap - namespace=lens-bundled-postgresql - kubectl create namespace "$namespace" - master_key="sk-$(openssl rand -hex 24)" - printf '%s' "$master_key" > "$qa_dir/bootstrap-master" - kubectl -n "$namespace" create secret generic bootstrap-master --from-file="key=$qa_dir/bootstrap-master" > /dev/null - cat > "$qa_dir/bootstrap.yaml" < /dev/null - kubectl -n "$namespace" get job "$release-migrations" -o json \ - | jq -e '.status.succeeded == 1 and .metadata.annotations["helm.sh/hook"] == null and .metadata.annotations["argocd.argoproj.io/hook"] == null' > /dev/null - qualification_record bundled-postgres-migrated-before-gateway-start passed - kubectl -n "$namespace" scale deployment/"$release" --replicas=1 - kubectl -n "$namespace" rollout status deployment/"$release" --timeout=180s - forward "$release" 14418 4000 - jq -n '{key_alias:"Bundled PostgreSQL persisted key"}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/generate 200 bundled-postgres-key-create "$qa_dir/request.json" - local generated - generated=$(jq -er .key "$qa_dir/release-response.json") - qualification_request "$generated" GET /v1/models 200 bundled-postgres-key-auth - kubectl -n "$namespace" rollout restart deployment/"$release" - kubectl -n "$namespace" rollout status deployment/"$release" --timeout=180s - stop_forwards - forward "$release" 14418 4000 - qualification_request "$generated" GET /v1/models 200 bundled-postgres-key-survives-gateway-restart - jq -n --arg key "$generated" '{keys:[$key]}' > "$qa_dir/request.json" - qualification_request "$master_key" POST /key/delete 200 bundled-postgres-key-delete "$qa_dir/request.json" - qualification_request "$generated" GET /v1/models 401 bundled-postgres-key-revoked - qualification_record bundled-postgres-ordinary-job-two-phase-bootstrap passed - stop_forwards - kubectl delete namespace "$namespace" --wait=true - namespace= -} diff --git a/tests/e2e_harness/migrations/test_lens_release_qualification.py b/tests/e2e_harness/migrations/test_lens_release_qualification.py deleted file mode 100644 index c0d5d272683..00000000000 --- a/tests/e2e_harness/migrations/test_lens_release_qualification.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -from pathlib import Path -from typing import Final - -import pytest - -HARNESS: Final = Path(__file__).resolve().parents[2] / "e2e/migrations/lens_release_qualification.sh" - - -@pytest.mark.parametrize("chart", ["litellm", "litellm-helm"]) -def test_inference_reaches_gateway_and_management_reaches_backend(tmp_path: Path, chart: str) -> None: - curl: Final = tmp_path / "curl" - curl.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "${@: -1}" >> "$qa_dir/requests"\nprintf 200\n') - curl.chmod(0o755) - forward: Final = tmp_path / "forward" - forward.write_text('#!/usr/bin/env bash\nprintf "%s %s %s\\n" "$@" >> "$qa_dir/forwards"\n') - forward.chmod(0o755) - result: Final = subprocess.run( - [ - "bash", - "-euo", - "pipefail", - "-c", - 'source "$HARNESS"\n' - "qualification_forward_control\n" - "qualification_initialize\n" - "qualification_request fixture POST /v1/chat/completions 200 inference\n" - "qualification_request fixture GET /spend/logs 200 billing\n" - "qualification_request fixture POST /model/new 200 management\n", - ], - env={ - **os.environ, - "PATH": f"{tmp_path}:{os.environ['PATH']}", - "HARNESS": str(HARNESS), - "qa_dir": str(tmp_path), - "chart": chart, - "qualification_mode": "release", - "control": "lens-backend" if chart == "litellm" else "lens", - "control_port": "4001" if chart == "litellm" else "4000", - "LENS_QUALIFICATION_RESULTS_DIR": "", - }, - capture_output=True, - text=True, - timeout=10, - check=False, - ) - assert result.returncode == 0, result.stderr - inference_port: Final = 14420 if chart == "litellm" else 14418 - assert (tmp_path / "requests").read_text().splitlines() == [ - f"http://127.0.0.1:{inference_port}/v1/chat/completions", - "http://127.0.0.1:14418/spend/logs", - "http://127.0.0.1:14418/model/new", - ] - expected_forwards: Final = ( - ["lens-backend 14418 4001", "lens-gateway 14420 4000"] if chart == "litellm" else ["lens 14418 4000"] - ) - assert (tmp_path / "forwards").read_text().splitlines() == expected_forwards From 23595f7fe56c7809f54ef051278a7a60a2976366 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 29 Oct 2026 19:46:57 -0700 Subject: [PATCH 25/25] fix(bedrock): backfill published regional service tier prices (#49113) * fix(bedrock): backfill published regional service tier prices * fix(bedrock): remove the eu.anthropic.claude-haiku-4-5-20251001-v1:0 row --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 460 ++++++++++++------ model_prices_and_context_window.json | 460 ++++++++++++------ .../test_get_supported_openai_params.py | 2 +- tests/unit/test_utils.py | 2 +- 4 files changed, 620 insertions(+), 304 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1eaac0fda24..1a1982fddd7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13804,13 +13804,17 @@ }, "bedrock/ap-northeast-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13818,34 +13822,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13854,7 +13864,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13872,20 +13884,24 @@ }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13896,13 +13912,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13959,13 +13975,17 @@ }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13973,34 +13993,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14009,7 +14035,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14027,20 +14055,24 @@ }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14051,22 +14083,24 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, + "input_cost_per_token_batches": 1.55e-07, "input_cost_per_token_flex": 1.55e-07, + "input_cost_per_token_priority": 5.425e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14075,7 +14109,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.24e-06, - "output_cost_per_token_flex": 6.2e-07 + "output_cost_per_token_batches": 6.2e-07, + "output_cost_per_token_flex": 6.2e-07, + "output_cost_per_token_priority": 2.17e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14086,14 +14122,14 @@ "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "supports_tool_choice": true, + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14105,13 +14141,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14120,16 +14156,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14140,13 +14176,13 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_audio_input": false, - "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "supports_vision": true }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14157,13 +14193,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14187,13 +14223,17 @@ }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14201,34 +14241,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14237,24 +14283,30 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -14337,29 +14389,35 @@ "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14368,7 +14426,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14379,13 +14439,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14410,29 +14470,35 @@ "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14441,7 +14507,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14452,13 +14520,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14483,29 +14551,35 @@ "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14514,18 +14588,24 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.86e-06, - "output_cost_per_token_flex": 9.3e-07 + "output_cost_per_token_batches": 9.3e-07, + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_batches": 1.15e-07, "input_cost_per_token_flex": 1.15e-07, + "input_cost_per_token_priority": 4.025e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, + "output_cost_per_token_batches": 5.05e-07, "output_cost_per_token_flex": 5.05e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_priority": 1.7675e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, @@ -14536,20 +14616,24 @@ }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, + "input_cost_per_token_batches": 3.9e-07, "input_cost_per_token_flex": 3.9e-07, + "input_cost_per_token_priority": 1.365e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14593,13 +14677,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14608,16 +14692,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14628,13 +14712,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14675,13 +14759,17 @@ }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14689,34 +14777,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14725,7 +14819,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14743,20 +14839,24 @@ }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14767,13 +14867,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -14908,13 +15008,17 @@ }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14922,42 +15026,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14975,47 +15087,59 @@ }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, + "input_cost_per_token_priority": 8.75e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15023,34 +15147,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -15059,7 +15189,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.2e-06, - "output_cost_per_token_flex": 6e-07 + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15077,20 +15209,24 @@ }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15101,13 +15237,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15543,13 +15679,17 @@ }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15557,42 +15697,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15610,20 +15758,24 @@ }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15634,13 +15786,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -25109,38 +25261,6 @@ "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, - "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { - "cache_creation_input_token_cost": 1.375e-06, - "cache_creation_input_token_cost_above_1hr": 2.2e-06, - "cache_read_input_token_cost": 1.1e-07, - "input_cost_per_token": 1.1e-06, - "deprecation_date": "2026-10-15", - "litellm_provider": "bedrock_converse", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096, - "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06, - "token_count_model": { - "model": "anthropic.claude-haiku-4-5-20251001-v1:0", - "litellm_provider": "bedrock" - } - }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -46798,12 +46918,16 @@ }, "qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, + "input_cost_per_token_priority": 2.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -46813,12 +46937,16 @@ }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46828,12 +46956,18 @@ }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_batches": 7.1e-07, + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46858,12 +46992,16 @@ }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46873,12 +47011,16 @@ }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46888,12 +47030,16 @@ }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -61028,11 +61174,17 @@ }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61041,15 +61193,21 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61058,7 +61216,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1eaac0fda24..1a1982fddd7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13804,13 +13804,17 @@ }, "bedrock/ap-northeast-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13818,34 +13822,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13854,7 +13864,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13872,20 +13884,24 @@ }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13896,13 +13912,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -13959,13 +13975,17 @@ }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13973,34 +13993,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14009,7 +14035,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14027,20 +14055,24 @@ }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14051,22 +14083,24 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, + "input_cost_per_token_batches": 1.55e-07, "input_cost_per_token_flex": 1.55e-07, + "input_cost_per_token_priority": 5.425e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14075,7 +14109,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.24e-06, - "output_cost_per_token_flex": 6.2e-07 + "output_cost_per_token_batches": 6.2e-07, + "output_cost_per_token_flex": 6.2e-07, + "output_cost_per_token_priority": 2.17e-06 }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, @@ -14086,14 +14122,14 @@ "mode": "chat", "output_cost_per_token": 2.22e-06, "output_cost_per_token_flex": 1.11e-06, - "supports_function_calling": true, - "supports_reasoning": true, - "supports_native_structured_output": true, - "supports_tool_choice": true, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html", "supports_audio_input": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_reasoning": true, "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "supports_tool_choice": true, + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", @@ -14105,13 +14141,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14120,16 +14156,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, @@ -14140,13 +14176,13 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "output_cost_per_token_flex": 1.8e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true, - "supports_audio_input": false, - "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "supports_vision": true }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14157,13 +14193,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/ca-central-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14187,13 +14223,17 @@ }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14201,34 +14241,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14237,24 +14283,30 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -14337,29 +14389,35 @@ "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14368,7 +14426,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14379,13 +14439,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-1/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14410,29 +14470,35 @@ "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14441,7 +14507,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14452,13 +14520,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/eu-west-2/meta.llama3-70b-instruct-v1:0": { "deprecation_date": "2027-03-30", @@ -14483,29 +14551,35 @@ "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, + "input_cost_per_token_batches": 2.35e-07, "input_cost_per_token_flex": 2.35e-07, + "input_cost_per_token_priority": 8.225e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14514,18 +14588,24 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.86e-06, - "output_cost_per_token_flex": 9.3e-07 + "output_cost_per_token_batches": 9.3e-07, + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_batches": 1.15e-07, "input_cost_per_token_flex": 1.15e-07, + "input_cost_per_token_priority": 4.025e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, + "output_cost_per_token_batches": 5.05e-07, "output_cost_per_token_flex": 5.05e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_priority": 1.7675e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, @@ -14536,20 +14616,24 @@ }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, + "input_cost_per_token_batches": 3.9e-07, "input_cost_per_token_flex": 3.9e-07, + "input_cost_per_token_priority": 1.365e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_batches": 9.3e-07, "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.255e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/eu-west-3/mistral.mistral-7b-instruct-v0:2": { "deprecation_date": "2027-03-30", @@ -14593,13 +14677,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "supports_vision": false }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, @@ -14608,16 +14692,16 @@ "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "supports_vision": false }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14628,13 +14712,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -14675,13 +14759,17 @@ }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_batches": 3.7e-07, "input_cost_per_token_flex": 3.7e-07, + "input_cost_per_token_priority": 1.295e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_batches": 1.11e-06, "output_cost_per_token_flex": 1.11e-06, + "output_cost_per_token_priority": 3.885e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14689,34 +14777,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_batches": 7.2e-07, "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_batches": 1.8e-07, "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 6.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14725,7 +14819,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.44e-06, - "output_cost_per_token_flex": 7.2e-07 + "output_cost_per_token_batches": 7.2e-07, + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.52e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14743,20 +14839,24 @@ }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_batches": 3.6e-07, "input_cost_per_token_flex": 3.6e-07, + "input_cost_per_token_priority": 1.26e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_batches": 1.8e-06, "output_cost_per_token_flex": 1.8e-06, + "output_cost_per_token_priority": 6.3e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14767,13 +14867,13 @@ "mode": "chat", "output_cost_per_token": 1.44e-06, "output_cost_per_token_flex": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-east-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.011, @@ -14908,13 +15008,17 @@ }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14922,42 +15026,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -14975,47 +15087,59 @@ }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_batches": 2.5e-07, "input_cost_per_token_flex": 2.5e-07, + "input_cost_per_token_priority": 8.75e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15023,34 +15147,40 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -15059,7 +15189,9 @@ "supports_response_schema": true, "supports_vision": false, "output_cost_per_token": 1.2e-06, - "output_cost_per_token_flex": 6e-07 + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15077,20 +15209,24 @@ }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15101,13 +15237,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us-gov-east-1/amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.4e-07, @@ -15543,13 +15679,17 @@ }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_batches": 3.1e-07, "input_cost_per_token_flex": 3.1e-07, + "input_cost_per_token_priority": 1.085e-06, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_batches": 9.25e-07, "output_cost_per_token_flex": 9.25e-07, + "output_cost_per_token_priority": 3.2375e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -15557,42 +15697,50 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_batches": 6e-07, "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, + "input_cost_per_token_priority": 5.25e-07, "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "mode": "chat", + "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_priority": 2.1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", @@ -15610,20 +15758,24 @@ }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_batches": 3e-07, "input_cost_per_token_flex": 3e-07, + "input_cost_per_token_priority": 1.05e-06, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 5.25e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -15634,13 +15786,13 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "output_cost_per_token_flex": 6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html", + "supports_audio_input": false, "supports_function_calling": true, + "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_audio_input": false, - "supports_response_schema": true, - "supports_vision": false, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" + "supports_vision": false }, "bedrock/us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -25109,38 +25261,6 @@ "cache_creation_input_token_cost": 1e-06, "prompt_cache_min_tokens": 2048 }, - "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { - "cache_creation_input_token_cost": 1.375e-06, - "cache_creation_input_token_cost_above_1hr": 2.2e-06, - "cache_read_input_token_cost": 1.1e-07, - "input_cost_per_token": 1.1e-06, - "deprecation_date": "2026-10-15", - "litellm_provider": "bedrock_converse", - "supports_tool_search": true, - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "mode": "chat", - "output_cost_per_token": 5.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", - "supports_assistant_prefill": true, - "supports_computer_use": true, - "supports_function_calling": true, - "supports_pdf_input": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supports_vision": true, - "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true, - "prompt_cache_min_tokens": 4096, - "input_cost_per_token_batches": 5.5e-07, - "output_cost_per_token_batches": 2.75e-06, - "token_count_model": { - "model": "anthropic.claude-haiku-4-5-20251001-v1:0", - "litellm_provider": "bedrock" - } - }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, "litellm_provider": "bedrock", @@ -46798,12 +46918,16 @@ }, "qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, + "input_cost_per_token_priority": 2.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "output_cost_per_token_flex": 6e-07, + "output_cost_per_token_priority": 2.1e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -46813,12 +46937,16 @@ }, "bedrock/ap-northeast-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46828,12 +46956,18 @@ }, "bedrock/ap-south-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_batches": 9e-08, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_batches": 7.1e-07, + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46858,12 +46992,16 @@ }, "bedrock/eu-west-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.41e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.1e-07, + "output_cost_per_token_priority": 2.47e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46873,12 +47011,16 @@ }, "bedrock/eu-west-2/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.2e-07, + "input_cost_per_token_priority": 4e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 9.3e-07, + "output_cost_per_token_priority": 3.26e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -46888,12 +47030,16 @@ }, "bedrock/sa-east-1/qwen.qwen3-next-80b-a3b": { "input_cost_per_token": 1.8e-07, + "input_cost_per_token_flex": 9e-08, + "input_cost_per_token_priority": 3.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.45e-06, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-next-80b-a3b.html", + "output_cost_per_token_flex": 7.2e-07, + "output_cost_per_token_priority": 2.54e-06, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json", "supports_function_calling": true, "supports_native_structured_output": true, "supports_system_messages": true, @@ -61028,11 +61174,17 @@ }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61041,15 +61193,21 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-west-2/zai.glm-5": { "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_priority": 1.75e-06, "output_cost_per_token": 3.2e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 128000, "mode": "chat", + "output_cost_per_token_batches": 1.6e-06, + "output_cost_per_token_flex": 1.6e-06, + "output_cost_per_token_priority": 5.6e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -61058,7 +61216,7 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, diff --git a/tests/unit/litellm_core_utils/test_get_supported_openai_params.py b/tests/unit/litellm_core_utils/test_get_supported_openai_params.py index f9e285cf9fb..8fca7bafedf 100644 --- a/tests/unit/litellm_core_utils/test_get_supported_openai_params.py +++ b/tests/unit/litellm_core_utils/test_get_supported_openai_params.py @@ -4,7 +4,7 @@ from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) -BEDROCK_REAL_MODEL = "eu.anthropic.claude-haiku-4-5-20251001-v1:0" +BEDROCK_REAL_MODEL = "us.anthropic.claude-haiku-4-5-20251001-v1:0" BEDROCK_LABEL = "claude-haiku-4-5" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 243a20a2551..0f471b1de71 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4391,7 +4391,7 @@ class TestBedrockBaseModelLabelKeepsTools: from litellm.utils import get_optional_params result = get_optional_params( - model="eu.anthropic.claude-haiku-4-5-20251001-v1:0", + model="us.anthropic.claude-haiku-4-5-20251001-v1:0", custom_llm_provider="bedrock", base_model="claude-haiku-4-5", tools=self.TOOLS,