From 32469bcab9b4cd795ddabcadca9473286a8cb90c Mon Sep 17 00:00:00 2001 From: Gopal Kamineni <72813883+Gopal3746@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:32:19 -0800 Subject: [PATCH 01/28] fix(vertex_ai): batch Claude models in Anthropic's native row shape (#45715) Backport of #45715 to lts/1.108.x. Cherry-picked from a7030a5a2cab8b1ff525e1e47f8e7bb741609d50 (main). Adapted to the line: the Gemini row mapper keeps the line's signature, since deployment_drop_params on it comes from #31915, which is not on the line, and the Claude row reads drop_params with the line's normalize_drop_params, since drops_unsupported_params is not in core_helpers here. --- litellm/llms/vertex_ai/batches/handler.py | 34 +- .../llms/vertex_ai/batches/transformation.py | 67 ++- .../llms/vertex_ai/files/transformation.py | 258 +++++++++-- tests/unit/batches/test_batch_utils.py | 30 ++ .../llms/vertex_ai/batches/test_handler.py | 47 +- .../vertex_ai/batches/test_transformation.py | 60 +++ .../test_vertex_ai_files_transformation.py | 433 +++++++++++++++++- 7 files changed, 860 insertions(+), 69 deletions(-) diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index de5fe39cfc9..a039347e824 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -14,8 +14,10 @@ from litellm.litellm_core_utils.url_utils import ( ) from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, + MaskedHTTPStatusError, get_async_httpx_client, get_httpx_client, + mask_sensitive_info, ) from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM @@ -80,6 +82,15 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str return response.json() +def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> MaskedHTTPStatusError: + body: Final = mask_sensitive_info(error.response.text) + return MaskedHTTPStatusError( + httpx.HTTPStatusError(f"{error}\n{body}", request=error.request, response=error.response), + message=body, + text=body, + ) + + class VertexAIBatchPrediction(VertexLLM): def __init__(self, gcs_bucket_name: str, *args, **kwargs): super().__init__(*args, **kwargs) @@ -165,11 +176,19 @@ class VertexAIBatchPrediction(VertexLLM): headers=headers, ) - response: Final = sync_handler.post( - url=api_base, - headers=headers, - data=json.dumps(vertex_batch_request), - ) + try: + response: Final = sync_handler.post( + url=api_base, + headers=headers, + data=json.dumps(vertex_batch_request), + ) + except httpx.HTTPStatusError as e: + litellm.verbose_logger.error( + "Vertex AI batch create failed: status=%s, body=%s", + e.response.status_code, + e.response.text[:1000], + ) + raise _vertex_batch_create_error(e) from e payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()} _json_response: Final = payload_view["payload"] @@ -265,13 +284,12 @@ class VertexAIBatchPrediction(VertexLLM): data=json.dumps(vertex_batch_request), ) except httpx.HTTPStatusError as e: - error_body: Final = e.response.text litellm.verbose_logger.error( "Vertex AI batch create failed: status=%s, body=%s", e.response.status_code, - error_body[:1000], + e.response.text[:1000], ) - raise + raise _vertex_batch_create_error(e) from e payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()} _json_response: Final = payload_view["payload"] diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index a8d02045740..2460278294c 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -9,14 +9,41 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.llms.vertex_ai.common_utils import ( VertexAIError, + VertexAIModelRoute, convert_vertex_datetime_to_openai_datetime, + get_vertex_ai_model_route, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest from litellm.types.llms.vertex_ai import * -from litellm.types.llms.vertex_ai import GenerateContentResponseBody +from litellm.types.llms.vertex_ai import GenerateContentResponseBody, VertexPartnerProvider from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage _NATIVE_VERTEX_RESPONSE: Final = TypeAdapter(GenerateContentResponseBody) +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object]) +VERTEX_ANTHROPIC_VERSION: Final = "vertex-2023-10-16" +_ANTHROPIC_PUBLISHER_MODEL_PATH: Final = "publishers/anthropic/models/" + + +def is_vertex_anthropic_batch_model(model: str) -> bool: + """ + Whether a Vertex batch model (bare, `vertex_ai/`-prefixed or a publisher path) is a Claude model, decided the + way Vertex chat routes Claude. Vertex batches Claude only as `publishers/anthropic/models/` with rows in + Anthropic's Messages shape + """ + bare_model: Final = model.rsplit("/", 1)[-1] + return _ANTHROPIC_PUBLISHER_MODEL_PATH in model or ( + get_vertex_ai_model_route(bare_model) is VertexAIModelRoute.PARTNER_MODELS + and VertexPartnerProvider.claude.value in bare_model + ) + + +def as_json_object(value: object) -> Mapping[str, object] | None: + return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None + + +def is_vertex_anthropic_batch_output_response(response_body: Mapping[str, object]) -> bool: + return response_body.get("type") == "message" and isinstance(response_body.get("usage"), dict) def _int_field(mapping: Mapping[str, object], key: str) -> int: @@ -43,8 +70,22 @@ def is_vertex_embedding_batch_output_response(response_body: Mapping[str, object return isinstance(response_body.get("embedding"), dict) +def is_vertex_anthropic_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool: + """ + Whether a Vertex batch output row came from a Claude (Anthropic Messages) request. Vertex echoes only the + request's top-level `custom_id` on these rows, with an Anthropic `message` response on success and an empty + response plus a `status` on failure, so a Claude input file read back is never mistaken for output + """ + if not ("custom_id" in vertex_output_row and "processed_time" in vertex_output_row): + return False + response: Final = as_json_object(vertex_output_row.get("response")) + return (response is not None and is_vertex_anthropic_batch_output_response(response)) or bool( + vertex_output_row.get("status") + ) + + def is_native_vertex_batch_output_row(row: Mapping[str, object]) -> bool: - return isinstance(row.get("request"), dict) + return isinstance(row.get("request"), dict) or is_vertex_anthropic_batch_output_row(row) class NativeVertexBatchCostCalculator(Protocol): @@ -72,6 +113,9 @@ def _native_vertex_row_usage( response_body: Mapping[str, object], calculate_usage: Callable[[GenerateContentResponseBody], Usage], ) -> Usage | None: + anthropic_usage: Final = as_json_object(response_body.get("usage")) + if is_vertex_anthropic_batch_output_response(response_body) and anthropic_usage is not None: + return VertexAIAnthropicConfig().calculate_usage(usage_object=anthropic_usage, reasoning_content=None) if "usageMetadata" not in response_body: if not is_vertex_embedding_batch_output_response(response_body): return None @@ -97,21 +141,24 @@ def native_vertex_batch_row_stats( """ Usage and cost of one native Vertex predictions.jsonl row, a `{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}, "modelVersion": ...}}` - generateContent object or a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}` - embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`). - `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` - does, else the wildcard name so explicit deployment prices still apply; a row without a response, a - generateContent row without `response.usageMetadata`, and a row whose response fails validation are + generateContent object, a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}` + embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`) or a + `{"custom_id": ..., "request": ..., "response": {"type": "message", "model": ..., "usage": {...}}}` Claude object. + `model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` (Claude + rows: `model`) does, else the wildcard name so explicit deployment prices still apply; a row without a response, + a generateContent row without `response.usageMetadata`, and a row whose response fails validation are None (failed). """ - response_body: Final = row.get("response") - if not isinstance(response_body, dict): + response_body: Final = as_json_object(row.get("response")) + if response_body is None: return None usage: Final = _native_vertex_row_usage(response_body, calculate_usage) if usage is None: return None total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens) - model_version: Final = response_body.get("modelVersion") + model_version: Final = response_body.get( + "model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion" + ) deployment_model: Final = model_name if model_name and "*" not in model_name else None model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name) if model is None: diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 396acd270fa..b64d81e4630 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -19,6 +19,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._uuid import uuid +from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, RESPONSE_FORMAT_TOOL_NAME from litellm.files.types import FileContentStreamingResult from litellm.files.utils import FilesAPIUtils from litellm.litellm_core_utils.cloud_storage_security import ( @@ -30,11 +31,13 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items, normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, extract_file_metadata, ) +from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body @@ -43,7 +46,14 @@ from litellm.llms.base_llm.files.transformation import ( BaseFileUploadStream, LiteLLMLoggingObj, ) -from litellm.llms.vertex_ai.batches.transformation import vertex_embedding_prompt_token_count +from litellm.llms.vertex_ai.batches.transformation import ( + VERTEX_ANTHROPIC_VERSION, + as_json_object, + is_vertex_anthropic_batch_model, + is_vertex_anthropic_batch_output_response, + is_vertex_anthropic_batch_output_row, + vertex_embedding_prompt_token_count, +) from litellm.llms.vertex_ai.common_utils import ( convert_vertex_datetime_to_openai_datetime, get_vertex_ai_fine_tuned_endpoint_id, @@ -55,6 +65,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( transform_openai_input_gemini_embed_content, ) +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig from litellm.types.files import StreamingMediaUploadConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -75,6 +86,7 @@ from litellm.types.utils import ( ModelResponse, Usage, ) +from litellm.utils import get_non_default_completion_params, get_optional_params from ..common_utils import VertexAIError from ..vertex_llm_base import VertexBase @@ -83,6 +95,7 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63 _CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_" _VERTEX_BATCH_KEY_FIELD: Final = "key" _MANAGED_GCS_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/([^/?]+)") +_PUBLISHER_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/") _EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = ( ("outputDimensionality", "output_dimensionality"), ("taskType", "task_type"), @@ -220,10 +233,14 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, object]) -> str """ Resolve the OpenAI `custom_id` for a Vertex batch output row. - Embedding rows carry it in the top-level `key` field that Vertex echoes back; - `generateContent` rows have no such field, so it is smuggled through request - labels instead (see `_set_litellm_batch_custom_id_labels`). + Claude rows carry it in their own top-level `custom_id` field and embedding rows in + the top-level `key` field that Vertex echoes back; `generateContent` rows have no + such field, so it is smuggled through request labels instead (see + `_set_litellm_batch_custom_id_labels`). """ + anthropic_custom_id: Final = vertex_output_row.get("custom_id") + if anthropic_custom_id is not None: + return str(anthropic_custom_id) key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD) if key is not None: return unquote(str(key)) @@ -284,6 +301,12 @@ def _is_vertex_generate_content_batch_output_row(vertex_output_row: Mapping[str, ) +def _is_vertex_chat_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool: + return _is_vertex_generate_content_batch_output_row(vertex_output_row) or is_vertex_anthropic_batch_output_row( + vertex_output_row + ) + + def _try_parse_vertex_batch_output_row(line: bytes) -> _VertexBatchRow | None: try: row: Final = _parse_vertex_batch_output_row(line.decode("utf-8")) @@ -405,6 +428,37 @@ def _openai_batch_output_row( } +def _vertex_anthropic_batch_output_row_to_openai( + custom_id: str, + response: object, + mock_httpx_response: httpx.Response, +) -> _OpenAIBatchOutputRow: + """ + One OpenAI batch output row from a Vertex Claude row's Anthropic `message` response + """ + anthropic_response: Final = as_json_object(response) + if anthropic_response is None or not is_vertex_anthropic_batch_output_response(anthropic_response): + return _openai_batch_output_row( + custom_id=custom_id, + error_code="vertex_ai_error", + error_message="Claude batch row has no message response", + ) + try: + model_response: Final = VertexAIAnthropicConfig().transform_parsed_response( + completion_response=dict(anthropic_response), + raw_response=mock_httpx_response, + model_response=ModelResponse(), + json_mode=True, + ) + except Exception as e: # noqa: BLE001 # one malformed row must fail alone, not the whole output file + return _openai_batch_output_row( + custom_id=custom_id, + error_code="transformation_error", + error_message=f"Failed to transform response: {e}", + ) + return _openai_batch_output_row(custom_id=custom_id, body=model_response.model_dump()) + + def _split_vertex_batch_key(vertex_output_row: Mapping[str, object]) -> tuple[str, int, int]: """ Resolve `(custom_id, index within that custom_id, group size)` for a Vertex batch @@ -679,21 +733,123 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows( ) +_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset( + ("model", "messages", "custom_llm_provider", "drop_params", "stream", "stream_options") +) +_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"} +_CLAUDE_BATCH_DEPLOYMENT_PARAMS: Final = ( + _CLAUDE_BATCH_OPENAI_PARAMS - _CLAUDE_BATCH_ROW_FIXED_PARAMS - {"extra_headers"} +) | {"drop_params"} + + +def _claude_batch_deployment_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + {key: value for key, value in litellm_params.items() if key in _CLAUDE_BATCH_DEPLOYMENT_PARAMS} + ) + + +def _names_the_response_format_tool(tool: object) -> bool: + match tool: + case {"function": {"name": str(name)}}: + return name == RESPONSE_FORMAT_TOOL_NAME + case _: + return False + + +def _openai_batch_jsonl_entry_to_vertex_anthropic_row( + custom_id: object, + messages: list[AllMessageValues], + request_params: Mapping[str, object], + model: str, + deployment_params: Mapping[str, object], +) -> Mapping[str, object]: + """ + Transforms an OpenAI chat batch entry into the Anthropic Messages row Vertex batches Claude with, + `{"custom_id": ..., "request": {"messages": [...], "anthropic_version": "vertex-2023-10-16", "max_tokens": ...}}` + """ + if custom_id is None: + raise ValueError("custom_id is required on Claude lines, since Claude output rows are matched back only by it") + config: Final = VertexAIAnthropicConfig() + bare_model: Final = model.rsplit("/", 1)[-1] + params: Final = {**deployment_params, **request_params} + drop_params: Final = litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True + chat_params: Final = { + **get_non_default_completion_params(params), + **{key: value for key, value in params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS}, + } + if any(map(_names_the_response_format_tool, iter_items(chat_params.get("tools") or ()))): + raise ValueError( + f"tool name {RESPONSE_FORMAT_TOOL_NAME!r} is reserved for response_format on Claude, " + "and batch output cannot tell a user tool by that name apart, so rename it" + ) + optional_params: Final[Mapping[str, object]] = get_optional_params( + model=bare_model, + custom_llm_provider="vertex_ai", + drop_params=drop_params, + messages=messages, + **{ # pyright: ignore[reportArgumentType] # batch JSONL bodies are untyped; get_optional_params checks each param as chat does + key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS + }, + ) + request_state: Final[dict[str, object]] = {"drop_params": drop_params} + anthropic_request: Final[Mapping[str, object]] = config.transform_request( + model=bare_model, + messages=messages, + optional_params={ + **{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")}, + "anthropic_version": VERTEX_ANTHROPIC_VERSION, + }, + litellm_params=request_state, + headers={}, + ) + renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)) + if renamed_tools: + raise ValueError( + f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, " + "and batch output cannot map a renamed tool back, so rename these tools" + ) + extra_body: Final = as_json_object(optional_params.get("extra_body")) + request: Final[Mapping[str, object]] = ( + anthropic_request + if extra_body is None + else { + **anthropic_request, + **config.transform_extra_body( + extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={} + ), + } + ) + return {"custom_id": str(custom_id), "request": request} + + def _openai_batch_jsonl_entry_to_vertex_rows( openai_entry: dict[str, Any], map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + batch_model: str | None = None, + deployment_params: Mapping[str, object] = MappingProxyType({}), ) -> tuple[Mapping[str, object], ...]: """ Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to. + `batch_model` is the deployment model the batch runs against and wins over the entry's own `body.model`, + which picks the row shape: Anthropic Messages for Claude, `GenerateContentRequest` otherwise. + jsonl body for vertex is {"request": } Example Vertex jsonl {"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}} """ + entry_body: Final = openai_entry.get("body") + model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "") + is_anthropic_model: Final = is_vertex_anthropic_batch_model(model) if _is_embeddings_batch_entry(openai_entry): + if is_anthropic_model: + raise VertexAIError( + status_code=400, + message=f"Vertex AI batch embeddings are not supported for Claude models ('{model}')", + ) return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry) - if not isinstance(openai_entry.get("body"), dict): + if not isinstance(entry_body, dict): raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object") openai_request_body: Final = openai_entry["body"] chat_request_body: Final = ( @@ -701,8 +857,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows( if _is_responses_batch_entry(openai_entry) else openai_request_body ) + chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", [])) + if is_anthropic_model: + return ( + _openai_batch_jsonl_entry_to_vertex_anthropic_row( + custom_id=openai_entry.get("custom_id"), + messages=chat_messages, + request_params={ + key: value for key, value in chat_request_body.items() if key not in ("model", "messages") + }, + model=model, + deployment_params=deployment_params, + ), + ) vertex_request_body: Final = transform_request_body( - messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])), + messages=chat_messages, model=chat_request_body.get("model", ""), optional_params=map_openai_to_vertex_params(chat_request_body), custom_llm_provider="vertex_ai", @@ -854,15 +1023,24 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream): self, openai_file_content: FileTypes, map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]], + batch_model: str | None = None, + deployment_params: Mapping[str, object] = MappingProxyType({}), ) -> None: self._openai_file_content = openai_file_content self._map_openai_to_vertex_params = map_openai_to_vertex_params + self._batch_model = batch_model + self._deployment_params = deployment_params def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]: first = True for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content): try: - vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params) + vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows( + entry, + self._map_openai_to_vertex_params, + batch_model=self._batch_model, + deployment_params=self._deployment_params, + ) except Exception as e: raise VertexAIError( status_code=400, @@ -918,16 +1096,31 @@ class _RawFileUploadStream(BaseFileUploadStream): return _iter_raw_file_chunks(self._file_content) +def _deployment_batch_model(litellm_params: Mapping[str, object]) -> str | None: + model: Final = litellm_params.get("model") + return model.removeprefix("vertex_ai/") if isinstance(model, str) and model else None + + +def _publisher_model_path(raw_model: str) -> str: + if _PUBLISHER_MODEL_PATH_PATTERN.search(raw_model): + return raw_model + publisher: Final = "anthropic" if is_vertex_anthropic_batch_model(raw_model) else "google" + return f"publishers/{publisher}/models/{raw_model}" + + def _managed_batch_object_name(raw_model: str, *, passthrough: bool) -> str: endpoint_id: Final = get_vertex_ai_fine_tuned_endpoint_id(raw_model) - model_path: Final = ( - f"endpoints/{endpoint_id}" - if endpoint_id is not None - else (raw_model if "publishers/google/models" in raw_model else f"publishers/google/models/{raw_model}") - ) - safe_model_path: Final = sanitize_cloud_object_path(model_path, fallback="model") + model_path: Final = f"endpoints/{endpoint_id}" if endpoint_id is not None else _publisher_model_path(raw_model) prefix: Final = _PASSTHROUGH_MANAGED_GCS_PREFIX if passthrough else VERTEX_AI_MANAGED_GCS_PREFIX - return f"{prefix}{safe_model_path}/{uuid.uuid4()}" + return f"{prefix}{_safe_managed_model_path(model_path)}/{uuid.uuid4()}" + + +def _safe_managed_model_path(model_path: str) -> str: + if not is_vertex_anthropic_batch_model(model_path): + return sanitize_cloud_object_path(model_path, fallback="model") + return "@".join( + sanitize_cloud_object_path(version_part, fallback="model") for version_part in model_path.split("@") + ) class VertexAIFilesConfig(VertexBase, BaseFilesConfig): @@ -1030,10 +1223,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): fallback_filename="file", ) - def _get_configured_bucket_name(self, litellm_params: dict) -> str: + def _get_configured_bucket_name(self, litellm_params: Mapping[str, object]) -> str: + configured_bucket_name: Final = litellm_params.get("gcs_bucket_name") or litellm_params.get("bucket_name") bucket_name: Final = ( - litellm_params.get("gcs_bucket_name") - or litellm_params.get("bucket_name") + (configured_bucket_name if isinstance(configured_bucket_name, str) else None) or os.getenv("GCS_BATCH_BUCKET_NAME") or os.getenv("GCS_BUCKET_NAME") ) @@ -1047,7 +1240,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): api_key: str | None, model: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], data: CreateFileRequest, ) -> str: """ @@ -1071,11 +1264,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): raise ValueError("file is required") if purpose is None: raise ValueError("purpose is required") - configured_model: Final = litellm_params.get("model") object_name = self.get_object_name( file_data, purpose, - deployment_model=configured_model if isinstance(configured_model, str) else None, + deployment_model=_deployment_batch_model(litellm_params), passthrough=is_passthrough_batch_upload(data, litellm_params), ) if object_prefix: @@ -1126,7 +1318,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> bytes | str | dict: """ 2 Cases: @@ -1157,6 +1349,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): body_stream=_OpenAIToVertexBatchUploadStream( file_data, self._map_openai_to_vertex_params, + batch_model=_deployment_batch_model(litellm_params), + deployment_params=_claude_batch_deployment_params(litellm_params), ), content_type="application/json", ) @@ -1207,7 +1401,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException: return VertexAIError(status_code=status_code, message=error_message, headers=headers) - def _parse_gcs_uri(self, file_id: str, litellm_params: dict | None = None) -> tuple[str, str]: + def _parse_gcs_uri(self, file_id: str, litellm_params: Mapping[str, object] | None = None) -> tuple[str, str]: """ Validate a managed GCS file_id and return (bucket, url-encoded-object-path). """ @@ -1225,7 +1419,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_id: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}" @@ -1255,7 +1449,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_id: str, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}" @@ -1298,7 +1492,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): self, file_content_request, optional_params: dict, - litellm_params: dict, + litellm_params: Mapping[str, object], ) -> tuple[str, dict]: file_id: Final = file_content_request.get("file_id", "") bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params) @@ -1395,7 +1589,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): headers=MappingProxyType({**headers, "content-length": str(len(transformed_content))}), ) - if not _is_vertex_generate_content_batch_output_row(first_row): + if not _is_vertex_chat_batch_output_row(first_row): return FileContentStreamingResult(stream_iterator=replayed_stream, headers=headers) return FileContentStreamingResult( @@ -1479,10 +1673,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): # first line is not valid UTF-8/JSON) raises and falls through to the # passthrough below, leaving the content untouched. first_row: Final = _parse_vertex_batch_output_row(first_line) - if not ( - _is_vertex_embeddings_batch_output_row(first_row) - or _is_vertex_generate_content_batch_output_row(first_row) - ): + if not (_is_vertex_embeddings_batch_output_row(first_row) or _is_vertex_chat_batch_output_row(first_row)): return content context: Final = _new_vertex_batch_output_row_transform_context() @@ -1547,6 +1738,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): error_message=status, ) + if is_vertex_anthropic_batch_output_row(vertex_output): + return _vertex_anthropic_batch_output_row_to_openai( + custom_id=custom_id, + response=vertex_output.get("response"), + mock_httpx_response=mock_httpx_response, + ) + # Transform successful response using existing transformation vertex_response: Final = vertex_output.get("response", {}) diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index ed9bceba927..1ee85522ae3 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -2296,6 +2296,36 @@ async def test_native_vertex_rows_route_to_vertex_cost_path_without_flag(monkeyp assert {(call["model"], call["custom_llm_provider"]) for call in calls} == {("gemini-2.5-flash", "vertex_ai")} +@pytest.mark.asyncio +async def test_raw_vertex_claude_rows_are_priced_like_with_the_native_output_flag(monkeypatch): + claude_message: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "Blue."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 5}, + } + rows: Final = [ + {"custom_id": "ok", "processed_time": "2026-10-29T03:00:00Z", "response": claude_message, "status": ""}, + {"custom_id": "bad", "processed_time": "2026-10-29T03:00:00Z", "response": {}, "status": "Bad Request"}, + ] + + async def cost_with_flag(flag: bool) -> bu.BatchCostUsageResult: + monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", flag, raising=False) + return await bu.calculate_batch_cost_and_usage( + file_content_dictionary=rows, custom_llm_provider="vertex_ai", model_name="vertex_ai/claude-sonnet-4-6" + ) + + unflagged: Final = await cost_with_flag(False) + flagged: Final = await cost_with_flag(True) + + assert unflagged.cost > 0 + assert (unflagged.successful_requests, unflagged.failed_requests) == (1, 1) + assert (unflagged.cost, unflagged.usage.total_tokens) == (flagged.cost, flagged.usage.total_tokens) + + @pytest.mark.asyncio async def test_openai_shaped_vertex_rows_keep_the_generic_path_without_flag(monkeypatch): monkeypatch.setattr(litellm, "disable_vertex_batch_output_transformation", False, raising=False) diff --git a/tests/unit/llms/vertex_ai/batches/test_handler.py b/tests/unit/llms/vertex_ai/batches/test_handler.py index 91a2adb9d19..2c246e065a8 100644 --- a/tests/unit/llms/vertex_ai/batches/test_handler.py +++ b/tests/unit/llms/vertex_ai/batches/test_handler.py @@ -30,6 +30,7 @@ from __future__ import annotations import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -41,9 +42,11 @@ from litellm.llms.vertex_ai.batches.handler import ( # noqa: E402 VertexAIBatchPrediction, ) from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 +from litellm.proxy.utils import handle_exception_on_proxy # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 HMOD = "litellm.llms.vertex_ai.batches.handler" +_CLIENT_ERROR_429: Final = "Client error '429 Too Many Requests' for url 'https://x/batchPredictionJobs'" TOKEN = "ya29.fake-access-token" PROJECT = "my-project" LOCATION = "us-central1" @@ -370,19 +373,19 @@ def test_create_batch_sync_endpoint_without_deployed_model_raises_400(): client.post.assert_not_called() -def test_create_batch_sync_httpstatuserror_propagates(): - """``HTTPHandler.post`` raises for non-2xx via ``raise_for_status``; the - sync create path must surface that error, not swallow it.""" +def test_create_batch_sync_httpstatuserror_carries_vertex_status_body_and_headers(): h = _make_handler() client = MagicMock() request = httpx.Request("POST", "https://x/batchPredictionJobs") - err_response = httpx.Response(status_code=500, request=request, text="boom") + err_response: Final = httpx.Response( + status_code=429, request=request, text="boom", headers={"retry-after": "30"} + ) client.post.side_effect = httpx.HTTPStatusError( - "boom", request=request, response=err_response + _CLIENT_ERROR_429, request=request, response=err_response ) with patch(f"{HMOD}.get_httpx_client", return_value=client): - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(httpx.HTTPStatusError) as exc_info: h.create_batch( _is_async=False, create_batch_data=CREATE_DATA, @@ -393,6 +396,11 @@ def test_create_batch_sync_httpstatuserror_propagates(): timeout=600.0, max_retries=None, ) + assert exc_info.value.response.status_code == 429 + assert exc_info.value.response.headers["retry-after"] == "30" + assert handle_exception_on_proxy(exc_info.value).code == "429" + assert (getattr(exc_info.value, "message", None), getattr(exc_info.value, "text", None)) == ("boom", "boom") + assert str(exc_info.value).splitlines() == [_CLIENT_ERROR_429, "boom"] def test_create_batch_input_file_id_without_model_raises_400_before_post(): @@ -800,14 +808,16 @@ def test_cancel_batch_sync_httpstatuserror_logged_and_reraised(): client.get.assert_not_called() -def test_create_batch_async_httpstatuserror_logged_and_reraised(): +def test_create_batch_async_httpstatuserror_carries_vertex_status_body_and_headers(): h = _make_handler() async_client = MagicMock() request = httpx.Request("POST", "https://x/batchPredictionJobs") - err_response = httpx.Response(status_code=500, request=request, text="boom") + err_response: Final = httpx.Response( + status_code=429, request=request, text="boom", headers={"retry-after": "30"} + ) async_client.post = AsyncMock( side_effect=httpx.HTTPStatusError( - "boom", request=request, response=err_response + _CLIENT_ERROR_429, request=request, response=err_response ) ) @@ -825,8 +835,13 @@ def test_create_batch_async_httpstatuserror_logged_and_reraised(): timeout=600.0, max_retries=None, ) - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(httpx.HTTPStatusError) as exc_info: _run(coro) + assert exc_info.value.response.status_code == 429 + assert exc_info.value.response.headers["retry-after"] == "30" + assert handle_exception_on_proxy(exc_info.value).code == "429" + assert (getattr(exc_info.value, "message", None), getattr(exc_info.value, "text", None)) == ("boom", "boom") + assert str(exc_info.value).splitlines() == [_CLIENT_ERROR_429, "boom"] def test_async_retrieve_batch_non_200_raises(): @@ -1043,15 +1058,7 @@ async def test_vertex_list_batches(monkeypatch): @pytest.mark.asyncio -async def test_vertex_async_create_batch_logs_error_body_on_http_error(): - """ - When Vertex AI returns an HTTP error (e.g. 400), _async_create_batch should - re-raise httpx.HTTPStatusError (not swallow it) and log the response body. - - Before the fix the error body was lost because AsyncHTTPHandler.post() - calls raise_for_status() internally, raising before the handler's own - status-code check could log the body. - """ +async def test_vertex_async_create_batch_httpstatuserror_carries_vertex_error_body(): from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction handler = VertexAIBatchPrediction(gcs_bucket_name="test-bucket") @@ -1081,4 +1088,4 @@ async def test_vertex_async_create_batch_logs_error_body_on_http_error(): ) assert exc_info.value.response.status_code == 400 - assert "gemini-2.0-flash" in exc_info.value.response.text + assert "Do not support publisher model gemini-2.0-flash" in str(exc_info.value) diff --git a/tests/unit/llms/vertex_ai/batches/test_transformation.py b/tests/unit/llms/vertex_ai/batches/test_transformation.py index 064845a7266..1440a57a15f 100644 --- a/tests/unit/llms/vertex_ai/batches/test_transformation.py +++ b/tests/unit/llms/vertex_ai/batches/test_transformation.py @@ -21,6 +21,8 @@ import pytest from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402 VertexAIBatchTransformation, + is_vertex_anthropic_batch_model, + native_vertex_batch_row_stats, ) from litellm.llms.vertex_ai.common_utils import VertexAIError # noqa: E402 from litellm.types.utils import LiteLLMBatch # noqa: E402 @@ -493,3 +495,61 @@ def test_get_gcs_uri_prefix_keeps_passthrough_segment_so_output_lands_beside_inp T._get_gcs_uri_prefix_from_file(PASSTHROUGH_INPUT_FILE) == "gs://litellm-testing-bucket/litellm-vertex-files/passthrough/publishers/google/models/gemini-2.5-flash" ) + + +CLAUDE_INPUT_FILE = ( + "gs://litellm-testing-bucket/litellm-vertex-files/publishers/anthropic/models/claude-sonnet-4-6/uuid-1" +) + + +def test_claude_batch_job_targets_the_anthropic_publisher_model(): + job: Final = T.transform_openai_batch_request_to_vertex_ai_batch_request( + {"input_file_id": CLAUDE_INPUT_FILE}, vertex_project="p", vertex_location="us-east5" + ) + assert job["model"] == "publishers/anthropic/models/claude-sonnet-4-6" + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("claude-sonnet-4-6", True), + ("vertex_ai/claude-opus-4-1@20250805", True), + ("publishers/anthropic/models/claude-haiku-4-5", True), + ("gemini-2.5-flash", False), + ("publishers/google/models/gemini-2.5-flash", False), + ], +) +def test_is_vertex_anthropic_batch_model(model: str, expected: bool): + assert is_vertex_anthropic_batch_model(model) is expected + + +def test_claude_output_row_is_costed_from_anthropic_usage_and_its_own_model(): + def cost_calculator(usage, model, custom_llm_provider=None, model_info=None, vertex_location=None) -> tuple[float, float]: + if model != "claude-sonnet-4-6": + return 0.0, 0.0 + return usage.prompt_tokens * 0.001, usage.completion_tokens * 0.002 + + stats: Final = native_vertex_batch_row_stats( + { + "custom_id": "r1", + "request": {"messages": [{"role": "user", "content": "hi"}], "anthropic_version": "vertex-2023-10-16"}, + "response": { + "id": "msg_vrtx_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 12, "output_tokens": 5}, + }, + }, + None, + model_info=None, + calculate_usage=lambda _: pytest.fail("Claude rows must not go through the Gemini usage parser"), + cost_calculator=cost_calculator, + ) + + assert stats is not None, "a successful Claude row must be costed, not counted as failed" + assert (stats.usage.prompt_tokens, stats.usage.completion_tokens, stats.total_tokens) == (12, 5, 17) + assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((0.012, 0.010)) + assert (stats.prompt_cost, stats.completion_cost) == pytest.approx((12 * 0.001, 5 * 0.002)) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index dde3df6fcc0..c235e438847 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -5,7 +5,8 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from typing import Final from types import MappingProxyType from urllib.parse import parse_qs, urlparse @@ -13,6 +14,8 @@ import httpx import pytest from unittest.mock import MagicMock +import litellm +from litellm.llms.base_llm.files.transformation import BaseFileUploadStream from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -165,6 +168,17 @@ class TestBatchObjectNaming: object_name = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "gemini-2.5-flash"}}]) assert object_name.startswith("litellm-vertex-files/publishers/google/models/gemini-2.5-flash/") + @pytest.mark.parametrize( + ("model", "stored_model"), + [ + ("multimodalembedding@001", "publishers/google/models/multimodalembedding_001/"), + ("claude-haiku-4-5@20251001", "publishers/anthropic/models/claude-haiku-4-5@20251001/"), + ], + ) + def test_only_claude_models_keep_the_at_version_in_their_object_name(self, config, model, stored_model): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": model}}]) + assert object_name.startswith(f"litellm-vertex-files/{stored_model}") + def test_should_store_fine_tuned_endpoint_under_endpoints_path(self, config): """A numeric endpoint id must not be filed under publishers/google/models/gemini/, which the batch transformation later mangles into a nonexistent publisher model (LIT-6899).""" @@ -1948,3 +1962,420 @@ class TestVertexEmbeddingsBatchOutputTranslation: content = json.dumps(legacy_row).encode("utf-8") assert config._try_transform_vertex_batch_output_to_openai(content) == content + + +CLAUDE_DEPLOYMENT = {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/claude-sonnet-4-6"} + + +def _claude_chat_entry(custom_id: str | None, content: str, **body: object) -> dict[str, object]: + entry: Final[dict[str, object]] = { + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": "claude-sonnet-4-6", "messages": [{"role": "user", "content": content}], **body}, + } + return entry if custom_id is None else {"custom_id": custom_id, **entry} + + +def _uploaded_rows( + config: VertexAIFilesConfig, litellm_params: Mapping[str, object], entries: Sequence[Mapping[str, object]] +) -> list: + upload: Final = config.transform_create_file_request( + model="", + create_file_data={ + "file": ("batch.jsonl", "\n".join(json.dumps(entry) for entry in entries).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={}, + litellm_params=dict(litellm_params), + ) + match upload: + case {"streaming_media_upload": {"body_stream": BaseFileUploadStream() as stream}}: + return [json.loads(line) for line in b"".join(stream.iter_bytes()).splitlines()] + case _: + pytest.fail(f"expected a streamed upload, got {upload!r}") + + +def _claude_output_row(custom_id: str, text: str, content: object = None) -> dict[str, object]: + return { + "custom_id": custom_id, + "status": "", + "processed_time": "2026-10-28T20:37:47.634+00:00", + "response": { + "id": "msg_vrtx_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": text}] if content is None else content, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 5}, + }, + } + + +class TestVertexClaudeBatch: + """Claude on Vertex batches only as `publishers/anthropic/models/` with Anthropic Messages rows + (https://github.com/BerriAI/litellm/issues/45671)""" + + @pytest.mark.parametrize("claude_model", ["claude-sonnet-4-6", "claude-haiku-4-5@20251001"]) + def test_claude_upload_creates_a_batch_job_for_the_anthropic_publisher_model(self, config, claude_model): + from litellm.llms.vertex_ai.batches.transformation import VertexAIBatchTransformation + + url: Final = config.get_complete_file_url( + api_base=None, + api_key=None, + model="", + optional_params={}, + litellm_params={**CLAUDE_DEPLOYMENT, "model": f"vertex_ai/{claude_model}"}, + data={"file": ("batch.jsonl", json.dumps(_claude_chat_entry("r1", "hi")).encode()), "purpose": "batch"}, + ) + object_name: Final = parse_qs(urlparse(url).query)["name"][0] + + job: Final = VertexAIBatchTransformation.transform_openai_batch_request_to_vertex_ai_batch_request( + {"input_file_id": f"gs://my-bucket/{object_name}"}, vertex_project="p", vertex_location="us-east5" + ) + + assert job["model"] == f"publishers/anthropic/models/{claude_model}", object_name + + def test_claude_body_model_without_deployment_is_filed_under_anthropic_publisher(self, config): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl([{"body": {"model": "claude-haiku-4-5"}}]) + assert object_name.startswith("litellm-vertex-files/publishers/anthropic/models/claude-haiku-4-5/") + + def test_publisher_path_model_is_not_nested_under_google(self, config): + object_name: Final = config._get_gcs_object_name_from_batch_jsonl( + [{"body": {}}], deployment_model="vertex_ai/publishers/anthropic/models/claude-sonnet-4-6" + ) + assert object_name.startswith("litellm-vertex-files/publishers/anthropic/models/claude-sonnet-4-6/") + + def test_claude_rows_are_uploaded_in_anthropic_messages_shape(self, config): + rows: Final = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [_claude_chat_entry("request-1", "Hello!", max_tokens=50, temperature=0.2)], + ) + + assert rows == [ + { + "custom_id": "request-1", + "request": { + "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello!"}]}], + "max_tokens": 50, + "temperature": 0.2, + "anthropic_version": "vertex-2023-10-16", + }, + } + ] + + def test_claude_system_message_becomes_the_anthropic_system_field(self, config): + entry: Final = _claude_chat_entry( + "r1", + "unused", + max_tokens=10, + messages=[{"role": "system", "content": "be brief"}, {"role": "user", "content": "Hello!"}], + ) + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert row["request"]["system"] == [{"type": "text", "text": "be brief"}] + assert [message["role"] for message in row["request"]["messages"]] == ["user"] + + def test_claude_row_drops_an_unsupported_param_under_deployment_drop_params(self, config, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + (row,) = _uploaded_rows( + config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9", "drop_params": True}, [entry] + ) + + assert row["request"]["max_tokens"] == 10 + assert "top_p" not in row["request"] + + def test_claude_row_drops_a_param_the_deployment_lists_in_additional_drop_params(self, config, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + (row,) = _uploaded_rows( + config, + {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9", "additional_drop_params": ["top_p"]}, + [entry], + ) + + assert row["request"]["max_tokens"] == 10 + assert "top_p" not in row["request"] + + def test_claude_rows_take_deployment_params_as_defaults_the_row_overrides(self, config): + rows: Final = _uploaded_rows( + config, + {**CLAUDE_DEPLOYMENT, "max_tokens": 64, "temperature": 0.3}, + [_claude_chat_entry("r1", "hi"), _claude_chat_entry("r2", "hi", max_tokens=10)], + ) + + assert [(row["request"]["max_tokens"], row["request"]["temperature"]) for row in rows] == [(64, 0.3), (10, 0.3)] + + def test_claude_row_with_an_unsupported_param_is_refused_without_drop_params(self, config, monkeypatch): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_p=0.5) + + with pytest.raises(VertexAIError, match=r"line 1 .*top_p=0.5.*drop_params") as exc_info: + _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"}, [entry]) + assert exc_info.value.status_code == 400 + + def test_claude_deployment_wins_over_a_gemini_body_model(self, config): + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, model="gemini-2.5-flash") + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert row["request"]["anthropic_version"] == "vertex-2023-10-16" + assert "contents" not in row["request"] + + def test_gemini_deployment_still_uploads_generate_content_rows(self, config): + (row,) = _uploaded_rows( + config, + {"gcs_bucket_name": "my-bucket", "model": "vertex_ai/gemini-2.5-flash"}, + [_claude_chat_entry("r1", "hi")], + ) + assert row["request"]["contents"] == [{"role": "user", "parts": [{"text": "hi"}]}] + assert "custom_id" not in row + + def test_claude_embeddings_batch_line_is_rejected_with_400(self): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + entry: Final = { + "custom_id": "e1", + "url": "/v1/embeddings", + "body": {"model": "claude-sonnet-4-6", "input": "hi"}, + } + with pytest.raises(VertexAIError) as exc_info: + _openai_batch_jsonl_entry_to_vertex_rows(entry, VertexAIFilesConfig()._map_openai_to_vertex_params) + assert exc_info.value.status_code == 400 + + def test_claude_output_rows_become_openai_batch_rows(self, config): + content: Final = b"\n".join( + json.dumps(row).encode() for row in (_claude_output_row("a", "first"), _claude_output_row("b", "second")) + ) + + rows: Final = [ + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ] + + assert [row["custom_id"] for row in rows] == ["a", "b"] + assert [row["error"] for row in rows] == [None, None] + bodies: Final = [row["response"]["body"] for row in rows] + assert [body["choices"][0]["message"]["content"] for body in bodies] == ["first", "second"] + assert [body["choices"][0]["finish_reason"] for body in bodies] == ["stop", "stop"] + assert [(body["usage"]["prompt_tokens"], body["usage"]["completion_tokens"]) for body in bodies] == [ + (12, 5), + (12, 5), + ] + assert {body["model"] for body in bodies} == {"claude-sonnet-4-6"} + + @pytest.mark.parametrize("failed_first", [False, True]) + def test_failed_claude_output_row_is_an_openai_error_row(self, config, failed_first): + failed_row: Final = { + "custom_id": "bad", + "status": 'Bad Request: {"type": "error", "error": {"type": "invalid_request_error", "message": "too long"}}', + "processed_time": "2026-10-28T20:37:48.413+00:00", + "response": {}, + } + rows_in: Final = (failed_row, _claude_output_row("ok", "fine")) + content: Final = b"\n".join(json.dumps(row).encode() for row in (rows_in if failed_first else rows_in[::-1])) + + rows_out: Final = { + row["custom_id"]: row + for row in ( + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ) + } + + assert rows_out["ok"]["response"]["body"]["choices"][0]["message"]["content"] == "fine" + assert rows_out["bad"]["response"] is None + assert rows_out["bad"]["error"]["code"] == "vertex_ai_error" + assert "too long" in rows_out["bad"]["error"]["message"] + + def test_claude_row_keeps_provider_only_params_and_leaves_out_litellm_ones(self, config): + (row,) = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [_claude_chat_entry("r1", "hi", max_tokens=10, top_k=5, drop_params=True, metadata={"team": "a"})], + ) + + assert row["request"]["top_k"] == 5 + assert not {"drop_params", "metadata", "extra_body"} & set(row["request"]) + + def test_claude_row_top_k_follows_drop_params_on_a_model_without_sampling_params(self, config, monkeypatch): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}) + monkeypatch.setattr(litellm, "drop_params", False) + deployment: Final = {**CLAUDE_DEPLOYMENT, "model": "vertex_ai/claude-zeta-9"} + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, top_k=5) + + with pytest.raises(VertexAIError, match=r"line 1 .*top_k") as exc_info: + _uploaded_rows(config, deployment, [entry]) + (row,) = _uploaded_rows(config, {**deployment, "drop_params": True}, [entry]) + + assert exc_info.value.status_code == 400 + assert "top_k" not in row["request"] + + @pytest.mark.parametrize("param", [{"n": 2}, {"seed": 7}, {"presence_penalty": 0.5}, {"logprobs": True}]) + def test_claude_row_param_claude_does_not_support_follows_drop_params(self, config, monkeypatch, param): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + monkeypatch.setattr(litellm, "drop_params", False) + (name,) = param + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, **param) + + with pytest.raises(VertexAIError, match=rf"line 1 .*{name}") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + (row,) = _uploaded_rows(config, {**CLAUDE_DEPLOYMENT, "drop_params": True}, [entry]) + + assert exc_info.value.status_code == 400 + assert name not in row["request"] + + def test_claude_row_refuses_a_tool_name_claude_would_rename(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + def tool(name: str) -> dict[str, object]: + return {"type": "function", "function": {"name": name, "parameters": {"type": "object", "properties": {}}}} + + renamed: Final = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool("get.weather")]) + kept: Final = _claude_chat_entry("r2", "hi", max_tokens=10, tools=[tool("get_weather")]) + + with pytest.raises(VertexAIError, match=r"line 1 .*get\.weather") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [renamed]) + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [kept]) + + assert exc_info.value.status_code == 400 + assert [t["name"] for t in row["request"]["tools"]] == ["get_weather"] + + def test_claude_row_refuses_a_user_tool_named_like_the_response_format_tool(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + tool: Final = {"type": "function", "function": {"name": "json_tool_call", "parameters": {"type": "object"}}} + entry: Final = _claude_chat_entry("r1", "hi", max_tokens=10, tools=[tool]) + + with pytest.raises(VertexAIError, match=r"line 1 .*json_tool_call") as exc_info: + _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert exc_info.value.status_code == 400 + + def test_claude_row_leaves_out_stream(self, config): + entry: Final = _claude_chat_entry( + "r1", "hi", max_tokens=10, stream=True, stream_options={"include_usage": True} + ) + + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [entry]) + + assert "stream" not in row["request"] + assert "stream_options" not in row["request"] + + def test_claude_row_merges_extra_body_into_the_request(self, config): + (row,) = _uploaded_rows( + config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10, extra_body={"top_k": 3})] + ) + + assert row["request"]["top_k"] == 3 + assert "extra_body" not in row["request"] + + def test_claude_response_format_row_round_trips_as_json_content(self, config): + schema: Final = {"type": "object", "properties": {"color": {"type": "string"}}, "required": ["color"]} + (row,) = _uploaded_rows( + config, + CLAUDE_DEPLOYMENT, + [ + _claude_chat_entry( + "r1", + "sky color?", + max_tokens=50, + response_format={"type": "json_schema", "json_schema": {"name": "sky", "schema": schema}}, + ) + ], + ) + assert "json_mode" not in row["request"] + (json_tool,) = row["request"]["tools"] + output_row: Final = _claude_output_row( + "r1", + "unused", + content=[{"type": "tool_use", "id": "toolu_1", "name": json_tool["name"], "input": {"color": "blue"}}], + ) + + (out,) = ( + json.loads(line) + for line in config._try_transform_vertex_batch_output_to_openai( + json.dumps(output_row).encode() + ).splitlines() + ) + + message: Final = out["response"]["body"]["choices"][0]["message"] + assert json.loads(message["content"]) == {"color": "blue"} + assert not message.get("tool_calls") + + def test_claude_output_keeps_a_user_tool_call_as_a_tool_call(self, config): + output_row: Final = _claude_output_row( + "r1", + "unused", + content=[{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}], + ) + + (out,) = ( + json.loads(line) + for line in config._try_transform_vertex_batch_output_to_openai( + json.dumps(output_row).encode() + ).splitlines() + ) + + (tool_call,) = out["response"]["body"]["choices"][0]["message"]["tool_calls"] + assert tool_call["function"]["name"] == "get_weather" + assert json.loads(tool_call["function"]["arguments"]) == {"city": "Paris"} + + def test_claude_input_file_read_back_is_returned_unchanged(self, config): + (row,) = _uploaded_rows(config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10)]) + content: Final = json.dumps(row).encode() + + assert config._try_transform_vertex_batch_output_to_openai(content) == content + + def test_malformed_claude_message_fails_only_its_own_row(self, config): + malformed: Final = _claude_output_row("bad", "unused", content=5) + content: Final = json.dumps(_claude_output_row("ok", "fine")).encode() + b"\n" + json.dumps(malformed).encode() + + ok_row, bad_row = ( + json.loads(line) for line in config._try_transform_vertex_batch_output_to_openai(content).splitlines() + ) + + assert ok_row["response"]["body"]["choices"][0]["message"]["content"] == "fine" + assert bad_row["custom_id"] == "bad" + assert bad_row["response"] is None + assert bad_row["error"]["code"] == "transformation_error" + + def test_claude_line_without_custom_id_is_a_400_at_upload(self, config): + from litellm.llms.vertex_ai.common_utils import VertexAIError + + with pytest.raises(VertexAIError, match=r"line 2 .*custom_id") as exc_info: + _uploaded_rows( + config, CLAUDE_DEPLOYMENT, [_claude_chat_entry("r1", "hi", max_tokens=10), _claude_chat_entry(None, "hi")] + ) + + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_streamed_claude_output_is_transformed(self, config): + async def chunks(): + yield json.dumps(_claude_output_row("a", "first")).encode() + b"\n" + + result: Final = await config.transform_file_content_stream( + stream_iterator=chunks(), + headers={}, + request_url="https://storage.googleapis.com/storage/v1/b/my-bucket/o/x?alt=media", + logging_obj=MagicMock(), + litellm_params={}, + ) + body: Final = b"".join([chunk async for chunk in result.stream_iterator]) + + (row,) = (json.loads(line) for line in body.splitlines()) + assert row["custom_id"] == "a" + assert row["response"]["body"]["choices"][0]["message"]["content"] == "first" From 7183904ce335a6db08a8734a3665c0417849c9f8 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Fri, 6 Nov 2026 04:32:37 -0800 Subject: [PATCH 02/28] fix(bedrock): drop another model's signed thinking when replaying to a non-Claude Converse target (#50073) Backport of #50073 to lts/1.108.x. Cherry-picked from 2a4f8b1d1918c074e0752fe50464514c1626ebd8 (main). The line's Converse message builders predate _thinking_in_replay_order and the drop_params/model_info arguments, so drop_signed is passed at the line's own thinking call sites and keeps_cache_point takes only drop_signed. --- .../prompt_templates/factory.py | 56 ++++- litellm/llms/bedrock/common_utils.py | 18 +- .../llms/openai/responses/transformation.py | 4 +- ...bedrock_converse_redacted_thinking_wire.py | 71 ++++++ ..._responses_minted_reasoning_replay_wire.py | 22 +- ...llm_core_utils_prompt_templates_factory.py | 228 +++++++++++++++++- .../test_converse_transformation_nova_2.py | 20 +- .../test_openai_responses_transformation.py | 12 +- 8 files changed, 392 insertions(+), 39 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 2613e0bbd28..60bc9cadee4 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4607,6 +4607,16 @@ def get_assistant_message_block_or_continue_message( raise ValueError(f"Unsupported content type: {type(content_block)}") +def _as_thinking_block(item: object) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None: + match item: + case {"type": "thinking" | "redacted_thinking"}: + return cast( # cast-ok: the mapping pattern matched a thinking block's type tag + ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, item + ) + case _: + return None + + class BedrockConverseMessagesProcessor: @staticmethod def _initial_message_setup( @@ -4655,11 +4665,15 @@ class BedrockConverseMessagesProcessor: deployment_model: str | None = None, base_model: str | None = None, ) -> list[BedrockMessageBlock]: - from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking + from litellm.llms.bedrock.common_utils import ( + bedrock_converse_accepts_redacted_thinking, + bedrock_converse_rejects_signed_thinking, + ) contents: list[BedrockMessageBlock] = [] msg_i = 0 to_claude: Final = bedrock_converse_accepts_redacted_thinking(model, base_model) + drop_signed_thinking: Final = bedrock_converse_rejects_signed_thinking(model, base_model) messages = BedrockConverseMessagesProcessor._initial_message_setup( messages, model, llm_provider, user_continue_message @@ -4811,7 +4825,7 @@ class BedrockConverseMessagesProcessor: if thinking_blocks is not None: converted_thinking_blocks = ( BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks, to_claude=to_claude + thinking_blocks, to_claude=to_claude, drop_signed=drop_signed_thinking ) ) assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( @@ -4825,7 +4839,7 @@ class BedrockConverseMessagesProcessor: if isinstance(element, dict): if element["type"] == "thinking" or element["type"] == "redacted_thinking": thinking_block = BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks=[element], to_claude=to_claude + thinking_blocks=[element], to_claude=to_claude, drop_signed=drop_signed_thinking ) assistants_parts = ( BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( @@ -4852,7 +4866,9 @@ class BedrockConverseMessagesProcessor: model=model, deployment_model=deployment_model, ) - if _cache_point_block is not None: + if _cache_point_block is not None and BedrockConverseMessagesProcessor.keeps_cache_point( + element, drop_signed=drop_signed_thinking + ): assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance(_assistant_content, str): @@ -4900,6 +4916,7 @@ class BedrockConverseMessagesProcessor: thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], *, to_claude: bool = False, + drop_signed: bool = False, ) -> list[BedrockContentBlock]: return [ BedrockContentBlock( @@ -4909,8 +4926,25 @@ class BedrockConverseMessagesProcessor: ) for thinking_block in thinking_blocks if not (to_claude and is_encrypted_reasoning_block(thinking_block)) + and not (drop_signed and BedrockConverseMessagesProcessor._is_signed(thinking_block)) ] + @staticmethod + def keeps_cache_point(element: object, *, drop_signed: bool) -> bool: + match _as_thinking_block(element): + case None: + return True + case thinking_block: + return not (drop_signed and BedrockConverseMessagesProcessor._is_signed(thinking_block)) + + @staticmethod + def _is_signed(thinking_block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock) -> bool: + match thinking_block: + case {"signature": signature} if signature: + return True + case _: + return False + @staticmethod def _to_reasoning_content_block( thinking_block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, @@ -5067,11 +5101,15 @@ def _bedrock_converse_messages_pt( - Please ensure that function response turn comes immediately after a function call turn - Conversation blocks and tool result blocks cannot be provided in the same turn. Issue: https://github.com/BerriAI/litellm/issues/6053 """ - from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking + from litellm.llms.bedrock.common_utils import ( + bedrock_converse_accepts_redacted_thinking, + bedrock_converse_rejects_signed_thinking, + ) contents: list[BedrockMessageBlock] = [] msg_i = 0 to_claude: Final = bedrock_converse_accepts_redacted_thinking(model, base_model) + drop_signed_thinking: Final = bedrock_converse_rejects_signed_thinking(model, base_model) messages = BedrockConverseMessagesProcessor._initial_message_setup( messages, model, llm_provider, user_continue_message @@ -5224,7 +5262,7 @@ def _bedrock_converse_messages_pt( if thinking_blocks is not None: converted_thinking_blocks = ( BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks, to_claude=to_claude + thinking_blocks, to_claude=to_claude, drop_signed=drop_signed_thinking ) ) assistant_content = BedrockConverseMessagesProcessor.add_thinking_blocks_to_assistant_content( @@ -5239,7 +5277,7 @@ def _bedrock_converse_messages_pt( if element["type"] == "thinking" or element["type"] == "redacted_thinking": thinking_block = ( BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( - thinking_blocks=[element], to_claude=to_claude + thinking_blocks=[element], to_claude=to_claude, drop_signed=drop_signed_thinking ) ) assistants_parts = ( @@ -5268,7 +5306,9 @@ def _bedrock_converse_messages_pt( model=model, deployment_model=deployment_model, ) - if _cache_point_block is not None: + if _cache_point_block is not None and BedrockConverseMessagesProcessor.keeps_cache_point( + element, drop_signed=drop_signed_thinking + ): assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance(_assistant_content, str): diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 999dd46b01a..44e33f1c43b 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1183,15 +1183,29 @@ def bedrock_converse_accepts_redacted_thinking(model: str, base_model: str | Non its own. An ARN that hides its model, like an application inference profile, falls back to the deployment's ``base_model`` and counts as non-Claude without one. """ + return _is_claude(_converse_target_model(model, base_model)) + + +def bedrock_converse_rejects_signed_thinking(model: str, base_model: str | None) -> bool: + if base_model is None and "." not in get_bedrock_base_model(model): + return False + return not _is_claude_behind_any_region_prefix(_converse_target_model(model, base_model)) + + +def _converse_target_model(model: str, base_model: str | None) -> str: if base_model is None or "." in get_bedrock_base_model(model): - return _is_claude(model) - return _is_claude(base_model) + return model + return base_model def _is_claude(model: str) -> bool: return get_bedrock_base_model(model).startswith(("anthropic", "claude")) +def _is_claude_behind_any_region_prefix(model: str) -> bool: + return _is_claude(model) or get_bedrock_base_model(model).partition(".")[2].startswith("anthropic.") + + def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: return any( (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 464dc799c16..811b5df9933 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -63,7 +63,9 @@ _NO_TOOL_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) _MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("gpt-4", "gpt-3.5", "chatgpt-4o", "o1", "o3", "o4") _PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) -_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS +_PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = frozenset( + {LlmProviders.AZURE, LlmProviders.BEDROCK, LlmProviders.BEDROCK_MANTLE, LlmProviders.OPENAI} +) _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) diff --git a/tests/integration/providers/test_bedrock_converse_redacted_thinking_wire.py b/tests/integration/providers/test_bedrock_converse_redacted_thinking_wire.py index 79ec0545c44..1da52b0b6dc 100644 --- a/tests/integration/providers/test_bedrock_converse_redacted_thinking_wire.py +++ b/tests/integration/providers/test_bedrock_converse_redacted_thinking_wire.py @@ -414,3 +414,74 @@ def test_converse_redacted_thinking_replays_to_the_anthropic_api_as_claude_nativ {"type": "redacted_thinking", "data": NATIVE_REDACTED}, {"type": "text", "text": FIRST_ANSWER}, ) + + +_SIGNED_THINKING: Final[dict[str, JsonValue]] = { + "type": "thinking", + "thinking": "Checking the forecast.", + "signature": "scripted-signature-" + "s" * 32, +} +_CONVERSE_SIGNED_PART: Final[dict[str, JsonValue]] = { + "reasoningContent": { + "reasoningText": {"text": _SIGNED_THINKING["thinking"], "signature": _SIGNED_THINKING["signature"]} + } +} + + +@pytest.mark.parametrize( + ("deployment", "replayed_thinking"), + [ + pytest.param({"model": f"bedrock/converse/{NOVA_MODEL_ID}"}, (), id="nova"), + pytest.param( + { + "model": f"bedrock/converse/{APPLICATION_PROFILE_ARN}", + "model_info": {"base_model": f"bedrock/{NOVA_MODEL_ID}"}, + }, + (), + id="application-profile-arn-with-nova-base-model", + ), + pytest.param( + {"model": f"bedrock/converse/{APPLICATION_PROFILE_ARN}"}, + (_CONVERSE_SIGNED_PART,), + id="application-profile-arn-without-base-model", + ), + pytest.param( + { + "model": f"bedrock/converse/{APPLICATION_PROFILE_ARN}", + "model_info": {"base_model": f"bedrock/{CLAUDE_MODEL_ID}"}, + }, + (_CONVERSE_SIGNED_PART,), + id="application-profile-arn-with-claude-base-model", + ), + pytest.param(_CLAUDE, (_CONVERSE_SIGNED_PART,), id="claude"), + ], +) +def test_signed_history_keeps_its_thinking_unless_the_converse_target_is_known_not_to_be_claude( + gateway: Gateway, deployment: dict[str, JsonValue], replayed_thinking: tuple[dict[str, JsonValue], ...] +) -> None: + model_info: Final = deployment.get("model_info") + parameters: Final = {key: value for key, value in deployment.items() if key != "model_info"} + with wire_server(_bedrock_peer()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model_info=model_info if isinstance(model_info, dict) else None, + api_key=TOKEN, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + **parameters, + ) + + reply: Final = _chat( + gateway, + { + "model": model, + "max_tokens": 256, + "messages": [ + {"role": "user", "content": QUESTION}, + {"role": "assistant", "content": FIRST_ANSWER, "thinking_blocks": [_SIGNED_THINKING]}, + {"role": "user", "content": FOLLOW_UP}, + ], + }, + ) + + assert reply.content == FINAL_ANSWER + assert _replayed_assistant_turn(wire).content == (*replayed_thinking, {"text": FIRST_ANSWER}) diff --git a/tests/integration/providers/test_responses_minted_reasoning_replay_wire.py b/tests/integration/providers/test_responses_minted_reasoning_replay_wire.py index 2253443dfc0..4ff5871ff97 100644 --- a/tests/integration/providers/test_responses_minted_reasoning_replay_wire.py +++ b/tests/integration/providers/test_responses_minted_reasoning_replay_wire.py @@ -77,7 +77,18 @@ _AZURE_AI_OPENAI_HOST: Final = _Deployment( "/openai/v1/responses?api-version=preview", strips_message_status=True, ) -_DROPPING: Final = (_OPENAI, _AZURE, _AZURE_AI_OPENAI_HOST) +_BEDROCK_MANTLE: Final = _Deployment( + "bedrock_mantle", "bedrock_mantle/openai.gpt-oss-120b", "synthetic-mantle-key", "/v1/responses" +) +_BEDROCK: Final = _Deployment( + "bedrock", + "bedrock/openai.gpt-oss-120b-1:0", + "synthetic-bedrock-key", + "/openai/v1/responses", + MappingProxyType({"aws_region_name": "us-east-1"}), + model_info=MappingProxyType({"supported_endpoints": ["/v1/responses"]}), +) +_DROPPING: Final = (_OPENAI, _AZURE, _AZURE_AI_OPENAI_HOST, _BEDROCK_MANTLE, _BEDROCK) _KEEPING: Final = ( _Deployment("litellm_proxy", f"litellm_proxy/{_GPT}", "synthetic-proxy-key", "/responses"), _Deployment("databricks", "databricks/gpt-5.6", "synthetic-databricks-key", "/responses"), @@ -95,15 +106,6 @@ _KEEPING: Final = ( "/v1/responses", types_untyped_items_as_messages=True, ), - _Deployment("bedrock_mantle", "bedrock_mantle/openai.gpt-oss-120b", "synthetic-mantle-key", "/v1/responses"), - _Deployment( - "bedrock", - "bedrock/openai.gpt-oss-120b-1:0", - "synthetic-bedrock-key", - "/openai/v1/responses", - MappingProxyType({"aws_region_name": "us-east-1"}), - model_info=MappingProxyType({"supported_endpoints": ["/v1/responses"]}), - ), *( _Deployment(slug, f"{slug}/{model}", f"synthetic-{slug}-key", "/responses") for slug, model in ( diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index ad216548d08..f0b7f9995c4 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -68,7 +68,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.llms.vertex_ai.gemini.transformation import gemini_convert_messages_with_history from litellm.types.llms.bedrock import BedrockConverseReasoningContentBlock -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage READ_FILE: Final[list[object]] = [{"name": "read_file", "description": "Read the contents of a file."}] @@ -422,6 +422,46 @@ def test_anthropic_messages_pt_drops_the_stream_index_from_replayed_thinking_blo ] +@pytest.mark.parametrize( + ("model", "expected_content"), + ( + ( + "global.anthropic.claude-haiku-5-5", + [ + {"reasoningContent": {"reasoningText": {"text": "pondering", "signature": "signed-reasoning"}}}, + {"text": "hello"}, + ], + ), + ("global.zai.glm-5.3", [{"text": "hello"}]), + ("amazon.nova-pro-v1:0", [{"text": "hello"}]), + ("converse/us.openai.gpt-6-luna", [{"text": "hello"}]), + ), +) +def test_bedrock_converse_replays_signed_thinking_only_to_claude( + model: str, expected_content: Sequence[Mapping[str, object]] +) -> None: + body: Final = litellm.AmazonConverseConfig().transform_request( + model=model, + messages=[ + ChatCompletionUserMessage(role="user", content="hi"), + ChatCompletionAssistantMessage( + role="assistant", + content="hello", + thinking_blocks=[{"type": "thinking", "thinking": "pondering", "signature": "signed-reasoning"}], + ), + ChatCompletionUserMessage(role="user", content="continue"), + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert body["messages"] == [ + {"role": "user", "content": [{"text": "hi"}]}, + {"role": "assistant", "content": expected_content}, + {"role": "user", "content": [{"text": "continue"}]}, + ] + + async def test_anthropic_bedrock_thinking_blocks_with_none_content(): """ Test the specific function that processes thinking_blocks when content is None @@ -695,6 +735,192 @@ async def test_bedrock_converse_keeps_dropping_redacted_thinking_for_non_claude_ assert assistant_content == [{"text": "hello"}] +_HIDDEN_MODEL_ARN: Final = "arn:aws:bedrock:us-west-2:123456789012:application-inference-profile/abc123" + + +async def _converse_messages( + messages: list[dict[str, object]], + model: str, + base_model: str | None, + use_async: bool, + assistant_continue_message: str | None = None, +) -> list[dict[str, object]]: + return ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model=model, + llm_provider="bedrock", + base_model=base_model, + assistant_continue_message=assistant_continue_message, + ) + if use_async + else _bedrock_converse_messages_pt( + messages=messages, + model=model, + llm_provider="bedrock", + base_model=base_model, + assistant_continue_message=assistant_continue_message, + ) + ) + + +def _assistant_with_thinking(placement: str, block: dict[str, object]) -> dict[str, object]: + return ( + {"role": "assistant", "content": "hello", "thinking_blocks": [block]} + if placement == "thinking_blocks" + else {"role": "assistant", "content": [block, {"type": "text", "text": "hello"}]} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("placement", ["thinking_blocks", "content"]) +@pytest.mark.parametrize( + "signature", ["claude-signature", encrypted_reasoning_signature("gAAAA_1")], ids=["claude", "encrypted"] +) +@pytest.mark.parametrize( + ("model", "base_model"), + [ + ("converse/us.openai.gpt-6-luna", None), + ("us.amazon.nova-pro-v1:0", None), + ("us.amazon.nova-pro-v1:0", "anthropic.claude-sonnet-4-6"), + (_HIDDEN_MODEL_ARN, "openai.gpt-oss-120b-1:0"), + ], +) +async def test_bedrock_converse_drops_signed_thinking_for_a_known_non_claude_target( + use_async: bool, placement: str, signature: str, model: str, base_model: str | None +): + block: Final = {"type": "thinking", "thinking": "private plan", "signature": signature} + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_thinking(placement, block), + {"role": "user", "content": "again"}, + ] + + assert await _converse_messages(messages, model, base_model, use_async) == [ + {"role": "user", "content": [{"text": "hi"}]}, + {"role": "assistant", "content": [{"text": "hello"}]}, + {"role": "user", "content": [{"text": "again"}]}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("placement", ["thinking_blocks", "content"]) +@pytest.mark.parametrize( + ("model", "base_model"), + [ + ("us.anthropic.claude-sonnet-4-6", None), + ("ca.anthropic.claude-haiku-5-5", None), + ("converse/in.anthropic.claude-sonnet-5", None), + ("claude-sonnet-4-6", None), + (_HIDDEN_MODEL_ARN, None), + (_HIDDEN_MODEL_ARN, "anthropic.claude-sonnet-4-6"), + ], +) +async def test_bedrock_converse_keeps_signed_thinking_for_a_target_that_may_be_claude( + use_async: bool, placement: str, model: str, base_model: str | None +): + block: Final = {"type": "thinking", "thinking": "private plan", "signature": "claude-signature"} + messages: Final = [ + {"role": "user", "content": "hi"}, + _assistant_with_thinking(placement, block), + {"role": "user", "content": "again"}, + ] + + result: Final = await _converse_messages(messages, model, base_model, use_async) + + assistant_content: Final = next(m["content"] for m in result if m["role"] == "assistant") + assert assistant_content == [ + {"reasoningContent": {"reasoningText": {"text": "private plan", "signature": "claude-signature"}}}, + {"text": "hello"}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_bedrock_converse_keeps_the_tool_call_when_dropping_signed_thinking(use_async: bool): + messages: Final = [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": None, + "thinking_blocks": [{"type": "thinking", "thinking": "private plan", "signature": "claude-signature"}], + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ] + + result: Final = await _converse_messages(messages, "converse/us.openai.gpt-6-luna", None, use_async) + + assert [m["role"] for m in result] == ["user", "assistant", "user"] + assert [list(part) for part in result[1]["content"]] == [["toolUse"]] + assert result[1]["content"][0]["toolUse"]["toolUseId"] == "call_1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + ("assistant_continue_message", "expected"), + [ + (None, [{"role": "user", "content": [{"text": "hi"}, {"text": "again"}]}]), + ( + "Please continue.", + [ + {"role": "user", "content": [{"text": "hi"}]}, + {"role": "assistant", "content": [{"text": "Please continue."}]}, + {"role": "user", "content": [{"text": "again"}]}, + ], + ), + ], + ids=["merged_into_one_user_turn", "split_by_the_continue_message"], +) +async def test_bedrock_converse_drops_a_turn_that_held_only_signed_thinking( + use_async: bool, assistant_continue_message: str | None, expected: list[dict[str, object]] +): + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "thinking_blocks": [{"type": "thinking", "thinking": "private plan", "signature": "claude-signature"}], + }, + {"role": "user", "content": "again"}, + ] + + result: Final = await _converse_messages( + messages, "converse/us.openai.gpt-6-luna", None, use_async, assistant_continue_message + ) + + assert result == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_bedrock_converse_drops_the_cache_point_of_a_dropped_signed_thinking_block(use_async: bool): + messages: Final = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "private plan", + "signature": "claude-signature", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": "again"}, + ] + + result: Final = await _converse_messages(messages, "converse/us.amazon.nova-2-lite-v1:0", None, use_async) + + assert result == [{"role": "user", "content": [{"text": "hi"}, {"text": "again"}]}] + + @pytest.mark.parametrize( "thinking_block", [ diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation_nova_2.py b/tests/unit/llms/bedrock/chat/test_converse_transformation_nova_2.py index 58058a2e1d4..f29ad3b5844 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation_nova_2.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation_nova_2.py @@ -637,8 +637,7 @@ class TestNova2MultiTurnMessageTranslation: assert "Simple addition" in texts assert "4." in texts - def test_should_keep_signed_thinking_blocks_as_reasoning_content(self): - """With a signature, reasoning is preserved as a reasoningContent block.""" + def test_should_drop_signed_thinking_blocks_from_another_model(self): bedrock_msgs = self._to_bedrock( [ {"role": "user", "content": "What is 2+2?"}, @@ -653,13 +652,9 @@ class TestNova2MultiTurnMessageTranslation: ] ) assistant = next(m for m in bedrock_msgs if m["role"] == "assistant") - rc_blocks = [b for b in assistant["content"] if "reasoningContent" in b] - assert len(rc_blocks) >= 1 - assert rc_blocks[0]["reasoningContent"]["reasoningText"]["text"] == "math" - assert rc_blocks[0]["reasoningContent"]["reasoningText"]["signature"] == "sig-1" + assert assistant["content"] == [{"text": "4."}] - def test_should_translate_inline_content_list_thinking_type(self): - """content=[{type:'thinking',...},{type:'text',...}] should also round-trip.""" + def test_should_drop_inline_signed_thinking_and_keep_the_answer(self): bedrock_msgs = self._to_bedrock( [ {"role": "user", "content": "Hi"}, @@ -674,11 +669,4 @@ class TestNova2MultiTurnMessageTranslation: ] ) assistant = next(m for m in bedrock_msgs if m["role"] == "assistant") - rc_blocks = [b for b in assistant["content"] if "reasoningContent" in b] - text_blocks = [ - b - for b in assistant["content"] - if "text" in b and "reasoningContent" not in b - ] - assert len(rc_blocks) >= 1 - assert any("Hello!" in b["text"] for b in text_blocks) + assert assistant["content"] == [{"text": "Hello!"}] diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 2fcb70ec40b..0a0f0d33543 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -13,6 +13,8 @@ import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig +from litellm.llms.bedrock_mantle.responses.transformation import BedrockMantleResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.types.llms.openai import ( @@ -1298,7 +1300,15 @@ class TestOpenAIResponsesAPIConfig: ) return list(bridged.output) - @pytest.mark.parametrize("config", [OpenAIResponsesAPIConfig(), AzureOpenAIResponsesAPIConfig()]) + @pytest.mark.parametrize( + "config", + [ + OpenAIResponsesAPIConfig(), + AzureOpenAIResponsesAPIConfig(), + BedrockOpenAIResponsesConfig(), + BedrockMantleResponsesAPIConfig(), + ], + ) def test_claude_reasoning_minted_by_the_bridge_is_dropped_before_the_history_reaches_openai(self, config): saved_claude_turn = json.loads( json.dumps([item.model_dump() for item in self._claude_turn_bridged_to_responses_output()]) From 1e513a4314039f6654c0defbb624b8459b7db772 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:56:05 -0800 Subject: [PATCH 03/28] fix(chatgpt): round-trip encrypted reasoning through /v1/messages The chat bridge stored encrypted reasoning on reasoning_items, the Anthropic adapter never read that field, stream chunks that only carried thinking blocks were dropped, and the messages adapter stripped the signed blocks before a chatgpt request could turn them back into reasoning items. Backport of #49076 to lts/1.108.x. Cherry-picked from ea9e29b1a89 (cursor/chatgpt-encrypted-reasoning-messages-1723). The line keeps the reasoning helpers in common_utils and the adapters under pass_through, gains resolve_model_and_provider_for_metadata, and streams redacted_thinking as its own block. Co-authored-by: DDT <141296842+Hugo-DDT@users.noreply.github.com> Co-authored-by: DDT <1786035110@qq.com> --- .../transformation.py | 73 +++++++- .../get_llm_provider_logic.py | 8 + .../prompt_templates/common_utils.py | 82 +++++++++ .../litellm_core_utils/streaming_handler.py | 1 + .../adapters/streaming_iterator.py | 10 +- .../pass_through/adapters/transformation.py | 76 ++++++--- litellm/responses/bridge_check.py | 25 ++- ...responses_transformation_transformation.py | 159 +++++++++++++++++ .../test_streaming_handler.py | 31 ++++ ...al_pass_through_adapters_transformation.py | 160 ++++++++++++++++++ 10 files changed, 591 insertions(+), 34 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 822ef1ea0be..7fe36c2e42a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,9 +26,11 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, + thinking_blocks_from_reasoning_items, with_prompt_cache_breakpoint, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -45,6 +47,8 @@ from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_response from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionReasoningItem, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -54,7 +58,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import GenericStreamingChunk, ModelResponseStream +from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam, ResponseOutputItem @@ -70,8 +74,6 @@ if TYPE_CHECKING: AllMessageValues, ChatCompletionFileObject, ChatCompletionImageObject, - ChatCompletionRedactedThinkingBlock, - ChatCompletionThinkingBlock, ChatCompletionToolReferenceObject, OpenAIMessageContentListBlock, ) @@ -443,6 +445,63 @@ def _as_chat_reasoning_items( return cast(list[ChatCompletionReasoningItem], list(reasoning_items)) +def _with_stream_block_index( + block: Mapping[str, object], + index: int, +) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock: + if block.get("type") == "redacted_thinking": + data: Final = block.get("data") + return ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=data if isinstance(data, str) else "", + index=index, + ) + thinking: Final = block.get("thinking") + signature: Final = block.get("signature") + return ChatCompletionThinkingBlock( + type="thinking", + thinking=thinking if isinstance(thinking, str) else "", + signature=signature if isinstance(signature, str) else None, + index=index, + ) + + +def _reasoning_done_thinking_blocks( + item: Mapping[str, object], + output_index: object, +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + index: Final = output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else 0 + return tuple(_with_stream_block_index(block, index) for block in thinking_blocks_from_reasoning_items((item,))) + + +def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseStream | None: + event_type: Final = chunk.get("type") + event_name: Final = ( + event_type.value + if isinstance(event_type, ResponsesAPIStreamEvents) + else event_type + if isinstance(event_type, str) + else None + ) + if event_name != ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + return None + item: Final = as_str_mapping(chunk.get("item")) + if item is None or item.get("type") != "reasoning": + return None + signed_blocks: Final = _reasoning_done_thinking_blocks(item, chunk.get("output_index")) + if not signed_blocks: + return None + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(thinking_blocks=list(signed_blocks)), + finish_reason=None, + ) + ] + ) + + def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation: shifted: Final = { key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value @@ -497,6 +556,7 @@ def _fold_choices_into_one( *chain.from_iterable(_message_reasoning_items(m) for m in messages), *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), ) + encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items) return Choices( message=Message( role=messages[0].role if messages else "assistant", @@ -505,6 +565,7 @@ def _fold_choices_into_one( tool_calls=list(tool_calls) or None, reasoning_content=" ".join(value for value in reasoning_contents if value) or None, reasoning_items=_as_chat_reasoning_items(reasoning_items), + thinking_blocks=list(encrypted_thinking_blocks) or None, provider_specific_fields=_tool_search_provider_fields(tool_search_items), ), finish_reason="tool_calls" if tool_calls else "stop", @@ -1310,10 +1371,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for summary_block in reasoning_item["summary"] if summary_block.get("text") ) + encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items) message: Final = Message( content=None if tool_search_items else "", reasoning_content=reasoning_content if reasoning_content else None, reasoning_items=_as_chat_reasoning_items(reasoning_items), + thinking_blocks=list(encrypted_thinking_blocks) or None, provider_specific_fields=_tool_search_provider_fields(tool_search_items), ) return Choices(message=message, finish_reason=finish_reason, index=0) @@ -2089,7 +2152,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - elif event_type == "response.output_text.delta": # Content part added to output content_part = parsed_chunk.get("delta", None) @@ -2193,9 +2255,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) self._remember_served_service_tier(chunk) + reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk) translated: Final = ( self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) + else reasoning_chunk + if reasoning_chunk is not None else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( chunk, tool_call_index_map=self._tool_call_index_map ) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a67c1bb7df8..a9be736360b 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -158,6 +158,14 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None +def resolve_model_and_provider_for_metadata(model: str, custom_llm_provider: str | None = None) -> tuple[str, str]: + declared: Final = declared_authenticating_provider(model, custom_llm_provider) + if declared is not None: + return model.removeprefix(f"{declared}/"), declared + resolved_model, resolved_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + return resolved_model, resolved_provider + + def inferred_provider(model: str | None) -> str | None: if not model: return None diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index ac5791957b9..790860ca561 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -20,6 +20,7 @@ from pydantic import TypeAdapter import litellm from litellm import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( AllMessageValues, @@ -2289,6 +2290,87 @@ def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: return encrypted_content_from_signature(_encrypted_reasoning_field(block)) +_REASONING_SUMMARY_SEPARATOR: Final = "\n\n" + + +def _object_sequence(value: object) -> tuple[object, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return () + return tuple(value) + + +def _iter_mappings(value: object) -> tuple[Mapping[str, object], ...]: + return tuple(mapping for item in _object_sequence(value) if (mapping := as_str_mapping(item)) is not None) + + +def _reasoning_summary_part_text(part: object) -> str: + mapping: Final = as_str_mapping(part) + if mapping is not None: + text: Final = mapping.get("text") + return text if isinstance(text, str) else "" + text_attr: Final = getattr(part, "text", None) + return text_attr if isinstance(text_attr, str) else "" + + +def _reasoning_item_summary_text(item: Mapping[str, object]) -> str: + parts: Final = tuple(_reasoning_summary_part_text(part) for part in _object_sequence(item.get("summary"))) + return _REASONING_SUMMARY_SEPARATOR.join(part for part in parts if part) + + +def thinking_block_from_reasoning_item( + item: object, +) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None: + """The chat thinking block that carries one Responses `reasoning` item back to an Anthropic client. + + `encrypted_content` rides the same `litellm_encrypted_reasoning:` signature the Responses + adapter uses. A summary becomes thinking text. Bytes with no summary become `redacted_thinking`, + because an empty thinking string is not something the client can replay. + """ + mapping: Final = as_str_mapping(item) + if mapping is None or mapping.get("type") not in (None, "reasoning"): + return None + encrypted_content: Final = mapping.get("encrypted_content") + if not isinstance(encrypted_content, str) or not encrypted_content: + return None + signature: Final = encrypted_reasoning_signature(encrypted_content) + text: Final = _reasoning_item_summary_text(mapping) + if not text: + return ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data=signature) + return ChatCompletionThinkingBlock(type="thinking", thinking=text, signature=signature) + + +def thinking_blocks_from_reasoning_items( + items: object, +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + return tuple( + block for item in _object_sequence(items) if (block := thinking_block_from_reasoning_item(item)) is not None + ) + + +def _block_encrypted_content(block: object) -> str | None: + mapping: Final = as_str_mapping(block) + if mapping is None: + return None + return encrypted_content_of_block(mapping) + + +def thinking_blocks_with_encrypted_reasoning( + thinking_blocks: object, + reasoning_items: object, +) -> tuple[Mapping[str, object], ...]: + """Thinking blocks already on the message, plus any encrypted reasoning item they do not already carry.""" + direct: Final = _iter_mappings(thinking_blocks) + represented: Final = frozenset( + content for block in direct if (content := _block_encrypted_content(block)) is not None + ) + extra: Final = tuple( + block + for block in thinking_blocks_from_reasoning_items(_iter_mappings(reasoning_items)) + if _block_encrypted_content(block) not in represented + ) + return (*extra, *direct) + + def is_encrypted_reasoning_block(block: object) -> bool: """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 7135b9c1071..d91a67a2d13 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -912,6 +912,7 @@ class CustomStreamWrapper: and model_response.choices[0].delta.role is not None ) or (getattr(model_response.choices[0].delta, "reasoning_items", None) is not None) + or getattr(model_response.choices[0].delta, "thinking_blocks", None) ): return True else: diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index 18db0f28338..0f1398ac9b2 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -24,7 +24,9 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.litellm_core_utils.hidden_params import set_hidden_params +from litellm.litellm_core_utils.prompt_templates.common_utils import thinking_blocks_from_reasoning_items from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -66,6 +68,10 @@ def _optional_attr(obj: object, name: str) -> object: return getattr(obj, name, None) +def _delta_has_encrypted_reasoning(delta: object) -> bool: + return bool(thinking_blocks_from_reasoning_items(attribute_or_none(delta, "reasoning_items"))) + + def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]: value: Final = getattr(obj, name, None) return value if value else () @@ -458,7 +464,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): sent_first_chunk: bool = False sent_content_block_start: bool = False sent_content_block_finish: bool = False - current_content_block_type: Literal["text", "tool_use", "thinking"] = "text" + current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text" sent_last_message: bool = False holding_chunk: ContentBlockDelta | None = None holding_stop_reason_chunk: MessageBlockDelta | None = None @@ -1306,6 +1312,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks ): return False + if _delta_has_encrypted_reasoning(delta): + return False return True def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index d1560c5b2ed..62f33bb11fd 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -122,6 +122,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( anthropic_image_source_to_openai_url, parse_tool_call_arguments, reasoning_content_from_thinking_blocks, + thinking_blocks_with_encrypted_reasoning, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -219,6 +220,19 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider) +def _keeps_encrypted_reasoning_for_responses(model: str | None, custom_llm_provider: str | None) -> bool: + from litellm.responses.bridge_check import chat_completion_ends_in_responses_api + + return chat_completion_ends_in_responses_api(model, custom_llm_provider) + + +def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]: + return thinking_blocks_with_encrypted_reasoning( + getattr(source, "thinking_blocks", None), + getattr(source, "reasoning_items", None), + ) + + def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam: eager_input_streaming: Final = eager_input_streaming_flag(tool) if eager_input_streaming is None: @@ -467,7 +481,11 @@ class LiteLLMAnthropicMessagesAdapter: preserve_midturn_system: bool = False, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] - replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + replayable_messages: Final = ( + messages + if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider) + else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + ) leading_count: Final = next( (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)), len(replayable_messages), @@ -1358,9 +1376,9 @@ class LiteLLMAnthropicMessagesAdapter: for choice in choices ): new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks) - # Handle thinking blocks first - if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: - for thinking_block in choice.message.thinking_blocks: + signed_thinking_blocks = _anthropic_thinking_blocks(choice.message) + if signed_thinking_blocks: + for thinking_block in signed_thinking_blocks: if thinking_block.get("type") == "thinking": if is_empty_unsigned_thinking_block(thinking_block): continue @@ -1627,7 +1645,7 @@ class LiteLLMAnthropicMessagesAdapter: def translate_streaming_openai_chunk_to_anthropic_content_block( self, choices: list[OpenAIStreamingChoice | StreamingChoices] ) -> tuple[ - Literal["text", "tool_use", "thinking"], + Literal["text", "tool_use", "thinking", "redacted_thinking"], "ContentBlockContentBlockDict", ]: from litellm._uuid import uuid @@ -1660,20 +1678,24 @@ class LiteLLMAnthropicMessagesAdapter: choice.delta ) is not None: return "text", TextBlock(type="text", text="") - elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): - thinking_blocks = choice.delta.thinking_blocks or [] - if len(thinking_blocks) > 0: - thinking_block = thinking_blocks[0] - if thinking_block["type"] == "thinking": - thinking = thinking_block.get("thinking") or "" - signature = thinking_block.get("signature") or "" + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)): + thinking_block = thinking_blocks[0] + if thinking_block["type"] == "redacted_thinking": + redacted_data = thinking_block.get("data") + return "redacted_thinking", ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=redacted_data if isinstance(redacted_data, str) else "", + ) + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" - assert isinstance(thinking, str) - assert isinstance(signature, str) + assert isinstance(thinking, str) + assert isinstance(signature, str) - return "thinking", ChatCompletionThinkingBlock( - type="thinking", thinking=thinking, signature=signature - ) + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking=thinking, signature=signature + ) # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning # parsers) populate ``reasoning_content`` without ``thinking_blocks``. # ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the @@ -1706,19 +1728,17 @@ class LiteLLMAnthropicMessagesAdapter: for tool in choice.delta.tool_calls: if tool.function is not None and tool.function.arguments is not None: partial_json = (partial_json or "") + tool.function.arguments - elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): - thinking_blocks = choice.delta.thinking_blocks or [] - if len(thinking_blocks) > 0: - for thinking_block in thinking_blocks: - if thinking_block["type"] == "thinking": - thinking = thinking_block.get("thinking") or "" - signature = thinking_block.get("signature") or "" + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)): + for thinking_block in thinking_blocks: + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" - assert isinstance(thinking, str) - assert isinstance(signature, str) + assert isinstance(thinking, str) + assert isinstance(signature, str) - reasoning_content += thinking - reasoning_signature += signature + reasoning_content += thinking + reasoning_signature += signature # Handle reasoning_content when thinking_blocks is not present # This handles providers like OpenRouter that return reasoning_content elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"): diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index d5e3dfc0c02..7ed3ceda1eb 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -4,7 +4,10 @@ from urllib.parse import urlsplit import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + resolve_model_and_provider_for_metadata, +) from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, @@ -186,3 +189,23 @@ def responses_api_bridge_check( model = model.replace("responses/", "") return model_info, model + + +def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider: str | None) -> bool: + """Whether chat completion for this model is served by a Responses API. + + Encrypted reasoning can be replayed only there. A normal chat provider cannot + verify the block, so the messages adapter strips it. ``chatgpt`` and a + ``responses/`` model keep it. + """ + if model is None and custom_llm_provider is None: + return False + try: + resolved_model, provider = resolve_model_and_provider_for_metadata(model or "", custom_llm_provider) + except Exception as error: # noqa: BLE001 # unknown model names must not fail message translation + verbose_logger.debug("responses replay check failed for %s: %s", model, error) + return custom_llm_provider == LlmProviders.CHATGPT.value or ( + isinstance(model, str) and (model == "chatgpt" or model.startswith("chatgpt/")) + ) + model_info, _resolved = responses_api_bridge_check(model=resolved_model, custom_llm_provider=provider) + return model_info.get("mode") == "responses" diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index eb308a2c513..f24f163541f 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3240,6 +3240,165 @@ def test_reasoning_items_streaming_emitted_on_response_completed(): assert ri["summary"][0]["text"] == summary_text +def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keeps(): + """A chatgpt /v1/messages stream drops the reasoning item unless this delta carries thinking_blocks. + + response.output_item.done is the only event that has encrypted_content. The chat delta has to + expose it as a signed thinking block, and that block alone has to count as stream content. + """ + from unittest.mock import MagicMock + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + + encrypted: Final = "gAAAA_round_trip" + done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_done", + "encrypted_content": encrypted, + "summary": [], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ).chunk_parser(done) + blocks: Final = result.choices[0].delta.thinking_blocks + assert blocks is not None + assert blocks[0]["type"] == "redacted_thinking" + assert blocks[0]["data"] == encrypted_reasoning_signature(encrypted) + assert blocks[0]["index"] == 0 + assert getattr(result.choices[0].delta, "reasoning_content", None) in (None, "") + + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="chatgpt/gpt-6.1-sol", + logging_obj=MagicMock(), + custom_llm_provider="chatgpt", + ) + assert wrapper.is_chunk_non_empty({}, result, {}) is True + + block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block( + choices=result.choices + ) + assert block_type == "redacted_thinking" + assert start["data"] == encrypted_reasoning_signature(encrypted) + + +def test_reasoning_output_item_done_with_summary_signs_the_thinking_text(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + + encrypted: Final = "gAAAA_with_summary" + done: Final = { + "type": "response.output_item.done", + "output_index": 2, + "item": { + "type": "reasoning", + "id": "rs_sum", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ).chunk_parser(done) + block: Final = result.choices[0].delta.thinking_blocks[0] + assert block["type"] == "thinking" + assert block["thinking"] == "Oslo is colder." + assert block["signature"] == encrypted_reasoning_signature(encrypted) + assert block["index"] == 2 + + +def test_reasoning_output_item_done_without_encrypted_content_stays_empty(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_plain", + "summary": [{"type": "summary_text", "text": "already streamed"}], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ).chunk_parser(done) + assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, []) + + +def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks(): + from unittest.mock import Mock + + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse, Usage + + encrypted: Final = "gAAAA_non_stream" + reasoning: Final = ResponseReasoningItem( + id="rs_tool", + summary=[], + type="reasoning", + content=None, + encrypted_content=encrypted, + status=None, + ) + function_call: Final = ResponseFunctionToolCall( + id="fc_1", + type="function_call", + status="completed", + arguments='{"city":"Oslo"}', + call_id="call_1", + name="get_weather", + ) + raw: Final = ResponsesAPIResponse.model_construct( + id="resp_tool", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-6.1-sol", + output=[reasoning, function_call], + status="completed", + usage=None, + ) + model_response: Final = ModelResponse( + id="chatcmpl-tool", + created=1, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + result: Final = LiteLLMResponsesTransformationHandler().transform_response( + model="gpt-6.1-sol", + raw_response=raw, + model_response=model_response, + logging_obj=Mock(), + request_data={"model": "gpt-6.1-sol"}, + messages=[{"role": "user", "content": "Which city?"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + message: Final = result.choices[0].message + assert message.reasoning_items[0]["encrypted_content"] == encrypted + assert message.thinking_blocks[0]["type"] == "redacted_thinking" + assert message.thinking_blocks[0]["data"] == encrypted_reasoning_signature(encrypted) + assert message.tool_calls[0].function.name == "get_weather" + + def test_streaming_function_call_tool_id_for_degenerate_call_id(): """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` (``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 637811f6387..8dfce043fe1 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -246,6 +246,37 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp ) +def test_is_chunk_non_empty_keeps_thinking_blocks_without_other_content( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """Encrypted reasoning arrives as thinking_blocks and nothing else, so that delta has to be yielded.""" + chunk: Final = ModelResponseStream( + id="chunk-thinking", + object="chat.completion.chunk", + created=1, + model="chatgpt/gpt-6.1-sol", + choices=[ + StreamingChoices( + index=0, + finish_reason=None, + delta=Delta( + thinking_blocks=[ + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"} + ] + ), + ) + ], + ) + assert ( + initialized_custom_stream_wrapper.is_chunk_non_empty( + completion_obj={}, + model_response=chunk, + response_obj={}, + ) + is True + ) + + def test_is_chunk_non_empty_with_annotations( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2f0a7ad070b..49a8341ce9e 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -509,6 +509,166 @@ def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"] +def test_chatgpt_messages_keep_encrypted_reasoning_and_replay_it(): + """chatgpt /v1/messages ends in the Responses API, so the signed block has to survive into the next request.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + summary_encrypted: Final = "gAAAA_summary" + bytes_encrypted: Final = "gAAAA_bytes" + messages: Final = [ + AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + { + "type": "thinking", + "thinking": "Oslo is colder.", + "signature": encrypted_reasoning_signature(summary_encrypted), + }, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(bytes_encrypted)}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], + ), + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}], + ), + ] + + kept: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="chatgpt/gpt-6.1-sol", + custom_llm_provider="chatgpt", + ) + assistant: Final = kept[1] + assert [block["type"] for block in assistant["thinking_blocks"]] == ["thinking", "redacted_thinking"] + assert assistant["thinking_blocks"][0]["signature"] == encrypted_reasoning_signature(summary_encrypted) + assert assistant["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted) + + responses_input, _instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( + kept + ) + reasoning: Final = tuple(item for item in responses_input if item.get("type") == "reasoning") + assert [item["encrypted_content"] for item in reasoning] == [summary_encrypted, bytes_encrypted] + types: Final = [item.get("type") for item in responses_input] + assert types.index("reasoning") < types.index("function_call") + + stripped: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="gpt-4o", + custom_llm_provider="openai", + ) + assert not stripped[1].get("thinking_blocks") + + prefixed: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="responses/gpt-5-mini", + custom_llm_provider="openai", + ) + assert prefixed[1]["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted) + + +def test_translate_openai_reasoning_items_become_signed_thinking_blocks(): + encrypted: Final = "gAAAA_outbound" + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content="Oslo.", + reasoning_items=[ + { + "type": "reasoning", + "id": "rs_1", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + } + ], + tool_calls=[ + ChatCompletionAssistantToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Oslo"}'), + ) + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert result[0]["type"] == "thinking" + assert result[0]["thinking"] == "Oslo is colder." + assert result[0]["signature"] == encrypted_reasoning_signature(encrypted) + assert result[1]["type"] == "text" + assert result[2]["type"] == "tool_use" + + +def test_translate_openai_reasoning_items_without_summary_become_redacted_thinking(): + encrypted: Final = "gAAAA_bytes_only" + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content=None, + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert result == [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(encrypted)}, + ] + + +def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_blocks_already_carry_them(): + encrypted: Final = "gAAAA_once" + signature: Final = encrypted_reasoning_signature(encrypted) + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content="Oslo.", + thinking_blocks=[{"type": "redacted_thinking", "data": signature}], + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert [block["type"] for block in result] == ["redacted_thinking", "text"] + + +def test_streaming_chunk_with_only_reasoning_items_opens_a_signed_thinking_block(): + encrypted: Final = "gAAAA_delta" + choices: Final = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ] + ), + ) + ] + + block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block( + choices=choices + ) + + assert block_type == "redacted_thinking" + assert start["data"] == encrypted_reasoning_signature(encrypted) + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. From 6fef70ac726abf7efee91e6b72ad262a81fb7380 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:17:32 -0800 Subject: [PATCH 04/28] fix(anthropic): keep Responses reasoning on tool turns and retry once when a backend cannot decrypt it Backport of the second #49076 commit to lts/1.108.x. Cherry-picked from 10c953d8556. Adapted to the line: chat_completion_ends_in_responses_api has no deployment_model_info parameter here, so only tools, reasoning_effort, and api_base are threaded through, and the moved tests import the adapters from pass_through instead of messages. --- .../transformation.py | 35 ++- .../prompt_templates/common_utils.py | 8 + .../pass_through/adapters/handler.py | 97 +++++++-- .../adapters/streaming_iterator.py | 8 - .../pass_through/adapters/transformation.py | 69 ++++-- litellm/responses/bridge_check.py | 21 +- ...responses_transformation_transformation.py | 85 +++++++- ...al_pass_through_adapters_transformation.py | 205 ++++++++++++++++-- 8 files changed, 443 insertions(+), 85 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 7fe36c2e42a..296f1949d11 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -448,6 +448,7 @@ def _as_chat_reasoning_items( def _with_stream_block_index( block: Mapping[str, object], index: int, + summary_streamed: bool, ) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock: if block.get("type") == "redacted_thinking": data: Final = block.get("data") @@ -460,21 +461,33 @@ def _with_stream_block_index( signature: Final = block.get("signature") return ChatCompletionThinkingBlock( type="thinking", - thinking=thinking if isinstance(thinking, str) else "", + thinking=thinking if isinstance(thinking, str) and not summary_streamed else "", signature=signature if isinstance(signature, str) else None, index=index, ) +def _stream_output_index(chunk: Mapping[str, object]) -> int | None: + output_index: Final = chunk.get("output_index") + return output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else None + + def _reasoning_done_thinking_blocks( item: Mapping[str, object], - output_index: object, + output_index: int | None, + summary_streamed: bool, ) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: - index: Final = output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else 0 - return tuple(_with_stream_block_index(block, index) for block in thinking_blocks_from_reasoning_items((item,))) + index: Final = output_index if output_index is not None else 0 + return tuple( + _with_stream_block_index(block, index, summary_streamed) + for block in thinking_blocks_from_reasoning_items((item,)) + ) -def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseStream | None: +def _reasoning_done_stream_chunk( + chunk: Mapping[str, object], + summary_streamed_outputs: frozenset[int | None], +) -> ModelResponseStream | None: event_type: Final = chunk.get("type") event_name: Final = ( event_type.value @@ -488,7 +501,8 @@ def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseSt item: Final = as_str_mapping(chunk.get("item")) if item is None or item.get("type") != "reasoning": return None - signed_blocks: Final = _reasoning_done_thinking_blocks(item, chunk.get("output_index")) + output_index: Final = _stream_output_index(chunk) + signed_blocks: Final = _reasoning_done_thinking_blocks(item, output_index, output_index in summary_streamed_outputs) if not signed_blocks: return None return ModelResponseStream( @@ -556,7 +570,6 @@ def _fold_choices_into_one( *chain.from_iterable(_message_reasoning_items(m) for m in messages), *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), ) - encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items) return Choices( message=Message( role=messages[0].role if messages else "assistant", @@ -565,7 +578,6 @@ def _fold_choices_into_one( tool_calls=list(tool_calls) or None, reasoning_content=" ".join(value for value in reasoning_contents if value) or None, reasoning_items=_as_chat_reasoning_items(reasoning_items), - thinking_blocks=list(encrypted_thinking_blocks) or None, provider_specific_fields=_tool_search_provider_fields(tool_search_items), ), finish_reason="tool_calls" if tool_calls else "stop", @@ -1371,12 +1383,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for summary_block in reasoning_item["summary"] if summary_block.get("text") ) - encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items) message: Final = Message( content=None if tool_search_items else "", reasoning_content=reasoning_content if reasoning_content else None, reasoning_items=_as_chat_reasoning_items(reasoning_items), - thinking_blocks=list(encrypted_thinking_blocks) or None, provider_specific_fields=_tool_search_provider_fields(tool_search_items), ) return Choices(message=message, finish_reason=finish_reason, index=0) @@ -1941,6 +1951,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state self._structured_output: Final = structured_output self._first_message_item_done: bool = False + self._summary_streamed_outputs: frozenset[int | None] = frozenset() def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -2255,7 +2266,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) self._remember_served_service_tier(chunk) - reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk) + reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk, self._summary_streamed_outputs) translated: Final = ( self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) @@ -2267,6 +2278,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) if self._structured_output and _is_message_item_done(chunk): self._first_message_item_done = True + if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"): + self._summary_streamed_outputs = self._summary_streamed_outputs | {_stream_output_index(chunk)} return self._with_served_service_tier(self._with_stream_scoped_id(translated)) def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 790860ca561..f5fa6f649a6 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2462,6 +2462,14 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: ) +def _content_carries_encrypted_reasoning(content: object) -> bool: + return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content)) + + +def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool: + return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages)) + + def _strip_encrypted_reasoning_from_blocks( content: object, *, diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 8c0c065c457..fda9fe61060 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -1,5 +1,6 @@ -from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence from dataclasses import dataclass +from functools import partial from typing import ( TYPE_CHECKING, Final, @@ -13,12 +14,15 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.prompt_templates.common_utils import messages_carry_encrypted_reasoning from litellm.litellm_core_utils.tool_fields import ( ROUTES_WITHOUT_MCP_SERVERS, drops_mcp_servers, tools_without_mcp_servers, ) +from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, ) @@ -51,6 +55,7 @@ from litellm.types.utils import ModelResponse from litellm.utils import get_model_info if TYPE_CHECKING: + from litellm.chat_completions.dispatch import ChatResult from litellm.router import Router from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -68,22 +73,68 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object): custom_llm_provider: str +_ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]]) +_PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]" + + @dataclass(frozen=True, slots=True) class _ChatTargetReasoning: effort: str summary: str | None = None +def _messages_without_rejected_reasoning( + messages: _AnthropicMessages, + error: litellm.BadRequestError, +) -> _AnthropicMessages | None: + error_message: Final = str(error) + rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message + if not rejected or not messages_carry_encrypted_reasoning(messages): + return None + verbose_logger.warning("Backend rejected replayed encrypted reasoning, retrying once without it: %s", error) + return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)) + + +async def _acompletion_without_rejected_reasoning( + completion_kwargs: _CompletionKwargs, + messages: _AnthropicMessages, + prepare_replay: _PrepareReplay, +) -> tuple[_CompletionKwargs, "ChatResult"]: + try: + return completion_kwargs, await litellm.acompletion(**completion_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error) + if replayed_messages is None: + raise + replay_kwargs: Final = prepare_replay(replayed_messages) + return replay_kwargs, await litellm.acompletion(**replay_kwargs) + + +def _completion_without_rejected_reasoning( + completion_kwargs: _CompletionKwargs, + messages: _AnthropicMessages, + prepare_replay: _PrepareReplay, +) -> tuple[_CompletionKwargs, "ChatResult | Coroutine[object, object, ChatResult]"]: + try: + return completion_kwargs, litellm.completion(**completion_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error) + if replayed_messages is None: + raise + replay_kwargs: Final = prepare_replay(replayed_messages) + return replay_kwargs, litellm.completion(**replay_kwargs) + + +def _is_compaction_block(block: object) -> bool: + mapping: Final = as_str_mapping(block) + return mapping is not None and mapping.get("type") == "compaction" + + def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool: """Return True when any message carries a ``compaction`` content block.""" - for msg in messages: - content = msg.get("content") - if not isinstance(content, list): - continue - for block in content: - if isinstance(block, dict) and block.get("type") == "compaction": - return True - return False + return any( + any(map(_is_compaction_block, content)) for msg in messages if isinstance(content := msg.get("content"), list) + ) def _proxy_router_fallback() -> "Router | None": @@ -646,12 +697,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: if "output_config" in extra_kwargs: request_data["output_config"] = extra_kwargs["output_config"] + api_base: Final = provider_kwargs.get("api_base") ( openai_request, tool_name_mapping, ) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping( request_data, custom_llm_provider=target_provider, + api_base=api_base if isinstance(api_base, str) else None, ) if openai_request is None: @@ -779,12 +832,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: parent_trace_id=sub_call_trace_id, ) - ( - completion_kwargs, - tool_name_mapping, - ) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + prepare: Final = partial( + LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs, max_tokens=max_tokens, - messages=effective_messages, model=model, metadata=metadata, stop_sequences=stop_sequences, @@ -799,8 +849,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: output_format=output_format, extra_kwargs=kwargs, ) - - completion_response: Final = await litellm.acompletion(**completion_kwargs) + initial_kwargs, tool_name_mapping = prepare(messages=effective_messages) + completion_kwargs, completion_response = await _acompletion_without_rejected_reasoning( + initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0] + ) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: @@ -924,12 +976,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages effective_system: Final = polyfill_result.system if polyfill_result is not None else system - ( - completion_kwargs, - tool_name_mapping, - ) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + prepare: Final = partial( + LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs, max_tokens=max_tokens, - messages=effective_messages, model=model, metadata=metadata, stop_sequences=stop_sequences, @@ -944,8 +993,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: output_format=output_format, extra_kwargs=kwargs, ) - - completion_response: Final = litellm.completion(**completion_kwargs) + initial_kwargs, tool_name_mapping = prepare(messages=effective_messages) + completion_kwargs, completion_response = _completion_without_rejected_reasoning( + initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0] + ) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index 0f1398ac9b2..cc705ab1184 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -24,9 +24,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError -from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.litellm_core_utils.hidden_params import set_hidden_params -from litellm.litellm_core_utils.prompt_templates.common_utils import thinking_blocks_from_reasoning_items from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -68,10 +66,6 @@ def _optional_attr(obj: object, name: str) -> object: return getattr(obj, name, None) -def _delta_has_encrypted_reasoning(delta: object) -> bool: - return bool(thinking_blocks_from_reasoning_items(attribute_or_none(delta, "reasoning_items"))) - - def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]: value: Final = getattr(obj, name, None) return value if value else () @@ -1312,8 +1306,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks ): return False - if _delta_has_encrypted_reasoning(delta): - return False return True def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 62f33bb11fd..ae7c433de2a 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -2,12 +2,14 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.pass_through.utils import ( is_reasoning_auto_summary_enabled, @@ -220,10 +222,30 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider) -def _keeps_encrypted_reasoning_for_responses(model: str | None, custom_llm_provider: str | None) -> bool: +@dataclass(frozen=True, slots=True) +class ChatRouteInputs: + tools: Sequence[Mapping[str, object]] | None = None + reasoning_effort: str | Mapping[str, object] | None = None + api_base: str | None = None + + +_NO_ROUTE_INPUTS: Final = ChatRouteInputs() + + +def _keeps_encrypted_reasoning_for_responses( + model: str | None, + custom_llm_provider: str | None, + route: ChatRouteInputs, +) -> bool: from litellm.responses.bridge_check import chat_completion_ends_in_responses_api - return chat_completion_ends_in_responses_api(model, custom_llm_provider) + return chat_completion_ends_in_responses_api( + model, + custom_llm_provider, + tools=route.tools, + reasoning_effort=route.reasoning_effort, + api_base=route.api_base, + ) def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]: @@ -233,6 +255,10 @@ def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], .. ) +def _streamed_thinking_blocks(delta: object) -> tuple[Mapping[str, object], ...]: + return tuple(block for item in attribute_items(delta, "thinking_blocks") if (block := as_str_mapping(item))) + + def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam: eager_input_streaming: Final = eager_input_streaming_flag(tool) if eager_input_streaming is None: @@ -260,7 +286,7 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None + self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -294,6 +320,7 @@ class AnthropicAdapter: ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, + api_base=api_base, ) return translated_body, tool_name_mapping @@ -479,11 +506,12 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, preserve_midturn_system: bool = False, + route: ChatRouteInputs = _NO_ROUTE_INPUTS, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] replayable_messages: Final = ( messages - if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider) + if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider, route) else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) ) leading_count: Final = next( @@ -1239,6 +1267,7 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, *, custom_llm_provider: str | None = None, + api_base: str | None = None, preserve_midturn_system: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ @@ -1250,26 +1279,15 @@ class LiteLLMAnthropicMessagesAdapter: for tools that exceeded OpenAI's 64-char limit """ # Debug: Processing Anthropic message request - new_messages: list[AllMessageValues] = [] tool_name_mapping: dict[str, str] = {} - ## CONVERT ANTHROPIC MESSAGES TO OPENAI messages_list: Final[list[AllAnthropicPassThroughMessageValues]] = cast( list[AllAnthropicPassThroughMessageValues], anthropic_message_request["messages"], ) - new_messages = self.translate_anthropic_messages_to_openai( - messages=messages_list, - model=anthropic_message_request.get("model"), - custom_llm_provider=custom_llm_provider, - preserve_midturn_system=preserve_midturn_system, - ) - ## ADD SYSTEM MESSAGE TO MESSAGES - self._add_system_message_to_messages(new_messages, anthropic_message_request) - new_kwargs: Final[ChatCompletionRequest] = { "model": anthropic_message_request["model"], - "messages": new_messages, + "messages": [], } ## CONVERT METADATA (user_id + litellm metadata) self._translate_metadata_to_openai( @@ -1293,6 +1311,21 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, ) + ## CONVERT ANTHROPIC MESSAGES TO OPENAI, after tools and thinking pick the route + new_messages: Final = self.translate_anthropic_messages_to_openai( + messages=messages_list, + model=anthropic_message_request.get("model"), + custom_llm_provider=custom_llm_provider, + preserve_midturn_system=preserve_midturn_system, + route=ChatRouteInputs( + tools=new_kwargs.get("tools"), + reasoning_effort=new_kwargs.get("reasoning_effort"), + api_base=api_base, + ), + ) + ## ADD SYSTEM MESSAGE TO MESSAGES + self._add_system_message_to_messages(new_messages, anthropic_message_request) + new_kwargs["messages"] = new_messages ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( anthropic_message_request=anthropic_message_request, @@ -1678,7 +1711,7 @@ class LiteLLMAnthropicMessagesAdapter: choice.delta ) is not None: return "text", TextBlock(type="text", text="") - elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)): + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)): thinking_block = thinking_blocks[0] if thinking_block["type"] == "redacted_thinking": redacted_data = thinking_block.get("data") @@ -1728,7 +1761,7 @@ class LiteLLMAnthropicMessagesAdapter: for tool in choice.delta.tool_calls: if tool.function is not None and tool.function.arguments is not None: partial_json = (partial_json or "") + tool.function.arguments - elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)): + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)): for thinking_block in thinking_blocks: if thinking_block["type"] == "thinking": thinking = thinking_block.get("thinking") or "" diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 7ed3ceda1eb..cdd10d489a1 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -191,12 +191,19 @@ def responses_api_bridge_check( return model_info, model -def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider: str | None) -> bool: +def chat_completion_ends_in_responses_api( + model: str | None, + custom_llm_provider: str | None, + *, + tools: Sequence[Mapping[str, object]] | None = None, + reasoning_effort: str | Mapping[str, object] | None = None, + api_base: str | None = None, +) -> bool: """Whether chat completion for this model is served by a Responses API. Encrypted reasoning can be replayed only there. A normal chat provider cannot - verify the block, so the messages adapter strips it. ``chatgpt`` and a - ``responses/`` model keep it. + verify the block, so the messages adapter strips it. ``chatgpt``, a ``responses/`` + model, and a request whose tools or reasoning effort the bridge sends to Responses keep it. """ if model is None and custom_llm_provider is None: return False @@ -207,5 +214,11 @@ def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider return custom_llm_provider == LlmProviders.CHATGPT.value or ( isinstance(model, str) and (model == "chatgpt" or model.startswith("chatgpt/")) ) - model_info, _resolved = responses_api_bridge_check(model=resolved_model, custom_llm_provider=provider) + model_info, _resolved = responses_api_bridge_check( + model=resolved_model, + custom_llm_provider=provider, + tools=tools, + reasoning_effort=reasoning_effort, + api_base=api_base, + ) return model_info.get("mode") == "responses" diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index f24f163541f..d1155e41bdf 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3253,7 +3253,7 @@ def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keep ) from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper - from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter encrypted: Final = "gAAAA_round_trip" done: Final = { @@ -3318,6 +3318,74 @@ def test_reasoning_output_item_done_with_summary_signs_the_thinking_text(): assert block["index"] == 2 +def test_streamed_reasoning_summary_reaches_messages_clients_once_with_its_signature(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import AnthropicStreamWrapper + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + encrypted: Final = "gAAAA_streamed_summary" + events: Final = ( + {"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0}, + {"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "Oslo is "}, + {"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "colder."}, + { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_streamed", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + }, + }, + {"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."}, + ) + parser: Final = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + chunks: Final = ( + *(parser.chunk_parser(event) for event in events), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), + ) + + sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="chatgpt/gpt-5.5")) + deltas: Final = [event["delta"] for event in sse_events if event.get("type") == "content_block_delta"] + assert "".join(d.get("thinking", "") for d in deltas if d["type"] == "thinking_delta") == "Oslo is colder." + assert [d["signature"] for d in deltas if d["type"] == "signature_delta"] == [ + encrypted_reasoning_signature(encrypted) + ] + + +def test_reasoning_item_repeated_at_stream_end_reaches_messages_clients_once(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import AnthropicStreamWrapper + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + encrypted: Final = "gAAAA_completed_repeat" + reasoning_item: Final = {"type": "reasoning", "id": "rs_repeat", "summary": [], "encrypted_content": encrypted} + events: Final = ( + {"type": "response.output_item.done", "output_index": 0, "item": reasoning_item}, + {"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."}, + ) + parser: Final = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + chunks: Final = ( + *(parser.chunk_parser(event) for event in events), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(reasoning_items=[reasoning_item]), finish_reason=None)] + ), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), + ) + + sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="azure/gpt-5.4-nano")) + started: Final = [event["content_block"] for event in sse_events if event.get("type") == "content_block_start"] + assert [block["type"] for block in started] == ["redacted_thinking", "text"] + assert started[0]["data"] == encrypted_reasoning_signature(encrypted) + + def test_reasoning_output_item_done_without_encrypted_content_stays_empty(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( OpenAiResponsesToChatCompletionStreamIterator, @@ -3338,10 +3406,13 @@ def test_reasoning_output_item_done_without_encrypted_content_stays_empty(): assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, []) -def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks(): +def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only(): from unittest.mock import Mock from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.llms.anthropic.pass_through.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse, Usage @@ -3394,10 +3465,16 @@ def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks(): ) message: Final = result.choices[0].message assert message.reasoning_items[0]["encrypted_content"] == encrypted - assert message.thinking_blocks[0]["type"] == "redacted_thinking" - assert message.thinking_blocks[0]["data"] == encrypted_reasoning_signature(encrypted) + assert "thinking_blocks" not in result.model_dump()["choices"][0]["message"] assert message.tool_calls[0].function.name == "get_weather" + anthropic_response: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=result, tool_name_mapping=None + ) + content: Final = anthropic_response["content"] + assert [block["type"] for block in content] == ["redacted_thinking", "tool_use"] + assert content[0]["data"] == encrypted_reasoning_signature(encrypted) + def test_streaming_function_call_tool_id_for_degenerate_call_id(): """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 49a8341ce9e..bc357878944 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -4,7 +4,9 @@ import json from types import MappingProxyType from typing import Any, Final, cast +import httpx import pytest +import respx import litellm @@ -31,6 +33,7 @@ from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.anthropic import ( AnthopicMessagesAssistantMessageParam, + AnthropicMessagesRequest, AnthropicMessagesUserMessageParam, ) from litellm.types.llms.openai import ChatCompletionAssistantToolCall @@ -647,26 +650,50 @@ def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_block assert [block["type"] for block in result] == ["redacted_thinking", "text"] -def test_streaming_chunk_with_only_reasoning_items_opens_a_signed_thinking_block(): - encrypted: Final = "gAAAA_delta" - choices: Final = [ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - reasoning_items=[ - {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} - ] +def _replayed_reasoning_request(tools: list[dict[str, object]]) -> AnthropicMessagesRequest: + return AnthropicMessagesRequest( + model="azure/gpt-5.4-nano", + max_tokens=64, + messages=[ + AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_azure_tools")}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], ), - ) - ] - - block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block( - choices=choices + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}], + ), + ], + tools=tools, ) - assert block_type == "redacted_thinking" - assert start["data"] == encrypted_reasoning_signature(encrypted) + +@pytest.mark.parametrize( + ("tools", "kept"), + [ + pytest.param( + [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + True, + id="function_tools_bridge_to_responses", + ), + pytest.param([], False, id="no_tools_stay_on_chat"), + ], +) +def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_responses( + tools: list[dict[str, object]], kept: bool +) -> None: + openai_request, _mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + _replayed_reasoning_request(tools), + custom_llm_provider="azure", + api_base="https://keep-check.openai.azure.com", + ) + + assistant: Final = openai_request["messages"][1] + assert bool(assistant.get("thinking_blocks")) is kept def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): @@ -5395,3 +5422,147 @@ def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta(): assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"] assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA + + +@pytest.fixture +def _chat_bridge_httpx(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +_RETRY_API_BASE: Final = "https://encrypted-retry.openai.azure.com" +_RETRY_URL: Final = r".*encrypted-retry\.openai\.azure\.com/openai/.*responses.*" +_RETRY_TOOLS: Final = [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}] +_RETRY_REJECTION: Final = httpx.Response( + 400, + json={ + "error": { + "message": "The encrypted content gAAA_foreign could not be verified.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_encrypted_content", + } + }, +) +_UNRELATED_REJECTION: Final = httpx.Response( + 400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}} +) +_RETRY_REPLY: Final = { + "id": "resp_retry", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.4-nano", + "output": [ + { + "type": "message", + "id": "msg_retry", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Oslo", "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], +} + + +def _history_with_foreign_reasoning() -> list[dict[str, object]]: + return [ + {"role": "user", "content": "Which city is colder?"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}]}, + ] + + +def _sent_reasoning_items(request: httpx.Request) -> list[object]: + sent: Final = json.loads(request.read()) + return [item for item in sent["input"] if item.get("type") == "reasoning"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert route.call_count == 2 + assert len(_sent_reasoning_items(route.calls[0].request)) == 1 + assert _sent_reasoning_items(route.calls[1].request) == [] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert route.call_count == 2 + assert _sent_reasoning_items(route.calls[1].request) == [] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize( + ("messages", "rejection"), + [ + pytest.param( + [{"role": "user", "content": "Which city is colder?"}], + _RETRY_REJECTION, + id="no_encrypted_reasoning_to_drop", + ), + pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"), + ], +) +async def test_messages_do_not_retry_other_rejections( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], rejection: httpx.Response +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(return_value=rejection) + + with pytest.raises(litellm.BadRequestError): + await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=messages, + ) + + assert route.call_count == 1 From 1fa046752a1df092b9cac7b9c9bc262556a334ca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:02:52 -0800 Subject: [PATCH 05/28] fix(anthropic): let Python 3.10 import the backported messages handler and clear its new lint and type errors Quote the rejected-reasoning return annotation, since a string TypeAlias cannot take `| None` at definition time before Python 3.14. Return only the response from the retry helpers, since this line's stream wrapper takes no completion request. Type the bridge check result and the content block union the way main does, and validate the object tuple through TypeAdapter. --- .../prompt_templates/common_utils.py | 4 +++- .../pass_through/adapters/handler.py | 20 +++++++++---------- litellm/responses/bridge_check.py | 2 +- litellm/types/llms/anthropic.py | 4 +++- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f5fa6f649a6..f9c4ebca350 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2292,11 +2292,13 @@ def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: _REASONING_SUMMARY_SEPARATOR: Final = "\n\n" +_OBJECT_TUPLE: Final = TypeAdapter(tuple[object, ...]) + def _object_sequence(value: object) -> tuple[object, ...]: if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): return () - return tuple(value) + return _OBJECT_TUPLE.validate_python(value) def _iter_mappings(value: object) -> tuple[Mapping[str, object], ...]: diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index fda9fe61060..e385ee194b3 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -86,7 +86,7 @@ class _ChatTargetReasoning: def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, -) -> _AnthropicMessages | None: +) -> "_AnthropicMessages | None": error_message: Final = str(error) rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message if not rejected or not messages_carry_encrypted_reasoning(messages): @@ -99,30 +99,28 @@ async def _acompletion_without_rejected_reasoning( completion_kwargs: _CompletionKwargs, messages: _AnthropicMessages, prepare_replay: _PrepareReplay, -) -> tuple[_CompletionKwargs, "ChatResult"]: +) -> "ChatResult": try: - return completion_kwargs, await litellm.acompletion(**completion_kwargs) + return await litellm.acompletion(**completion_kwargs) except litellm.BadRequestError as error: replayed_messages: Final = _messages_without_rejected_reasoning(messages, error) if replayed_messages is None: raise - replay_kwargs: Final = prepare_replay(replayed_messages) - return replay_kwargs, await litellm.acompletion(**replay_kwargs) + return await litellm.acompletion(**prepare_replay(replayed_messages)) def _completion_without_rejected_reasoning( completion_kwargs: _CompletionKwargs, messages: _AnthropicMessages, prepare_replay: _PrepareReplay, -) -> tuple[_CompletionKwargs, "ChatResult | Coroutine[object, object, ChatResult]"]: +) -> "ChatResult | Coroutine[object, object, ChatResult]": try: - return completion_kwargs, litellm.completion(**completion_kwargs) + return litellm.completion(**completion_kwargs) except litellm.BadRequestError as error: replayed_messages: Final = _messages_without_rejected_reasoning(messages, error) if replayed_messages is None: raise - replay_kwargs: Final = prepare_replay(replayed_messages) - return replay_kwargs, litellm.completion(**replay_kwargs) + return litellm.completion(**prepare_replay(replayed_messages)) def _is_compaction_block(block: object) -> bool: @@ -850,7 +848,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: extra_kwargs=kwargs, ) initial_kwargs, tool_name_mapping = prepare(messages=effective_messages) - completion_kwargs, completion_response = await _acompletion_without_rejected_reasoning( + completion_response: Final = await _acompletion_without_rejected_reasoning( initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0] ) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) @@ -994,7 +992,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: extra_kwargs=kwargs, ) initial_kwargs, tool_name_mapping = prepare(messages=effective_messages) - completion_kwargs, completion_response = _completion_without_rejected_reasoning( + completion_response: Final = _completion_without_rejected_reasoning( initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0] ) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index cdd10d489a1..d8a9440bc57 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -72,7 +72,7 @@ def responses_api_bridge_check( reasoning_effort: str | Mapping[str, object] | None = None, reasoning_summary: object | None = None, api_base: str | None = None, -) -> tuple[dict, str]: +) -> tuple[dict[str, object], str]: model_info: dict[str, object] = {} # Global flag: route ALL OpenAI chat completions through Responses API. diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 53abac75162..2f59c1765a5 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -545,7 +545,9 @@ class ContentBlockStartText(TypedDict): content_block: TextBlock -ContentBlockContentBlockDict = ToolUseBlock | TextBlock | ChatCompletionThinkingBlock +ContentBlockContentBlockDict = ( + ToolUseBlock | TextBlock | ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock +) ContentBlockStart = ContentBlockStartToolUse | ContentBlockStartText From 0b2cfa5c1ed57606b57cfb8cd3fcd1454fc7ba50 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:16:10 -0800 Subject: [PATCH 06/28] fix(anthropic): keep encrypted reasoning when OpenAI thinking routes the chat adapter to Responses With use_chat_completions_url_for_anthropic_messages on, an openai model with thinking enabled is rewritten to openai/responses/, which accepts the same organization's encrypted reasoning. The keep check now uses the same routing predicate, so turn 2 replays the block instead of stripping it. --- .../pass_through/adapters/handler.py | 34 ++++--------------- .../pass_through/adapters/transformation.py | 30 +++++++++++++++- ...al_pass_through_adapters_transformation.py | 33 ++++++++++++++++++ 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index e385ee194b3..8b83fe5b7a4 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.tool_fields import ( from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages from litellm.llms.anthropic.pass_through.adapters.transformation import ( AnthropicAdapter, + openai_thinking_routes_to_responses, ) from litellm.llms.anthropic.pass_through.context_management import ( AnthropicContextManagementError, @@ -52,7 +53,6 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import OpenAIWebSearchOptions from litellm.types.utils import ModelResponse -from litellm.utils import get_model_info if TYPE_CHECKING: from litellm.chat_completions.dispatch import ChatResult @@ -462,34 +462,14 @@ class LiteLLMMessagesToCompletionTransformationHandler: If the user provides a `summary` field in the thinking dict, it is passed through to the OpenAI reasoning params (opt-in per OpenAI spec). """ - custom_llm_provider = completion_kwargs.get("custom_llm_provider") - if custom_llm_provider is None: - try: - _, inferred_provider, _, _ = litellm.utils.get_llm_provider( - model=cast(str, completion_kwargs.get("model")) - ) - custom_llm_provider = inferred_provider - except Exception: - custom_llm_provider = None - - if custom_llm_provider != "openai": - return - - if not isinstance(thinking, dict) or thinking.get("type") != "enabled": - return - model: Final = completion_kwargs.get("model") - try: - model_info: Final = get_model_info(model=cast(str, model), custom_llm_provider=custom_llm_provider) - if model_info and model_info.get("supports_reasoning") is False: - # Model doesn't support reasoning/responses API, don't route - return - except Exception: - pass + if not isinstance(model, str) or thinking is None: + return + if not openai_thinking_routes_to_responses(model, completion_kwargs.get("custom_llm_provider"), thinking): + return - if isinstance(model, str) and model and "responses/" not in model: - local_model: Final = model.removeprefix(f"{custom_llm_provider}/") - completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}" + if "responses/" not in model: + completion_kwargs["model"] = f"openai/responses/{model.removeprefix('openai/')}" auto_summary: Final = is_reasoning_auto_summary_enabled() diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index ae7c433de2a..ee553749c74 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -227,11 +227,36 @@ class ChatRouteInputs: tools: Sequence[Mapping[str, object]] | None = None reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None + thinking: Mapping[str, object] | None = None _NO_ROUTE_INPUTS: Final = ChatRouteInputs() +def _inferred_provider(model: str) -> str | None: + try: + return litellm.utils.get_llm_provider(model=model)[1] + except Exception: # noqa: BLE001 # an unknown model name has no inferred provider + return None + + +def openai_thinking_routes_to_responses( + model: str | None, + custom_llm_provider: str | None, + thinking: Mapping[str, object] | None, +) -> bool: + if not model or thinking is None or thinking.get("type") != "enabled": + return False + provider: Final = custom_llm_provider or _inferred_provider(model) + if provider != "openai": + return False + try: + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=provider) + except Exception: # noqa: BLE001 # a model missing from the cost map still routes, as before + return True + return model_info.get("supports_reasoning") is not False + + def _keeps_encrypted_reasoning_for_responses( model: str | None, custom_llm_provider: str | None, @@ -239,7 +264,9 @@ def _keeps_encrypted_reasoning_for_responses( ) -> bool: from litellm.responses.bridge_check import chat_completion_ends_in_responses_api - return chat_completion_ends_in_responses_api( + return openai_thinking_routes_to_responses( + model, custom_llm_provider, route.thinking + ) or chat_completion_ends_in_responses_api( model, custom_llm_provider, tools=route.tools, @@ -1321,6 +1348,7 @@ class LiteLLMAnthropicMessagesAdapter: tools=new_kwargs.get("tools"), reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, + thinking=anthropic_message_request.get("thinking"), ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index bc357878944..7a8caa417df 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5566,3 +5566,36 @@ async def test_messages_do_not_retry_other_rejections( ) assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_openai_thinking_on_the_chat_adapter_keeps_reasoning_its_responses_route_replays( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + route: Final = respx_mock.post(url__regex=r".*openai-thinking\.example/v1/responses").mock( + return_value=httpx.Response(200, json=_RETRY_REPLY) + ) + + await litellm.anthropic.messages.acreate( + model="openai/gpt-5-mini", + api_base="https://openai-thinking.example/v1", + api_key="fake-key", + max_tokens=2048, + thinking={"type": "enabled", "budget_tokens": 1024}, + messages=[ + {"role": "user", "content": "Which city is colder?"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")}, + {"type": "text", "text": "Oslo."}, + ], + }, + {"role": "user", "content": "And by how much?"}, + ], + ) + + assert route.call_count == 1 + assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] From 4fc1d5bb0d077898f46b2649abe1c805abbf8354 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:16:29 -0800 Subject: [PATCH 07/28] fix(anthropic): stop logging rejected encrypted reasoning The retry warning keeps provider, model, status, and error code, and leaves the ciphertext out. Backport of the third #49076 commit to lts/1.108.x. Cherry-picked from c62c92d2ad0. Co-authored-by: DDT <141296842+Hugo-DDT@users.noreply.github.com> --- .../pass_through/adapters/handler.py | 37 +++++++++++++++++-- ...al_pass_through_adapters_transformation.py | 34 ++++++++++++----- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 8b83fe5b7a4..963dd050804 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -83,15 +83,46 @@ class _ChatTargetReasoning: summary: str | None = None +def _provider_error_code(body: object) -> str | None: + mapping: Final = as_str_mapping(body) + if mapping is None: + return None + nested: Final = as_str_mapping(mapping.get("error")) + source: Final = nested if nested is not None else mapping + code: Final = source.get("code") + return code if isinstance(code, str) else None + + +def _logged_rejection_code(error: litellm.BadRequestError) -> str: + if error.code == "invalid_encrypted_content" or _provider_error_code(error.body) == "invalid_encrypted_content": + return "invalid_encrypted_content" + if "invalid_encrypted_content" in str(error): + return "invalid_encrypted_content" + return "could_not_be_verified" + + +def _completion_text(completion_kwargs: _CompletionKwargs, key: str) -> str: + value: Final = completion_kwargs.get(key) + return value if isinstance(value, str) else "" + + def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, + completion_kwargs: _CompletionKwargs, ) -> "_AnthropicMessages | None": error_message: Final = str(error) rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message if not rejected or not messages_carry_encrypted_reasoning(messages): return None - verbose_logger.warning("Backend rejected replayed encrypted reasoning, retrying once without it: %s", error) + verbose_logger.warning( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "provider=%s model=%s status=%s code=%s", + _completion_text(completion_kwargs, "custom_llm_provider"), + _completion_text(completion_kwargs, "model"), + error.status_code, + _logged_rejection_code(error), + ) return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)) @@ -103,7 +134,7 @@ async def _acompletion_without_rejected_reasoning( try: return await litellm.acompletion(**completion_kwargs) except litellm.BadRequestError as error: - replayed_messages: Final = _messages_without_rejected_reasoning(messages, error) + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, completion_kwargs) if replayed_messages is None: raise return await litellm.acompletion(**prepare_replay(replayed_messages)) @@ -117,7 +148,7 @@ def _completion_without_rejected_reasoning( try: return litellm.completion(**completion_kwargs) except litellm.BadRequestError as error: - replayed_messages: Final = _messages_without_rejected_reasoning(messages, error) + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, completion_kwargs) if replayed_messages is None: raise return litellm.completion(**prepare_replay(replayed_messages)) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 7a8caa417df..3d87772595a 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -1,6 +1,7 @@ import base64 import functools import json +import logging from types import MappingProxyType from typing import Any, Final, cast @@ -5492,25 +5493,40 @@ def _sent_reasoning_items(request: httpx.Request) -> list[object]: @pytest.mark.usefixtures("_chat_bridge_httpx") async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( respx_mock: respx.MockRouter, + caplog: pytest.LogCaptureFixture, ) -> None: route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] ) - response: Final = await litellm.anthropic.messages.acreate( - model="azure/gpt-5.4-nano", - api_base=_RETRY_API_BASE, - api_key="fake-key", - api_version="2025-04-01-preview", - max_tokens=64, - tools=_RETRY_TOOLS, - messages=_history_with_foreign_reasoning(), - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + warnings: Final = tuple( + record.getMessage() + for record in caplog.records + if record.levelno >= logging.WARNING and record.name == "LiteLLM" + ) + retry_warnings: Final = tuple(message for message in warnings if "retrying once without it" in message) assert route.call_count == 2 assert len(_sent_reasoning_items(route.calls[0].request)) == 1 assert _sent_reasoning_items(route.calls[1].request) == [] assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + assert retry_warnings == ( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content", + ) + assert all( + "gAAA_foreign" not in message and "litellm_encrypted_reasoning:" not in message for message in warnings + ) @pytest.mark.usefixtures("_chat_bridge_httpx") From ba5d7b83b950b9913847e0a63e0cc7afdf5fbeb8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:30:43 -0800 Subject: [PATCH 08/28] fix(anthropic): start a new streamed block for each reasoning item The backported Responses bridge tags each reasoning item's thinking block with its output index, but this line's Anthropic stream wrapper only opened a new block when the block type changed, so a second reasoning item of the same type was dropped. This ports the index check from #46863, which main already has. --- .../adapters/streaming_iterator.py | 23 ++++++ ...al_pass_through_adapters_transformation.py | 73 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index cc705ab1184..d8f4ad0b802 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -24,6 +24,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame @@ -71,6 +72,14 @@ def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]: return value if value else () +def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None: + delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None + blocks: Final = _optional_attr_sequence(delta, "thinking_blocks") + first_block: Final = as_str_mapping(blocks[0]) if blocks else None + index: Final = first_block.get("index") if first_block is not None else None + return index if isinstance(index, int) else None + + def _error_status_and_message(exc: Exception) -> tuple[int, str]: if isinstance(exc, (BaseLLMException, MidStreamFallbackError, ContextWindowExceededError)): return exc.status_code, exc.message @@ -464,6 +473,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): holding_stop_reason_chunk: MessageBlockDelta | None = None queued_usage_chunk: bool = False current_content_block_index: int = 0 + current_thinking_block_index: int | None = None def __init__( self, @@ -1352,11 +1362,24 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name) tool_block["name"] = original_name + thinking_block_index: Final = _first_thinking_block_index(chunk) + previous_thinking_block_index: Final = self.current_thinking_block_index + if thinking_block_index is not None or block_type != self.current_content_block_type: + self.current_thinking_block_index = thinking_block_index + if block_type != self.current_content_block_type: self.current_content_block_type = block_type self.current_content_block_start = content_block_start return True + if ( + thinking_block_index is not None + and previous_thinking_block_index is not None + and thinking_block_index != previous_thinking_block_index + ): + self.current_content_block_start = content_block_start + return True + # For parallel tool calls, we'll necessarily have a new content block # if we get a function name since it signals a new tool call if block_type == "tool_use": diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 3d87772595a..6411d57d1bd 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -2,6 +2,7 @@ import base64 import functools import json import logging +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Any, Final, cast @@ -5615,3 +5616,75 @@ async def test_openai_thinking_on_the_chat_adapter_keeps_reasoning_its_responses assert route.call_count == 1 assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] + + +def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes: + return b"".join( + f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode() + for number, event in enumerate(events) + ) + + +def _reasoning_item(item_id: str, encrypted: str) -> dict[str, object]: + return {"type": "reasoning", "id": item_id, "summary": [], "encrypted_content": encrypted} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_streamed_consecutive_reasoning_items_each_become_their_own_block( + respx_mock: respx.MockRouter, +) -> None: + response: Final = {**_RETRY_REPLY, "id": "resp_two_reasoning", "status": "in_progress", "output": []} + message: Final = { + "type": "message", + "id": "msg_two_reasoning", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Oslo", "annotations": []}], + } + first: Final = _reasoning_item("rs_1", "gAAA_first") + second: Final = _reasoning_item("rs_2", "gAAA_second") + completed: Final = {**response, "status": "completed", "output": [first, second, message]} + events: Final = ( + {"type": "response.created", "response": response}, + {"type": "response.output_item.added", "output_index": 0, "item": _reasoning_item("rs_1", "")}, + {"type": "response.output_item.done", "output_index": 0, "item": first}, + {"type": "response.output_item.added", "output_index": 1, "item": _reasoning_item("rs_2", "")}, + {"type": "response.output_item.done", "output_index": 1, "item": second}, + {"type": "response.output_item.added", "output_index": 2, "item": {**message, "content": []}}, + { + "type": "response.output_text.delta", + "output_index": 2, + "item_id": "msg_two_reasoning", + "content_index": 0, + "delta": "Oslo", + }, + {"type": "response.output_item.done", "output_index": 2, "item": message}, + {"type": "response.completed", "response": completed}, + ) + respx_mock.post(url__regex=_RETRY_URL).mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) + ) + + stream: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + stream=True, + messages=[{"role": "user", "content": "Which city is colder?"}], + ) + frames: Final = b"".join([chunk async for chunk in stream]).decode() + starts: Final = tuple( + json.loads(line.removeprefix("data: "))["content_block"] + for line in frames.splitlines() + if line.startswith("data: ") and '"content_block_start"' in line + ) + + assert [(block["type"], block.get("data")) for block in starts] == [ + ("redacted_thinking", encrypted_reasoning_signature("gAAA_first")), + ("redacted_thinking", encrypted_reasoning_signature("gAAA_second")), + ("text", None), + ] From 27574a429a54903c8708f3153e06a667b7bf69cf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 20:45:06 -0800 Subject: [PATCH 09/28] fix(anthropic): keep each streamed reasoning summary with its own item A summary delta carries no item index, so the next item's summary joined the block that already got its signature. A thinking chunk after an indexed signature now opens a new block. --- .../adapters/streaming_iterator.py | 27 +++-- ...al_pass_through_adapters_transformation.py | 100 +++++++++++++++++- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index d8f4ad0b802..6f094256197 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -72,14 +72,21 @@ def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]: return value if value else () -def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None: +def _first_thinking_block(chunk: "ModelResponseStream") -> Mapping[str, object] | None: delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None blocks: Final = _optional_attr_sequence(delta, "thinking_blocks") - first_block: Final = as_str_mapping(blocks[0]) if blocks else None - index: Final = first_block.get("index") if first_block is not None else None + return as_str_mapping(blocks[0]) if blocks else None + + +def _thinking_block_index(block: Mapping[str, object] | None) -> int | None: + index: Final = block.get("index") if block is not None else None return index if isinstance(index, int) else None +def _closes_thinking_block(block: Mapping[str, object] | None) -> bool: + return block is not None and _thinking_block_index(block) is not None and bool(block.get("signature")) + + def _error_status_and_message(exc: Exception) -> tuple[int, str]: if isinstance(exc, (BaseLLMException, MidStreamFallbackError, ContextWindowExceededError)): return exc.status_code, exc.message @@ -474,6 +481,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): queued_usage_chunk: bool = False current_content_block_index: int = 0 current_thinking_block_index: int | None = None + current_thinking_block_closed: bool = False def __init__( self, @@ -1362,9 +1370,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name) tool_block["name"] = original_name - thinking_block_index: Final = _first_thinking_block_index(chunk) + thinking_block: Final = _first_thinking_block(chunk) + thinking_block_index: Final = _thinking_block_index(thinking_block) previous_thinking_block_index: Final = self.current_thinking_block_index - if thinking_block_index is not None or block_type != self.current_content_block_type: + follows_closed_thinking_block: Final = self.current_thinking_block_closed and block_type == "thinking" + self.current_thinking_block_closed = _closes_thinking_block(thinking_block) + if ( + thinking_block_index is not None + or block_type != self.current_content_block_type + or follows_closed_thinking_block + ): self.current_thinking_block_index = thinking_block_index if block_type != self.current_content_block_type: @@ -1372,7 +1387,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): self.current_content_block_start = content_block_start return True - if ( + if follows_closed_thinking_block or ( thinking_block_index is not None and previous_thinking_block_index is not None and thinking_block_index != previous_thinking_block_index diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 6411d57d1bd..cda53b82931 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5625,8 +5625,13 @@ def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes: ) -def _reasoning_item(item_id: str, encrypted: str) -> dict[str, object]: - return {"type": "reasoning", "id": item_id, "summary": [], "encrypted_content": encrypted} +def _reasoning_item(item_id: str, encrypted: str, summary: str = "") -> dict[str, object]: + return { + "type": "reasoning", + "id": item_id, + "summary": [{"type": "summary_text", "text": summary}] if summary else [], + "encrypted_content": encrypted, + } @pytest.mark.asyncio @@ -5688,3 +5693,94 @@ async def test_streamed_consecutive_reasoning_items_each_become_their_own_block( ("redacted_thinking", encrypted_reasoning_signature("gAAA_second")), ("text", None), ] + + +def _streamed_blocks(frames: str) -> list[tuple[str, str, str | None]]: + events: Final = tuple( + json.loads(line.removeprefix("data: ")) for line in frames.splitlines() if line.startswith("data: ") + ) + deltas: Final = tuple(event for event in events if event["type"] == "content_block_delta") + + def block_deltas(index: int) -> tuple[Mapping[str, str], ...]: + return tuple(delta["delta"] for delta in deltas if delta["index"] == index) + + return [ + ( + event["content_block"]["type"], + "".join(delta.get("thinking", "") for delta in block_deltas(event["index"])), + next( + (delta["signature"] for delta in block_deltas(event["index"]) if delta["type"] == "signature_delta"), + None, + ), + ) + for event in events + if event["type"] == "content_block_start" + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_streamed_reasoning_summaries_stay_with_their_own_encrypted_item( + respx_mock: respx.MockRouter, +) -> None: + response: Final = {**_RETRY_REPLY, "id": "resp_two_summaries", "status": "in_progress", "output": []} + message: Final = { + "type": "message", + "id": "msg_two_summaries", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Oslo", "annotations": []}], + } + first: Final = _reasoning_item("rs_1", "gAAA_first", "Oslo is further north.") + second: Final = _reasoning_item("rs_2", "gAAA_second", "Check the forecast.") + completed: Final = {**response, "status": "completed", "output": [first, second, message]} + + def summary_delta(output_index: int, item_id: str, text: str) -> dict[str, object]: + return { + "type": "response.reasoning_summary_text.delta", + "output_index": output_index, + "item_id": item_id, + "summary_index": 0, + "delta": text, + } + + events: Final = ( + {"type": "response.created", "response": response}, + {"type": "response.output_item.added", "output_index": 0, "item": _reasoning_item("rs_1", "")}, + summary_delta(0, "rs_1", "Oslo is further north."), + {"type": "response.output_item.done", "output_index": 0, "item": first}, + {"type": "response.output_item.added", "output_index": 1, "item": _reasoning_item("rs_2", "")}, + summary_delta(1, "rs_2", "Check the forecast."), + {"type": "response.output_item.done", "output_index": 1, "item": second}, + {"type": "response.output_item.added", "output_index": 2, "item": {**message, "content": []}}, + { + "type": "response.output_text.delta", + "output_index": 2, + "item_id": "msg_two_summaries", + "content_index": 0, + "delta": "Oslo", + }, + {"type": "response.output_item.done", "output_index": 2, "item": message}, + {"type": "response.completed", "response": completed}, + ) + respx_mock.post(url__regex=_RETRY_URL).mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) + ) + + stream: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + stream=True, + messages=[{"role": "user", "content": "Which city is colder?"}], + ) + frames: Final = b"".join([chunk async for chunk in stream]).decode() + + assert _streamed_blocks(frames) == [ + ("thinking", "Oslo is further north.", encrypted_reasoning_signature("gAAA_first")), + ("thinking", "Check the forecast.", encrypted_reasoning_signature("gAAA_second")), + ("text", "", None), + ] From f86222be48c20554817f8a35e1e83cd3216a0854 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:23:15 -0800 Subject: [PATCH 10/28] fix(anthropic): stream reasoning signatures only to Messages clients, keep xAI web search reasoning, and retry only when the sent request carried it --- .../handler.py | 10 ++ .../transformation.py | 15 ++- .../pass_through/adapters/handler.py | 25 ++++- .../pass_through/adapters/transformation.py | 4 + litellm/responses/bridge_check.py | 2 + ...responses_transformation_transformation.py | 104 +++++++++++++++++- ...al_pass_through_adapters_transformation.py | 56 ++++++++++ 7 files changed, 206 insertions(+), 10 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 3f7017eccc2..eef9cc6c4c9 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -13,7 +13,9 @@ from typing import ( from typing_extensions import TypedDict +from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import CallTypes if TYPE_CHECKING: from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse @@ -36,6 +38,10 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: return f"{custom_llm_provider}/{model}" +def _serves_anthropic_messages(logging_obj: "LiteLLMLoggingObj") -> bool: + return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value + + class ResponsesToCompletionBridgeHandler: def __init__(self): from .transformation import LiteLLMResponsesTransformationHandler @@ -182,6 +188,7 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] + serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -260,6 +267,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + streams_reasoning_signatures=serves_anthropic_messages, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -282,6 +290,7 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] + serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -363,6 +372,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + streams_reasoning_signatures=serves_anthropic_messages, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 296f1949d11..33ac222b06b 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1563,9 +1563,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + streams_reasoning_signatures: bool = False, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( - streaming_response, sync_stream, json_mode, structured_output=structured_output + streaming_response, + sync_stream, + json_mode, + structured_output=structured_output, + streams_reasoning_signatures=streams_reasoning_signatures, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -1944,8 +1949,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): sync_stream: bool, json_mode: bool | None = False, structured_output: bool = False, + streams_reasoning_signatures: bool = False, ): super().__init__(streaming_response, sync_stream, json_mode) + self._streams_reasoning_signatures: Final = streams_reasoning_signatures self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state @@ -2266,7 +2273,11 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) self._remember_served_service_tier(chunk) - reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk, self._summary_streamed_outputs) + reasoning_chunk: Final = ( + _reasoning_done_stream_chunk(chunk, self._summary_streamed_outputs) + if self._streams_reasoning_signatures + else None + ) translated: Final = ( self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 963dd050804..739077d3566 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -14,9 +14,12 @@ from typing_extensions import TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject -from litellm.litellm_core_utils.prompt_templates.common_utils import messages_carry_encrypted_reasoning +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + is_encrypted_reasoning_block, + thinking_blocks_with_encrypted_reasoning, +) from litellm.litellm_core_utils.tool_fields import ( ROUTES_WITHOUT_MCP_SERVERS, drops_mcp_servers, @@ -106,6 +109,22 @@ def _completion_text(completion_kwargs: _CompletionKwargs, key: str) -> str: return value if isinstance(value, str) else "" +def _message_carries_encrypted_reasoning(message: object) -> bool: + mapping: Final = as_str_mapping(message) + if mapping is None: + return False + blocks: Final = thinking_blocks_with_encrypted_reasoning( + mapping.get("thinking_blocks"), mapping.get("reasoning_items") + ) + return any(is_encrypted_reasoning_block(block) for block in blocks) + + +def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool: + messages: Final = completion_kwargs.get("messages") + sent: Final = iter_items(messages) if messages is not None else () + return any(_message_carries_encrypted_reasoning(message) for message in sent) + + def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, @@ -113,7 +132,7 @@ def _messages_without_rejected_reasoning( ) -> "_AnthropicMessages | None": error_message: Final = str(error) rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message - if not rejected or not messages_carry_encrypted_reasoning(messages): + if not rejected or not _sends_encrypted_reasoning(completion_kwargs): return None verbose_logger.warning( "Backend rejected replayed encrypted reasoning, retrying once without it. " diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index ee553749c74..0f0f7cd22f0 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -201,6 +201,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolParamFunctionChunk, ChatCompletionToolReferenceObject, ChatCompletionUserMessage, + OpenAIWebSearchOptions, ToolMessageContentPart, ) from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage @@ -228,6 +229,7 @@ class ChatRouteInputs: reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None thinking: Mapping[str, object] | None = None + web_search: bool = False _NO_ROUTE_INPUTS: Final = ChatRouteInputs() @@ -272,6 +274,7 @@ def _keeps_encrypted_reasoning_for_responses( tools=route.tools, reasoning_effort=route.reasoning_effort, api_base=route.api_base, + web_search_options=OpenAIWebSearchOptions() if route.web_search else None, ) @@ -1349,6 +1352,7 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, thinking=anthropic_message_request.get("thinking"), + web_search="web_search_options" in new_kwargs, ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index d8a9440bc57..4a25bc815ba 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -198,6 +198,7 @@ def chat_completion_ends_in_responses_api( tools: Sequence[Mapping[str, object]] | None = None, reasoning_effort: str | Mapping[str, object] | None = None, api_base: str | None = None, + web_search_options: OpenAIWebSearchOptions | None = None, ) -> bool: """Whether chat completion for this model is served by a Responses API. @@ -217,6 +218,7 @@ def chat_completion_ends_in_responses_api( model_info, _resolved = responses_api_bridge_check( model=resolved_model, custom_llm_provider=provider, + web_search_options=web_search_options, tools=tools, reasoning_effort=reasoning_effort, api_base=api_base, diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index d1155e41bdf..df9d048fb45 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3267,7 +3267,7 @@ def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keep }, } result: Final = OpenAiResponsesToChatCompletionStreamIterator( - streaming_response=None, sync_stream=True + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True ).chunk_parser(done) blocks: Final = result.choices[0].delta.thinking_blocks assert blocks is not None @@ -3291,6 +3291,96 @@ def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keep assert start["data"] == encrypted_reasoning_signature(encrypted) +_SIGNED_REASONING_STREAM_BODY: Final[Mapping[str, object]] = { + "id": "resp_signed_reasoning", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-6-sol", + "output": [ + {"type": "reasoning", "id": "rs_signed", "summary": [], "encrypted_content": "gAAAA_consumer"}, + { + "type": "message", + "id": "msg_logged_call_type", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + }, + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, +} + +_GET_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + + +def _signed_reasoning_sse() -> str: + events: Final = ( + { + "type": "response.created", + "response": {**_SIGNED_REASONING_STREAM_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + {"type": "response.output_item.done", "output_index": 0, "item": _SIGNED_REASONING_STREAM_BODY["output"][0]}, + { + "type": "response.output_text.delta", + "item_id": "msg_logged_call_type", + "output_index": 1, + "content_index": 0, + "delta": "pong", + }, + {"type": "response.completed", "response": _SIGNED_REASONING_STREAM_BODY}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +_AZURE_BRIDGE_KWARGS: Final[Mapping[str, object]] = { + "api_base": "https://example.invalid", + "api_version": "2025-04-01-preview", + "api_key": "test-key", +} + + +@pytest.mark.asyncio +async def test_streamed_reasoning_signature_reaches_messages_clients_and_never_chat_completions_deltas( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://example.invalid/openai/responses").mock( + side_effect=lambda _request: httpx.Response( + 200, text=_signed_reasoning_sse(), headers={"content-type": "text/event-stream"} + ) + ) + chat_stream: Final = await litellm.acompletion( + model="azure/gpt-6-sol", + messages=[{"role": "user", "content": "ping"}], + tools=[_GET_WEATHER_TOOL], + stream=True, + **_AZURE_BRIDGE_KWARGS, + ) + chat_deltas: Final = [chunk.choices[0].delta async for chunk in chat_stream if chunk.choices] + messages_stream: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-6-sol", + messages=[{"role": "user", "content": "ping"}], + tools=[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + max_tokens=64, + stream=True, + **_AZURE_BRIDGE_KWARGS, + ) + sse: Final = b"".join([chunk async for chunk in messages_stream]).decode() + + assert [getattr(delta, "thinking_blocks", None) for delta in chat_deltas if getattr(delta, "thinking_blocks", None)] == [] + assert "".join(delta.content or "" for delta in chat_deltas) == "pong" + messages_events: Final = [json.loads(line[len("data: ") :]) for line in sse.splitlines() if line.startswith("data: ")] + started: Final = [event["content_block"] for event in messages_events if event.get("type") == "content_block_start"] + assert [block["type"] for block in started] == ["redacted_thinking", "text"] + assert started[0]["data"] == encrypted_reasoning_signature("gAAAA_consumer") + + def test_reasoning_output_item_done_with_summary_signs_the_thinking_text(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( OpenAiResponsesToChatCompletionStreamIterator, @@ -3309,7 +3399,7 @@ def test_reasoning_output_item_done_with_summary_signs_the_thinking_text(): }, } result: Final = OpenAiResponsesToChatCompletionStreamIterator( - streaming_response=None, sync_stream=True + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True ).chunk_parser(done) block: Final = result.choices[0].delta.thinking_blocks[0] assert block["type"] == "thinking" @@ -3343,7 +3433,9 @@ def test_streamed_reasoning_summary_reaches_messages_clients_once_with_its_signa }, {"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."}, ) - parser: Final = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + parser: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ) chunks: Final = ( *(parser.chunk_parser(event) for event in events), ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), @@ -3371,7 +3463,9 @@ def test_reasoning_item_repeated_at_stream_end_reaches_messages_clients_once(): {"type": "response.output_item.done", "output_index": 0, "item": reasoning_item}, {"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."}, ) - parser: Final = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + parser: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ) chunks: Final = ( *(parser.chunk_parser(event) for event in events), ModelResponseStream( @@ -3401,7 +3495,7 @@ def test_reasoning_output_item_done_without_encrypted_content_stays_empty(): }, } result: Final = OpenAiResponsesToChatCompletionStreamIterator( - streaming_response=None, sync_stream=True + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True ).chunk_parser(done) assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, []) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index cda53b82931..ce7ff6d6eb0 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5585,6 +5585,62 @@ async def test_messages_do_not_retry_other_rejections( assert route.call_count == 1 +_STRIPPED_CHAT_MODEL: Final = "azure/gpt-5.6" +_STRIPPED_CHAT_API_BASE: Final = "https://stripped-reasoning.openai.azure.com" +_STRIPPED_CHAT_URL: Final = r".*stripped-reasoning\.openai\.azure\.com/openai/deployments/gpt-5\.6/chat/completions.*" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_messages_do_not_retry_when_the_translated_request_already_dropped_the_reasoning( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_STRIPPED_CHAT_URL).mock(return_value=_RETRY_REJECTION) + + with pytest.raises(litellm.BadRequestError): + await litellm.anthropic.messages.acreate( + model=_STRIPPED_CHAT_MODEL, + api_base=_STRIPPED_CHAT_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_history_with_foreign_reasoning(), + ) + + assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_xai_web_search_turn_replays_the_reasoning_its_responses_route_minted( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=r".*xai-search\.example/v1/responses").mock( + return_value=httpx.Response(200, json=_RETRY_REPLY) + ) + + await litellm.anthropic.messages.acreate( + model="xai/grok-4.7", + api_base="https://xai-search.example/v1", + api_key="fake-key", + max_tokens=2048, + tools=[{"type": "web_search_20250305", "name": "web_search"}], + messages=[ + {"role": "user", "content": "Which city is colder today?"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")}, + {"type": "text", "text": "Oslo."}, + ], + }, + {"role": "user", "content": "And by how much?"}, + ], + ) + + assert route.call_count == 1 + assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] + + @pytest.mark.asyncio @pytest.mark.usefixtures("_chat_bridge_httpx") async def test_openai_thinking_on_the_chat_adapter_keeps_reasoning_its_responses_route_replays( From d6625245b49a03345a2bea9cd8d5719835a268db Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:53:19 -0800 Subject: [PATCH 11/28] fix(anthropic): send openai/ Messages turns again once without encrypted reasoning a backend rejects The openai/ Responses path on /v1/messages had no retry, so a chatgpt/ block left in history answered 400 on every later openai/ turn. That path now sends the turn once more without encrypted reasoning, the same way the chat bridge does. The retry matches only invalid_encrypted_content or a "could not be verified" message naming the content, so the exception mapper's own hint no longer triggers it. The xAI web search keep path is backed out, since xAI sends no encrypted reasoning over the bridge. Backport of 15e60170124 from #49076 to lts/1.108.x, ported by hand onto the pass_through modules. Co-authored-by: DDT <141296842+Hugo-DDT@users.noreply.github.com> --- .../pass_through/adapters/handler.py | 51 ++++-- .../pass_through/adapters/transformation.py | 4 - .../responses_adapters/handler.py | 152 +++++++++++++----- litellm/responses/bridge_check.py | 2 - ...al_pass_through_adapters_transformation.py | 39 +---- .../test_responses_adapters_handler.py | 141 +++++++++++++++- 6 files changed, 294 insertions(+), 95 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 739077d3566..9cd08aca1d6 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -1,3 +1,4 @@ +import re from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial @@ -77,6 +78,7 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object): _ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]]) +_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE) _PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]" @@ -125,24 +127,49 @@ def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool: return any(_message_carries_encrypted_reasoning(message) for message in sent) +def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool: + error_message: Final = str(error) + return ( + "invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None + ) + + +def messages_without_rejected_reasoning( + messages: Sequence[Mapping[str, object]], + error: litellm.BadRequestError, + *, + sent_encrypted_reasoning: bool, + provider: str, + model: str, +) -> "_AnthropicMessages | None": + if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error): + return None + verbose_logger.warning( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " + "provider=%s model=%s status=%s code=%s", + provider, + model, + error.status_code, + _logged_rejection_code(error), + ) + return _ANTHROPIC_MESSAGES.validate_python( + strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages]) + ) + + def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, completion_kwargs: _CompletionKwargs, ) -> "_AnthropicMessages | None": - error_message: Final = str(error) - rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message - if not rejected or not _sends_encrypted_reasoning(completion_kwargs): - return None - verbose_logger.warning( - "Backend rejected replayed encrypted reasoning, retrying once without it. " - "provider=%s model=%s status=%s code=%s", - _completion_text(completion_kwargs, "custom_llm_provider"), - _completion_text(completion_kwargs, "model"), - error.status_code, - _logged_rejection_code(error), + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=_sends_encrypted_reasoning(completion_kwargs), + provider=_completion_text(completion_kwargs, "custom_llm_provider"), + model=_completion_text(completion_kwargs, "model"), ) - return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)) async def _acompletion_without_rejected_reasoning( diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 0f0f7cd22f0..ee553749c74 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -201,7 +201,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolParamFunctionChunk, ChatCompletionToolReferenceObject, ChatCompletionUserMessage, - OpenAIWebSearchOptions, ToolMessageContentPart, ) from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage @@ -229,7 +228,6 @@ class ChatRouteInputs: reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None thinking: Mapping[str, object] | None = None - web_search: bool = False _NO_ROUTE_INPUTS: Final = ChatRouteInputs() @@ -274,7 +272,6 @@ def _keeps_encrypted_reasoning_for_responses( tools=route.tools, reasoning_effort=route.reasoning_effort, api_base=route.api_base, - web_search_options=OpenAIWebSearchOptions() if route.web_search else None, ) @@ -1352,7 +1349,6 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, thinking=anthropic_message_request.get("thinking"), - web_search="web_search_options" in new_kwargs, ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 3eeaee8ad43..b705216eed4 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -14,9 +14,10 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items, normalize_drop_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( AllAnthropicMessageValues, @@ -31,7 +32,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( from litellm.types.llms.openai import ResponsesAPIResponse from litellm.utils import ProviderConfigManager -from ..adapters.handler import request_safeguards_evaluator +from ..adapters.handler import messages_without_rejected_reasoning, request_safeguards_evaluator from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -254,6 +255,69 @@ def _build_responses_kwargs( return responses_kwargs +_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]] +_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator +_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult] + + +def _is_encrypted_reasoning_item(item: object) -> bool: + mapping: Final = as_str_mapping(item) + return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content")) + + +def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool: + input_items: Final = responses_kwargs.get("input") + sent: Final = iter_items(input_items) if input_items is not None else () + return any(map(_is_encrypted_reasoning_item, sent)) + + +def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str: + value: Final = responses_kwargs.get(key) + return value if isinstance(value, str) else "" + + +def _messages_without_rejected_reasoning( + messages: AnthropicRequestMessages, + error: litellm.BadRequestError, + responses_kwargs: Mapping[str, object], +) -> AnthropicRequestMessages | None: + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=_sends_encrypted_reasoning(responses_kwargs), + provider=_kwarg_text(responses_kwargs, "custom_llm_provider"), + model=_kwarg_text(responses_kwargs, "model"), + ) + + +async def _aresponses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _ResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + try: + return responses_kwargs, await litellm.aresponses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, await litellm.aresponses(**replay_kwargs) + + +def _responses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _SyncResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + try: + return responses_kwargs, litellm.responses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, litellm.responses(**replay_kwargs) + + class LiteLLMMessagesToResponsesAPIHandler: """ Handles Anthropic /v1/messages requests for OpenAI / Azure models by @@ -281,28 +345,30 @@ class LiteLLMMessagesToResponsesAPIHandler: output_format: AnthropicOutputSchema | None = None, **kwargs: object, ) -> AnthropicMessagesResponse | AsyncIterator[bytes]: - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=messages, - model=model, - context_management=context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) + + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) - result: Final = await litellm.aresponses(**responses_kwargs) + responses_kwargs, result = await _aresponses_without_rejected_reasoning(messages, responses_kwargs_for) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( @@ -366,28 +432,28 @@ class LiteLLMMessagesToResponsesAPIHandler: **kwargs, ) - # Sync path - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=messages, - model=model, - context_management=context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) - result: Final = litellm.responses(**responses_kwargs) + responses_kwargs, result = _responses_without_rejected_reasoning(messages, responses_kwargs_for) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 4a25bc815ba..d8a9440bc57 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -198,7 +198,6 @@ def chat_completion_ends_in_responses_api( tools: Sequence[Mapping[str, object]] | None = None, reasoning_effort: str | Mapping[str, object] | None = None, api_base: str | None = None, - web_search_options: OpenAIWebSearchOptions | None = None, ) -> bool: """Whether chat completion for this model is served by a Responses API. @@ -218,7 +217,6 @@ def chat_completion_ends_in_responses_api( model_info, _resolved = responses_api_bridge_check( model=resolved_model, custom_llm_provider=provider, - web_search_options=web_search_options, tools=tools, reasoning_effort=reasoning_effort, api_base=api_base, diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index ce7ff6d6eb0..595f9bb9837 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5449,6 +5449,9 @@ _RETRY_REJECTION: Final = httpx.Response( _UNRELATED_REJECTION: Final = httpx.Response( 400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}} ) +_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response( + 400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}} +) _RETRY_REPLY: Final = { "id": "resp_retry", "object": "response", @@ -5523,6 +5526,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] assert retry_warnings == ( "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " "provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content", ) assert all( @@ -5564,6 +5568,9 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( id="no_encrypted_reasoning_to_drop", ), pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"), + pytest.param( + _history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection" + ), ], ) async def test_messages_do_not_retry_other_rejections( @@ -5609,38 +5616,6 @@ async def test_messages_do_not_retry_when_the_translated_request_already_dropped assert route.call_count == 1 -@pytest.mark.asyncio -@pytest.mark.usefixtures("_chat_bridge_httpx") -async def test_xai_web_search_turn_replays_the_reasoning_its_responses_route_minted( - respx_mock: respx.MockRouter, -) -> None: - route: Final = respx_mock.post(url__regex=r".*xai-search\.example/v1/responses").mock( - return_value=httpx.Response(200, json=_RETRY_REPLY) - ) - - await litellm.anthropic.messages.acreate( - model="xai/grok-4.7", - api_base="https://xai-search.example/v1", - api_key="fake-key", - max_tokens=2048, - tools=[{"type": "web_search_20250305", "name": "web_search"}], - messages=[ - {"role": "user", "content": "Which city is colder today?"}, - { - "role": "assistant", - "content": [ - {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")}, - {"type": "text", "text": "Oslo."}, - ], - }, - {"role": "user", "content": "And by how much?"}, - ], - ) - - assert route.call_count == 1 - assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] - - @pytest.mark.asyncio @pytest.mark.usefixtures("_chat_bridge_httpx") async def test_openai_thinking_on_the_chat_adapter_keeps_reasoning_its_responses_route_replays( diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 6aec78bdb97..042f99be001 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -2,10 +2,11 @@ import datetime import json import os import sys -from collections.abc import Mapping -from typing import Final +from collections.abc import AsyncIterator, Mapping +from typing import Final, cast from unittest.mock import AsyncMock, patch +import httpx import pytest import respx @@ -15,6 +16,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, @@ -684,3 +686,138 @@ def test_drop_params_drops_the_mcp_toolset_tools_of_dropped_mcp_servers(monkeypa ) assert [tool["name"] for tool in responses_kwargs["tools"]] == ["get_weather"] + +TEXT_RESPONSE: Final = { + "id": "resp_replayed", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6-luna", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} + +REJECTED_REPLAYED_REASONING: Final = { + "error": { + "message": "The encrypted content gAAA_foreign could not be verified.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_encrypted_content", + } +} +UNVERIFIED_ORGANIZATION: Final = { + "error": {"message": "The organization could not be verified.", "type": "invalid_request_error"} +} + + +def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]: + reasoning: Final = ( + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},) + if replayed_reasoning + else () + ) + return [ + {"role": "user", "content": "List the logs."}, + {"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]}, + ] + + +def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]: + return [ + sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"]) + for call in upstream.calls + ] + + +def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool): + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + served: Final = ( + httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + if stream + else httpx.Response(200, json=TEXT_RESPONSE) + ) + return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream) + + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + ) + events: Final = ( + [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)] + if stream + else [cast(Mapping[str, object], response)] + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert events[-1]["type"] in ("message", "message_stop") + + +def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "replayed_reasoning, rejection", + [ + pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"), + pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"), + ], +) +async def test_openai_turn_is_not_sent_again_for_other_rejections( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + replayed_reasoning: bool, + rejection: Mapping[str, object], +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False) + + with pytest.raises(litellm.BadRequestError): + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=replayed_reasoning), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert upstream.call_count == 1 From ce35f27bfaecc0b513b0cd5921b095d80b406a83 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 04:08:08 -0800 Subject: [PATCH 12/28] fix(anthropic): keep every non-streamed reasoning item for Messages clients and read a deployment's reasoning_summary in the keep check Backport of #49076 to lts/1.108.x. Ported from cadfe1d1798 (cursor/chatgpt-encrypted-reasoning-messages-1723) onto the line's pass_through/adapters layout. --- .../handler.py | 18 +++-- .../transformation.py | 40 +++++++---- .../pass_through/adapters/handler.py | 2 + .../pass_through/adapters/transformation.py | 12 +++- litellm/responses/bridge_check.py | 2 + ...responses_transformation_transformation.py | 66 +++++++++++++++++++ ...al_pass_through_adapters_transformation.py | 28 ++++++++ 7 files changed, 144 insertions(+), 24 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index eef9cc6c4c9..e824d9bcfc1 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -13,9 +13,7 @@ from typing import ( from typing_extensions import TypedDict -from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import CallTypes if TYPE_CHECKING: from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse @@ -38,10 +36,6 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: return f"{custom_llm_provider}/{model}" -def _serves_anthropic_messages(logging_obj: "LiteLLMLoggingObj") -> bool: - return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value - - class ResponsesToCompletionBridgeHandler: def __init__(self): from .transformation import LiteLLMResponsesTransformationHandler @@ -188,7 +182,9 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] - serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj) + from .transformation import serves_anthropic_messages + + streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -267,7 +263,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - streams_reasoning_signatures=serves_anthropic_messages, + streams_reasoning_signatures=streams_reasoning_signatures, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -290,7 +286,9 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] - serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj) + from .transformation import serves_anthropic_messages + + streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -372,7 +370,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - streams_reasoning_signatures=serves_anthropic_messages, + streams_reasoning_signatures=streams_reasoning_signatures, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 33ac222b06b..cf8d845bf26 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,7 +26,7 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, @@ -58,7 +58,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices +from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam, ResponseOutputItem @@ -227,6 +227,10 @@ class _BuiltReasoningItem(TypedDict): summary: Sequence[_ReasoningSummaryText] +def serves_anthropic_messages(logging_obj: object) -> bool: + return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value + + def _get_reasoning_items( msg: "AllMessageValues", ) -> list[ChatCompletionReasoningItem]: @@ -549,7 +553,7 @@ def _fold_choices_into_one( choices: Sequence["Choices"], tool_calls: Sequence[Mapping[str, object]], trailing_reasoning_content: str | None, - trailing_reasoning_item: _BuiltReasoningItem | None, + trailing_reasoning_items: Sequence[_BuiltReasoningItem], tool_search_items: Sequence[Mapping[str, object]], structured_output: bool = False, ) -> "Choices": @@ -568,7 +572,7 @@ def _fold_choices_into_one( reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content) reasoning_items: Final = ( *chain.from_iterable(_message_reasoning_items(m) for m in messages), - *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), + *trailing_reasoning_items, ) return Choices( message=Message( @@ -1222,6 +1226,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items: Sequence[object], handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None, structured_output: bool = False, + keeps_every_reasoning_item: bool = False, ) -> list["Choices"]: """ Convert Responses API output items to chat completion choices. @@ -1231,6 +1236,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): handle_raw_dict_callback: Optional callback for handling raw dict items structured_output: The chat request carried a json response_format, so only the first message item becomes content + keeps_every_reasoning_item: Keep each reasoning item before an answer, not only the last Returns: List of Choices objects @@ -1253,7 +1259,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices: Final[list[Choices]] = [] index = 0 reasoning_content: str | None = None - pending_reasoning_item: _BuiltReasoningItem | None = None + pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = () # Collect all tool calls to put them in a single choice # (Chat Completions API expects all tool calls in one message) @@ -1262,12 +1268,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - pending_reasoning_item = _build_reasoning_item( - item_id=item.id, - encrypted_content=getattr(item, "encrypted_content", None), - summary_raw=item.summary, + pending_reasoning_items = ( + *(pending_reasoning_items if keeps_every_reasoning_item else ()), + _build_reasoning_item( + item_id=item.id, + encrypted_content=getattr(item, "encrypted_content", None), + summary_raw=item.summary, + ), + ) + reasoning_content = " ".join( + s["text"] + for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items) + if s.get("text") ) - reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text")) elif isinstance(item, ResponseOutputMessage): for content in item.content: @@ -1284,7 +1297,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotations=annotations, reasoning_items=cast( list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), + (list(pending_reasoning_items) or None), ), ) @@ -1297,7 +1310,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) reasoning_content = None # flush - pending_reasoning_item = None # flush + pending_reasoning_items = () # flush index += 1 elif isinstance(item, ResponseFunctionToolCall): @@ -1362,7 +1375,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices=choices, tool_calls=accumulated_tool_calls, trailing_reasoning_content=reasoning_content, - trailing_reasoning_item=pending_reasoning_item, + trailing_reasoning_items=pending_reasoning_items, tool_search_items=tool_search_items, structured_output=structured_output, ) @@ -1500,6 +1513,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + keeps_every_reasoning_item=serves_anthropic_messages(logging_obj), ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 9cd08aca1d6..4f21c7dbe5e 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -17,6 +17,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_encrypted_reasoning_block, thinking_blocks_with_encrypted_reasoning, @@ -760,6 +761,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data, custom_llm_provider=target_provider, api_base=api_base if isinstance(api_base, str) else None, + reasoning_summary=peek_reasoning_summary_aliases(provider_kwargs), ) if openai_request is None: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index ee553749c74..17e50498d46 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -228,6 +228,7 @@ class ChatRouteInputs: reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None thinking: Mapping[str, object] | None = None + reasoning_summary: object | None = None _NO_ROUTE_INPUTS: Final = ChatRouteInputs() @@ -271,6 +272,7 @@ def _keeps_encrypted_reasoning_for_responses( custom_llm_provider, tools=route.tools, reasoning_effort=route.reasoning_effort, + reasoning_summary=route.reasoning_summary, api_base=route.api_base, ) @@ -313,7 +315,12 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None + self, + kwargs, + *, + custom_llm_provider: str | None = None, + api_base: str | None = None, + reasoning_summary: object | None = None, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -348,6 +355,7 @@ class AnthropicAdapter: anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, api_base=api_base, + reasoning_summary=reasoning_summary, ) return translated_body, tool_name_mapping @@ -1295,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, api_base: str | None = None, + reasoning_summary: object | None = None, preserve_midturn_system: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ @@ -1349,6 +1358,7 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, thinking=anthropic_message_request.get("thinking"), + reasoning_summary=reasoning_summary, ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index d8a9440bc57..6b80132a077 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -197,6 +197,7 @@ def chat_completion_ends_in_responses_api( *, tools: Sequence[Mapping[str, object]] | None = None, reasoning_effort: str | Mapping[str, object] | None = None, + reasoning_summary: object | None = None, api_base: str | None = None, ) -> bool: """Whether chat completion for this model is served by a Responses API. @@ -219,6 +220,7 @@ def chat_completion_ends_in_responses_api( custom_llm_provider=provider, tools=tools, reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, api_base=api_base, ) return model_info.get("mode") == "responses" diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index df9d048fb45..cb4cbfc1c57 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3570,6 +3570,72 @@ def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only() assert content[0]["data"] == encrypted_reasoning_signature(encrypted) +@pytest.mark.parametrize( + ("call_type", "kept_ids"), + [ + pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"), + pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"), + ], +) +def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item( + call_type: str, kept_ids: list[str] +) -> None: + from types import SimpleNamespace + from unittest.mock import Mock + + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse, Usage + + def reasoning(item_id: str) -> ResponseReasoningItem: + return ResponseReasoningItem( + id=item_id, summary=[], type="reasoning", content=None, encrypted_content=f"gAAAA_{item_id}", status=None + ) + + def call(index: int, city: str) -> ResponseFunctionToolCall: + return ResponseFunctionToolCall( + id=f"fc_{index}", + type="function_call", + status="completed", + arguments=json.dumps({"city": city}), + call_id=f"call_{index}", + name="get_weather", + ) + + raw: Final = ResponsesAPIResponse.model_construct( + id="resp_two_reasoning", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-5.4-nano", + output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")], + status="completed", + usage=None, + ) + result: Final = LiteLLMResponsesTransformationHandler().transform_response( + model="gpt-5.4-nano", + raw_response=raw, + model_response=ModelResponse( + id="chatcmpl-two-reasoning", + created=1, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ), + logging_obj=SimpleNamespace(call_type=call_type), + request_data={"model": "gpt-5.4-nano"}, + messages=[{"role": "user", "content": "Oslo or Lyon?"}], + optional_params={}, + litellm_params={"custom_llm_provider": "azure"}, + encoding=Mock(), + ) + + message: Final = result.choices[0].message + assert [item["id"] for item in message.reasoning_items] == kept_ids + assert [tool_call.function.name for tool_call in message.tool_calls] == ["get_weather", "get_weather"] + + def test_streaming_function_call_tool_id_for_degenerate_call_id(): """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` (``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 595f9bb9837..1223185a05c 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -698,6 +698,34 @@ def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_response assert bool(assistant.get("thinking_blocks")) is kept +@pytest.mark.parametrize( + ("deployment_params", "kept"), + [ + pytest.param({"reasoning_summary": "auto"}, True, id="deployment_summary_bridges_to_responses"), + pytest.param({"extra_body": {"reasoningSummary": "auto"}}, True, id="extra_body_summary_bridges_to_responses"), + pytest.param({}, False, id="no_summary_stays_on_chat"), + ], +) +def test_encrypted_reasoning_is_kept_when_the_deployment_reasoning_summary_routes_it_to_responses( + deployment_params: dict[str, object], kept: bool +) -> None: + from litellm.llms.anthropic.pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + request: Final = _replayed_reasoning_request([]) + completion_kwargs, _mapping = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=cast(list[dict[str, object]], request["messages"]), + model="azure/gpt-5.1", + thinking={"type": "enabled", "budget_tokens": 2048}, + extra_kwargs={"api_base": "https://keep-check.openai.azure.com", **deployment_params}, + ) + + assistant: Final = cast(list[Mapping[str, object]], completion_kwargs["messages"])[1] + assert bool(assistant.get("thinking_blocks")) is kept + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. From 97df7cb369397dc9326fcc59d3be492b79707fe4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 06:06:59 -0800 Subject: [PATCH 13/28] fix(anthropic): keep every non-streamed reasoning item for Messages clients on the line's bridge This line's transform_request rewrites the logging call_type to responses, so the non-streamed reply read the Messages-client check after the rewrite and kept only the last reasoning item. The bridge handler now reads it before the request transform and passes it to transform_response, as the streamed path already does. --- .../handler.py | 12 ++- .../transformation.py | 3 +- ...itellm_responses_transformation_handler.py | 98 +++++++++++++++++++ ...responses_transformation_transformation.py | 12 +-- 4 files changed, 114 insertions(+), 11 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index e824d9bcfc1..c47bb046021 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -184,7 +184,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] from .transformation import serves_anthropic_messages - streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) + serves_messages_client: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -225,6 +225,7 @@ class ResponsesToCompletionBridgeHandler: encoding=validated_kwargs["encoding"], api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), + keeps_every_reasoning_item=serves_messages_client, ) elif isinstance(result, ModelResponse): if not stream: @@ -250,6 +251,7 @@ class ResponsesToCompletionBridgeHandler: encoding=validated_kwargs["encoding"], api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), + keeps_every_reasoning_item=serves_messages_client, ) else: if self._is_preformatted_cached_chat_stream(result): @@ -263,7 +265,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - streams_reasoning_signatures=streams_reasoning_signatures, + streams_reasoning_signatures=serves_messages_client, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -288,7 +290,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] from .transformation import serves_anthropic_messages - streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) + serves_messages_client: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -332,6 +334,7 @@ class ResponsesToCompletionBridgeHandler: encoding=validated_kwargs["encoding"], api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), + keeps_every_reasoning_item=serves_messages_client, ) elif isinstance(result, ModelResponse): if not stream: @@ -357,6 +360,7 @@ class ResponsesToCompletionBridgeHandler: encoding=validated_kwargs["encoding"], api_key=kwargs.get("api_key"), json_mode=kwargs.get("json_mode"), + keeps_every_reasoning_item=serves_messages_client, ) else: if self._is_preformatted_cached_chat_stream(result): @@ -370,7 +374,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - streams_reasoning_signatures=streams_reasoning_signatures, + streams_reasoning_signatures=serves_messages_client, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index cf8d845bf26..4c15f2cee4a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1482,6 +1482,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): encoding: object, api_key: str | None = None, json_mode: bool | None = None, + keeps_every_reasoning_item: bool = False, ) -> "ModelResponse": """Transform Responses API response to chat completion response""" from litellm.responses.utils import ResponseAPILoggingUtils @@ -1513,7 +1514,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - keeps_every_reasoning_item=serves_anthropic_messages(logging_obj), + keeps_every_reasoning_item=keeps_every_reasoning_item, ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index c5d7ca96a21..0899c86cfb7 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -335,3 +335,101 @@ async def test_acompletion_keeps_provider_native_model_id_through_responses( handed_model = responses_call.call_args.kwargs["model"] assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model + + +def _two_reasoning_tool_turn(): + from openai.types.responses import ResponseFunctionToolCall + from openai.types.responses.response_reasoning_item import ResponseReasoningItem + + from litellm.types.llms.openai import ResponsesAPIResponse + + def reasoning(item_id: str) -> ResponseReasoningItem: + return ResponseReasoningItem( + id=item_id, summary=[], type="reasoning", encrypted_content=f"gAAAA_{item_id}" + ) + + def call(index: int, city: str) -> ResponseFunctionToolCall: + return ResponseFunctionToolCall( + id=f"fc_{index}", + type="function_call", + status="completed", + arguments=f'{{"city": "{city}"}}', + call_id=f"call_{index}", + name="get_weather", + ) + + return ResponsesAPIResponse.model_construct( + id="resp_two_reasoning", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-5.4-nano", + output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")], + status="completed", + usage=None, + ) + + +class _CompletedStream: + def __init__(self, response): + self.completed_response = MagicMock(response=response) + self._hidden_params = {} + + def __iter__(self): + return iter(()) + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + +def _messages_client_kwargs(call_type: str): + messages = [{"role": "user", "content": "Weather in Oslo and Lyon?"}] + return { + "model": "gpt-5.4-nano", + "custom_llm_provider": "azure", + "messages": messages, + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + "model_response": ModelResponse(), + "logging_obj": LiteLLMLogging( + litellm_call_id="test-call", + call_type=call_type, + model="gpt-5.4-nano", + messages=messages, + function_id="fn-id", + stream=False, + start_time=datetime.now(), + ), + } + + +@pytest.mark.parametrize("is_async", [True, False], ids=["acompletion", "completion"]) +@pytest.mark.parametrize("arrives_streamed", [False, True], ids=["response", "collected_stream"]) +@pytest.mark.parametrize( + ("call_type", "kept_ids"), + [ + pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client"), + pytest.param("acompletion", ["rs_second"], id="chat_client"), + ], +) +@pytest.mark.asyncio +async def test_non_streamed_bridge_hands_messages_clients_every_reasoning_item( + is_async: bool, arrives_streamed: bool, call_type: str, kept_ids: list[str] +): + turn = _two_reasoning_tool_turn() + upstream = _CompletedStream(turn) if arrives_streamed else turn + bridge = ResponsesToCompletionBridgeHandler() + kwargs = _messages_client_kwargs(call_type) + + if is_async: + with patch("litellm.aresponses", new=AsyncMock(return_value=upstream)): + result = await bridge.acompletion(**kwargs) + else: + with patch("litellm.responses", return_value=upstream): + result = bridge.completion(**kwargs) + + assert [item["id"] for item in result.choices[0].message.reasoning_items] == kept_ids diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index cb4cbfc1c57..04505b717af 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3571,16 +3571,15 @@ def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only() @pytest.mark.parametrize( - ("call_type", "kept_ids"), + ("keeps_every_reasoning_item", "kept_ids"), [ - pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"), - pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"), + pytest.param(True, ["rs_first", "rs_second"], id="messages_client_keeps_every_item"), + pytest.param(False, ["rs_second"], id="chat_client_keeps_the_last_item"), ], ) def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item( - call_type: str, kept_ids: list[str] + keeps_every_reasoning_item: bool, kept_ids: list[str] ) -> None: - from types import SimpleNamespace from unittest.mock import Mock from litellm.types.llms.openai import ResponsesAPIResponse @@ -3623,12 +3622,13 @@ def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item( choices=[], usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), ), - logging_obj=SimpleNamespace(call_type=call_type), + logging_obj=Mock(), request_data={"model": "gpt-5.4-nano"}, messages=[{"role": "user", "content": "Oslo or Lyon?"}], optional_params={}, litellm_params={"custom_llm_provider": "azure"}, encoding=Mock(), + keeps_every_reasoning_item=keeps_every_reasoning_item, ) message: Final = result.choices[0].message From a1c02f7d1a24ba1e03d4ec9831dd471ef3ad18a9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 07:33:23 -0800 Subject: [PATCH 14/28] fix(anthropic): log only the outcome of a turn sent again, and keep every reasoning item in Messages logs on the line A /v1/messages turn whose replayed encrypted block the backend rejects is sent once more without it. The rejected first attempt no longer fires failure callbacks or releases its parallel slot and budget reservation, so only the turn sent again is logged and counted. Success logs for a Messages client now keep every reasoning item of a non-streamed turn, matching the reply the client got. --- litellm/_internal_context.py | 18 ++ litellm/litellm_core_utils/litellm_logging.py | 2 + .../pass_through/adapters/handler.py | 29 ++- .../responses_adapters/handler.py | 25 ++- litellm/utils.py | 6 +- ...itellm_responses_transformation_handler.py | 183 ++++++++++-------- ...al_pass_through_adapters_transformation.py | 84 +++++++- .../test_responses_adapters_handler.py | 139 +++++++++++++ 8 files changed, 384 insertions(+), 102 deletions(-) diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index df549701cf4..1b07c017a3c 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -36,6 +36,8 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families" _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None) +_resent_on: Final[ContextVar[Callable[[Exception], bool] | None]] = ContextVar("resent_on", default=None) + @contextmanager def post_response_phase() -> Generator[None]: @@ -113,6 +115,22 @@ def current_service_caller() -> str | None: return _service_caller.get() +@contextmanager +def resent_on(rejection: Callable[[Exception], bool]) -> Generator[None]: + """An attempt its caller sends again when it fails with ``rejection``. That failure is not the request's + outcome, so nothing logs it and its budget reservation and rate-limit slot stay with the attempt sent again.""" + token: Final = _resent_on.set(rejection) + try: + yield + finally: + _restore(_resent_on, token) + + +def is_resent_failure(error: Exception) -> bool: + rejection: Final = _resent_on.get() + return rejection is not None and rejection(error) + + @contextmanager def pinned_billing_time(moment: datetime) -> Generator[None]: """Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read.""" diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 95323c2ef5e..8cd8e55c292 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4593,6 +4593,7 @@ class Logging(LiteLLMLoggingBaseClass): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, responses_provider_metadata, + serves_anthropic_messages, ) served_id: Final = _provider_response_id(result) @@ -4607,6 +4608,7 @@ class Logging(LiteLLMLoggingBaseClass): optional_params={}, litellm_params={}, encoding=litellm.encoding, + keeps_every_reasoning_item=serves_anthropic_messages(self), ) translated.id = served_id or translated.id return translated diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 4f21c7dbe5e..a818d6f3565 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -1,5 +1,6 @@ import re from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from functools import partial from typing import ( @@ -13,6 +14,7 @@ from pydantic import TypeAdapter from typing_extensions import TypedDict import litellm +from litellm._internal_context import resent_on from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items @@ -135,6 +137,14 @@ def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool: ) +def _is_rejected_reasoning(error: Exception) -> bool: + return isinstance(error, litellm.BadRequestError) and _rejects_encrypted_reasoning(error) + + +def resent_without_rejected_reasoning(sent_encrypted_reasoning: bool) -> AbstractContextManager[None]: + return resent_on(_is_rejected_reasoning) if sent_encrypted_reasoning else nullcontext() + + def messages_without_rejected_reasoning( messages: Sequence[Mapping[str, object]], error: litellm.BadRequestError, @@ -163,11 +173,12 @@ def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, completion_kwargs: _CompletionKwargs, + sent_encrypted_reasoning: bool, ) -> "_AnthropicMessages | None": return messages_without_rejected_reasoning( messages, error, - sent_encrypted_reasoning=_sends_encrypted_reasoning(completion_kwargs), + sent_encrypted_reasoning=sent_encrypted_reasoning, provider=_completion_text(completion_kwargs, "custom_llm_provider"), model=_completion_text(completion_kwargs, "model"), ) @@ -178,10 +189,14 @@ async def _acompletion_without_rejected_reasoning( messages: _AnthropicMessages, prepare_replay: _PrepareReplay, ) -> "ChatResult": + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(completion_kwargs) try: - return await litellm.acompletion(**completion_kwargs) + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return await litellm.acompletion(**completion_kwargs) except litellm.BadRequestError as error: - replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, completion_kwargs) + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, completion_kwargs, sent_encrypted_reasoning + ) if replayed_messages is None: raise return await litellm.acompletion(**prepare_replay(replayed_messages)) @@ -192,10 +207,14 @@ def _completion_without_rejected_reasoning( messages: _AnthropicMessages, prepare_replay: _PrepareReplay, ) -> "ChatResult | Coroutine[object, object, ChatResult]": + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(completion_kwargs) try: - return litellm.completion(**completion_kwargs) + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return litellm.completion(**completion_kwargs) except litellm.BadRequestError as error: - replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, completion_kwargs) + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, completion_kwargs, sent_encrypted_reasoning + ) if replayed_messages is None: raise return litellm.completion(**prepare_replay(replayed_messages)) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index b705216eed4..abe0f2f419d 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -32,7 +32,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( from litellm.types.llms.openai import ResponsesAPIResponse from litellm.utils import ProviderConfigManager -from ..adapters.handler import messages_without_rejected_reasoning, request_safeguards_evaluator +from ..adapters.handler import ( + messages_without_rejected_reasoning, + request_safeguards_evaluator, + resent_without_rejected_reasoning, +) from ..safeguards import with_safeguard_results from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -280,11 +284,12 @@ def _messages_without_rejected_reasoning( messages: AnthropicRequestMessages, error: litellm.BadRequestError, responses_kwargs: Mapping[str, object], + sent_encrypted_reasoning: bool, ) -> AnthropicRequestMessages | None: return messages_without_rejected_reasoning( messages, error, - sent_encrypted_reasoning=_sends_encrypted_reasoning(responses_kwargs), + sent_encrypted_reasoning=sent_encrypted_reasoning, provider=_kwarg_text(responses_kwargs, "custom_llm_provider"), model=_kwarg_text(responses_kwargs, "model"), ) @@ -294,10 +299,14 @@ async def _aresponses_without_rejected_reasoning( messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor ) -> tuple[Mapping[str, object], _ResponsesResult]: responses_kwargs: Final = responses_kwargs_for(messages) + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(responses_kwargs) try: - return responses_kwargs, await litellm.aresponses(**responses_kwargs) + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return responses_kwargs, await litellm.aresponses(**responses_kwargs) except litellm.BadRequestError as error: - replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, responses_kwargs, sent_encrypted_reasoning + ) if replayed_messages is None: raise replay_kwargs: Final = responses_kwargs_for(replayed_messages) @@ -308,10 +317,14 @@ def _responses_without_rejected_reasoning( messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor ) -> tuple[Mapping[str, object], _SyncResponsesResult]: responses_kwargs: Final = responses_kwargs_for(messages) + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(responses_kwargs) try: - return responses_kwargs, litellm.responses(**responses_kwargs) + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return responses_kwargs, litellm.responses(**responses_kwargs) except litellm.BadRequestError as error: - replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, responses_kwargs, sent_encrypted_reasoning + ) if replayed_messages is None: raise replay_kwargs: Final = responses_kwargs_for(replayed_messages) diff --git a/litellm/utils.py b/litellm/utils.py index da619a5aefe..3213e343c4f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -51,7 +51,7 @@ import litellm.litellm_core_utils # audio_utils.utils is lazy-loaded - only imported when needed for transcription calls import litellm.litellm_core_utils.json_validation_rule -from litellm._internal_context import is_internal_call +from litellm._internal_context import is_internal_call, is_resent_failure from litellm._lazy_imports import ( _get_modified_max_tokens, get_default_encoding, @@ -2032,7 +2032,7 @@ def client(original_function): end_time = datetime.datetime.now() # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's deprecated - if logging_obj: + if logging_obj and not is_resent_failure(e): logging_obj.failure_handler( e, traceback_exception, start_time, end_time ) # DO NOT MAKE THREADED - router retry fallback relies on this! @@ -2279,7 +2279,7 @@ def client(original_function): # Reuse the timestamp taken right when the deployment call itself failed, before # the failure hook ran, so a slow callback doesn't inflate the reported duration. end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with - if logging_obj and not _is_litellm_internal_call: + if logging_obj and not _is_litellm_internal_call and not is_resent_failure(e): unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: logging_obj.failure_handler( diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index 0899c86cfb7..af0a0dbe9b0 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -1,7 +1,13 @@ +import json +import time from datetime import datetime +from pathlib import Path +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx import litellm @@ -11,6 +17,7 @@ from litellm.completion_extras.litellm_responses_transformation.handler import ( from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.types.router import GenericLiteLLMParams +from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.types.utils import ModelResponse @@ -337,99 +344,105 @@ async def test_acompletion_keeps_provider_native_model_id_through_responses( assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model -def _two_reasoning_tool_turn(): - from openai.types.responses import ResponseFunctionToolCall - from openai.types.responses.response_reasoning_item import ResponseReasoningItem - - from litellm.types.llms.openai import ResponsesAPIResponse - - def reasoning(item_id: str) -> ResponseReasoningItem: - return ResponseReasoningItem( - id=item_id, summary=[], type="reasoning", encrypted_content=f"gAAAA_{item_id}" - ) - - def call(index: int, city: str) -> ResponseFunctionToolCall: - return ResponseFunctionToolCall( - id=f"fc_{index}", - type="function_call", - status="completed", - arguments=f'{{"city": "{city}"}}', - call_id=f"call_{index}", - name="get_weather", - ) - - return ResponsesAPIResponse.model_construct( - id="resp_two_reasoning", - created_at=1, - error=None, - incomplete_details=None, - model="gpt-5.4-nano", - output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")], - status="completed", - usage=None, - ) +_CHATGPT_RESPONSES_URL: Final = "https://chatgpt.com/backend-api/codex/responses" -class _CompletedStream: - def __init__(self, response): - self.completed_response = MagicMock(response=response) - self._hidden_params = {} - - def __iter__(self): - return iter(()) - - def __aiter__(self): - return self - - async def __anext__(self): - raise StopAsyncIteration +def _reasoning_item(item_id: str) -> dict[str, object]: + return {"type": "reasoning", "id": item_id, "summary": [], "encrypted_content": f"gAAAA_{item_id}"} -def _messages_client_kwargs(call_type: str): - messages = [{"role": "user", "content": "Weather in Oslo and Lyon?"}] +def _weather_call(index: int, city: str) -> dict[str, object]: return { - "model": "gpt-5.4-nano", - "custom_llm_provider": "azure", - "messages": messages, - "optional_params": {}, - "litellm_params": {}, - "headers": {}, - "model_response": ModelResponse(), - "logging_obj": LiteLLMLogging( - litellm_call_id="test-call", - call_type=call_type, - model="gpt-5.4-nano", - messages=messages, - function_id="fn-id", - stream=False, - start_time=datetime.now(), - ), + "type": "function_call", + "id": f"fc_{index}", + "status": "completed", + "arguments": json.dumps({"city": city}), + "call_id": f"call_{index}", + "name": "get_weather", } -@pytest.mark.parametrize("is_async", [True, False], ids=["acompletion", "completion"]) -@pytest.mark.parametrize("arrives_streamed", [False, True], ids=["response", "collected_stream"]) -@pytest.mark.parametrize( - ("call_type", "kept_ids"), - [ - pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client"), - pytest.param("acompletion", ["rs_second"], id="chat_client"), - ], -) +def _two_reasoning_tool_turn_stream() -> httpx.Response: + turn: Final = { + "id": "resp_two_reasoning", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6-sol", + "output": [ + _reasoning_item("rs_first"), + _weather_call(1, "Oslo"), + _reasoning_item("rs_second"), + _weather_call(2, "Lyon"), + ], + "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, + } + body: Final = f"data: {json.dumps({'type': 'response.completed', 'response': turn})}\n\ndata: [DONE]\n\n" + return httpx.Response(200, headers={"content-type": "text/event-stream"}, text=body) + + +@pytest.fixture +def _chatgpt_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None: + auth: Final = {"access_token": "chatgpt-test-token", "expires_at": time.time() + 3600, "account_id": "acct_test"} + (tmp_path / "auth.json").write_text(json.dumps(auth)) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post(_CHATGPT_RESPONSES_URL).mock(side_effect=lambda _request: _two_reasoning_tool_turn_stream()) + + +_WEATHER_QUESTION: Final = "Weather in Oslo and Lyon?" +_WEATHER_TOOL_INPUT: Final = {"type": "object", "properties": {"city": {"type": "string"}}} +_EVERY_REASONING_SIGNATURE: Final = [ + encrypted_reasoning_signature("gAAAA_rs_first"), + encrypted_reasoning_signature("gAAAA_rs_second"), +] + + +def _thinking_signatures(reply: object) -> list[object]: + assert isinstance(reply, dict) + return [block.get("data") for block in reply["content"] if block["type"] == "redacted_thinking"] + + @pytest.mark.asyncio -async def test_non_streamed_bridge_hands_messages_clients_every_reasoning_item( - is_async: bool, arrives_streamed: bool, call_type: str, kept_ids: list[str] -): - turn = _two_reasoning_tool_turn() - upstream = _CompletedStream(turn) if arrives_streamed else turn - bridge = ResponsesToCompletionBridgeHandler() - kwargs = _messages_client_kwargs(call_type) +@pytest.mark.usefixtures("_chatgpt_backend") +async def test_async_messages_client_gets_every_reasoning_item_of_a_non_streamed_turn() -> None: + reply: Final = await litellm.anthropic.messages.acreate( + model="chatgpt/gpt-5.6-sol", + max_tokens=64, + messages=[{"role": "user", "content": _WEATHER_QUESTION}], + tools=[{"name": "get_weather", "input_schema": _WEATHER_TOOL_INPUT}], + ) - if is_async: - with patch("litellm.aresponses", new=AsyncMock(return_value=upstream)): - result = await bridge.acompletion(**kwargs) - else: - with patch("litellm.responses", return_value=upstream): - result = bridge.completion(**kwargs) + assert _thinking_signatures(reply) == _EVERY_REASONING_SIGNATURE - assert [item["id"] for item in result.choices[0].message.reasoning_items] == kept_ids + +_CHAT_WEATHER_TOOL: Final = {"type": "function", "function": {"name": "get_weather", "parameters": _WEATHER_TOOL_INPUT}} + + +def _reasoning_item_ids(response: object) -> list[object]: + assert isinstance(response, ModelResponse) + return [item["id"] for item in response.choices[0].message.reasoning_items] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chatgpt_backend") +async def test_async_chat_client_keeps_only_the_last_reasoning_item_of_a_turn() -> None: + response: Final = await litellm.acompletion( + model="chatgpt/gpt-5.6-sol", + messages=[{"role": "user", "content": _WEATHER_QUESTION}], + tools=[_CHAT_WEATHER_TOOL], + ) + + assert _reasoning_item_ids(response) == ["rs_second"] + + +@pytest.mark.usefixtures("_chatgpt_backend") +def test_sync_chat_client_keeps_only_the_last_reasoning_item_of_a_turn() -> None: + response: Final = litellm.completion( + model="chatgpt/gpt-5.6-sol", + messages=[{"role": "user", "content": _WEATHER_QUESTION}], + tools=[_CHAT_WEATHER_TOOL], + ) + + assert _reasoning_item_ids(response) == ["rs_second"] diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1223185a05c..0463d37c4a8 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -3,6 +3,7 @@ import functools import json import logging from collections.abc import Mapping, Sequence +from contextlib import nullcontext from types import MappingProxyType from typing import Any, Final, cast @@ -11,9 +12,7 @@ import pytest import respx import litellm - - - +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, encrypted_reasoning_signature, @@ -5585,6 +5584,85 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] +class _FailureLog(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.rejections: tuple[str, ...] = () + + def _record(self, kwargs) -> None: + error: Final = str(kwargs.get("exception")) + self.rejections = ( + *self.rejections, + next((phrase for phrase in _UPSTREAM_REJECTION_PHRASES if phrase in error), error), + ) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._record(kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._record(kwargs) + + +_UPSTREAM_REJECTION_PHRASES: Final = ("could not be verified", "max_output_tokens is too small") +_OUTCOMES_OF_THE_TURN_SENT_AGAIN: Final = ( + pytest.param(httpx.Response(200, json=_RETRY_REPLY), (), id="served"), + pytest.param(_UNRELATED_REJECTION, ("max_output_tokens is too small",), id="rejected"), +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +async def test_async_messages_log_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + respx_mock.post(url__regex=_RETRY_URL).mock(side_effect=[_RETRY_REJECTION, sent_again]) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert failure_log.rejections == logged_rejections + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +def test_sync_messages_log_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + respx_mock.post(url__regex=_RETRY_URL).mock(side_effect=[_RETRY_REJECTION, sent_again]) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert failure_log.rejections == logged_rejections + + @pytest.mark.asyncio @pytest.mark.usefixtures("_chat_bridge_httpx") @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 042f99be001..c11023b256a 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -1,8 +1,10 @@ +import asyncio import datetime import json import os import sys from collections.abc import AsyncIterator, Mapping +from contextlib import nullcontext from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -13,6 +15,8 @@ import respx sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, @@ -795,6 +799,85 @@ def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_back assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] +class _FailureLog(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.rejections: tuple[str, ...] = () + + def _record(self, kwargs) -> None: + error: Final = str(kwargs.get("exception")) + self.rejections = ( + *self.rejections, + next((phrase for phrase in ("encrypted content", "organization") if phrase in error), error), + ) + + def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._record(kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._record(kwargs) + + +_OUTCOMES_OF_THE_TURN_SENT_AGAIN: Final = ( + pytest.param(httpx.Response(200, json=TEXT_RESPONSE), (), id="served"), + pytest.param(httpx.Response(400, json=UNVERIFIED_ORGANIZATION), ("organization",), id="rejected"), +) + + +def _rejected_then(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sent_again: httpx.Response) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[httpx.Response(400, json=REJECTED_REPLAYED_REASONING), sent_again] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +async def test_openai_turn_logs_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + _rejected_then(respx_mock, monkeypatch, sent_again) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + ) + + assert failure_log.rejections == logged_rejections + + +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +def test_sync_openai_turn_logs_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + _rejected_then(respx_mock, monkeypatch, sent_again) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + ) + + assert failure_log.rejections == logged_rejections + + @pytest.mark.asyncio @pytest.mark.parametrize( "replayed_reasoning, rejection", @@ -821,3 +904,59 @@ async def test_openai_turn_is_not_sent_again_for_other_rejections( ) assert upstream.call_count == 1 + + +class _SuccessLog(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged_reasoning_item_ids: tuple[tuple[object, ...], ...] = () + self.logged: Final = asyncio.Event() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + logged: Final = kwargs["standard_logging_object"]["response"] + items: Final = logged["choices"][0]["message"].get("reasoning_items") or () + self.logged_reasoning_item_ids = (*self.logged_reasoning_item_ids, tuple(item["id"] for item in items)) + self.logged.set() + + +def _reasoning_before_call(index: int) -> tuple[dict[str, object], dict[str, object]]: + return ( + {"type": "reasoning", "id": f"rs_{index}", "summary": [], "encrypted_content": f"gAAA_rs_{index}"}, + { + "type": "function_call", + "id": f"fc_{index}", + "status": "completed", + "arguments": "{}", + "call_id": f"call_{index}", + "name": "Bash", + }, + ) + + +@pytest.mark.asyncio +async def test_openai_turn_logs_every_reasoning_item_the_messages_client_was_served( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + success_log: Final = _SuccessLog() + monkeypatch.setattr(litellm, "callbacks", [success_log]) + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={**TEXT_RESPONSE, "output": [*_reasoning_before_call(1), *_reasoning_before_call(2)]}, + ) + ) + + reply: Final = await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=[{"role": "user", "content": "List the logs twice."}], + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + ) + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(success_log.logged.wait(), timeout=30.0) + + assert [block["type"] for block in reply["content"]].count("redacted_thinking") == 2 + assert success_log.logged_reasoning_item_ids == (("rs_1", "rs_2"),) From 5a6475d112037d7968e7b52f31016760f96f66d6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:46:14 -0800 Subject: [PATCH 15/28] test(anthropic): type the parameters of the failure-recording test loggers Backport of 3d53f0047d1 (#49076) to lts/1.108.x. --- ...xperimental_pass_through_adapters_transformation.py | 10 +++++++--- .../test_responses_adapters_handler.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 0463d37c4a8..6222aa7aab5 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5589,17 +5589,21 @@ class _FailureLog(CustomLogger): super().__init__() self.rejections: tuple[str, ...] = () - def _record(self, kwargs) -> None: + def _record(self, kwargs: Mapping[str, object]) -> None: error: Final = str(kwargs.get("exception")) self.rejections = ( *self.rejections, next((phrase for phrase in _UPSTREAM_REJECTION_PHRASES if phrase in error), error), ) - def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + def log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: self._record(kwargs) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: self._record(kwargs) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index c11023b256a..320fb6123bf 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -804,17 +804,21 @@ class _FailureLog(CustomLogger): super().__init__() self.rejections: tuple[str, ...] = () - def _record(self, kwargs) -> None: + def _record(self, kwargs: Mapping[str, object]) -> None: error: Final = str(kwargs.get("exception")) self.rejections = ( *self.rejections, next((phrase for phrase in ("encrypted content", "organization") if phrase in error), error), ) - def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + def log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: self._record(kwargs) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: self._record(kwargs) From c72cb031d8fdb33bb37d30e281847f4dc856e563 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:55:37 -0800 Subject: [PATCH 16/28] test(anthropic): type the success logger and use a fixed token expiry in the bridge tests --- ...tras_litellm_responses_transformation_handler.py | 6 ++++-- .../test_responses_adapters_handler.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index af0a0dbe9b0..ab31cee5e07 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -1,5 +1,4 @@ import json -import time from datetime import datetime from pathlib import Path from typing import Final @@ -381,9 +380,12 @@ def _two_reasoning_tool_turn_stream() -> httpx.Response: return httpx.Response(200, headers={"content-type": "text/event-stream"}, text=body) +_FAR_FUTURE_EXPIRY: Final = 4_102_444_800 + + @pytest.fixture def _chatgpt_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None: - auth: Final = {"access_token": "chatgpt-test-token", "expires_at": time.time() + 3600, "account_id": "acct_test"} + auth: Final = {"access_token": "chatgpt-test-token", "expires_at": _FAR_FUTURE_EXPIRY, "account_id": "acct_test"} (tmp_path / "auth.json").write_text(json.dumps(auth)) monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 320fb6123bf..f21eb9eb6ac 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -3,7 +3,7 @@ import datetime import json import os import sys -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Mapping, Sequence from contextlib import nullcontext from typing import Final, cast from unittest.mock import AsyncMock, patch @@ -28,6 +28,7 @@ from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( ) from litellm.router import Router from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested +from litellm.types.utils import StandardLoggingPayload MESSAGES = [{"role": "user", "content": "hello"}] CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) @@ -916,9 +917,13 @@ class _SuccessLog(CustomLogger): self.logged_reasoning_item_ids: tuple[tuple[object, ...], ...] = () self.logged: Final = asyncio.Event() - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: - logged: Final = kwargs["standard_logging_object"]["response"] - items: Final = logged["choices"][0]["message"].get("reasoning_items") or () + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"]) + logged: Final = cast(Mapping[str, Sequence[Mapping[str, Mapping[str, object]]]], payload["response"]) + message: Final = logged["choices"][0]["message"] + items: Final = cast(Sequence[Mapping[str, object]], message.get("reasoning_items") or ()) self.logged_reasoning_item_ids = (*self.logged_reasoning_item_ids, tuple(item["id"] for item in items)) self.logged.set() From 433a76f754a4bfc4a804f40a3721be79df07ff33 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:28:59 -0800 Subject: [PATCH 17/28] fix(anthropic): skip the SDK num_retries retry for an attempt that is sent again --- litellm/_internal_context.py | 3 +- litellm/utils.py | 14 ++++-- ...al_pass_through_adapters_transformation.py | 45 +++++++++++++++++++ .../test_responses_adapters_handler.py | 33 ++++++++++++++ 4 files changed, 90 insertions(+), 5 deletions(-) diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 1b07c017a3c..c4527e1d099 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -118,7 +118,8 @@ def current_service_caller() -> str | None: @contextmanager def resent_on(rejection: Callable[[Exception], bool]) -> Generator[None]: """An attempt its caller sends again when it fails with ``rejection``. That failure is not the request's - outcome, so nothing logs it and its budget reservation and rate-limit slot stay with the attempt sent again.""" + outcome, so it is neither logged as the request's failure nor retried, and its budget reservation and rate-limit + slot stay with the attempt sent again.""" token: Final = _resent_on.set(rejection) try: yield diff --git a/litellm/utils.py b/litellm/utils.py index 3213e343c4f..4e1c6a91130 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1985,7 +1985,7 @@ def client(original_function): is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_completion_litellm_router_call + num_retries and not is_completion_litellm_router_call and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2019,7 +2019,7 @@ def client(original_function): is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_responses_litellm_router_call + num_retries and not is_responses_litellm_router_call and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2300,7 +2300,10 @@ def client(original_function): is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_acompletion_litellm_router_call + and not isinstance(e, ImportError) + and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -2331,7 +2334,10 @@ def client(original_function): is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_aresponses_litellm_router_call + and not isinstance(e, ImportError) + and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 6222aa7aab5..1683b4a41e7 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5584,6 +5584,51 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_with_num_retries_send_the_rejected_reasoning_only_once( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0] + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_with_num_retries_send_the_rejected_reasoning_only_once( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0] + + class _FailureLog(CustomLogger): def __init__(self) -> None: super().__init__() diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index f21eb9eb6ac..19de6ccd634 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -800,6 +800,39 @@ def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_back assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] +@pytest.mark.asyncio +async def test_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + + +def test_sync_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + + class _FailureLog(CustomLogger): def __init__(self) -> None: super().__init__() From b277d687b5e6f596b8cd26a5286ab2bc4c2f0034 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:22:13 -0800 Subject: [PATCH 18/28] fix(anthropic): stop SDK retries from resending a rejected block or clearing the global num_retries A transient 429 or 500 on a /v1/messages turn that carries a replayed encrypted block was retried by an SDK call's num_retries loop, which sent the block again until every retry was rejected. On async calls the loop then raised the first transient error, so the turn failed instead of being sent again without the block. The retry loop now stops at that rejection and passes it on, so the turn is sent again once without encrypted blocks. A rejected first try on a sync call also cleared the process-wide litellm.num_retries, so later calls in the process were no longer retried. That setting now stays as configured. --- litellm/main.py | 33 ++++++++-- litellm/utils.py | 16 +++-- ...al_pass_through_adapters_transformation.py | 59 ++++++++++++++++- .../test_responses_adapters_handler.py | 63 ++++++++++++++++++- 4 files changed, 155 insertions(+), 16 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 42d5ecc8f20..8626d64ebd7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -27,6 +27,7 @@ from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from litellm._internal_context import is_resent_failure from litellm._logging import redact_string from litellm._uuid import uuid @@ -6242,6 +6243,10 @@ def completion( ) +def _is_not_resent_failure(error: BaseException) -> bool: + return not (isinstance(error, Exception) and is_resent_failure(error)) + + def completion_with_retries(*args, **kwargs): """ Executes a litellm.completion() with 3 retries @@ -6261,10 +6266,15 @@ def completion_with_retries(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.Retrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return retryer(original_function, *args, **kwargs) @@ -6287,10 +6297,15 @@ async def acompletion_with_retries(*args, **kwargs): retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return await retryer(original_function, *args, **kwargs) @@ -6315,10 +6330,15 @@ def responses_with_retries(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.Retrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return retryer(original_function, *args, **kwargs) @@ -6342,10 +6362,15 @@ async def aresponses_with_retries(*args, **kwargs): retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return await retryer(original_function, *args, **kwargs) diff --git a/litellm/utils.py b/litellm/utils.py index 4e1c6a91130..c2acb89b6e6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1980,7 +1980,8 @@ def client(original_function): retry_policy=kwargs.get("retry_policy"), ) kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + if not is_resent_failure(e): + litellm.num_retries = None # set retries to None to prevent infinite loops context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) @@ -2015,7 +2016,8 @@ def client(original_function): retry_policy=kwargs.get("retry_policy"), ) kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + if not is_resent_failure(e): + litellm.num_retries = None # set retries to None to prevent infinite loops is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( @@ -2314,8 +2316,9 @@ def client(original_function): elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" result = await litellm.acompletion_with_retries(*args, **kwargs) - except Exception: - pass + except Exception as retry_error: + if is_resent_failure(retry_error): + raise else: return result elif ( @@ -2348,8 +2351,9 @@ def client(original_function): elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" result = await litellm.aresponses_with_retries(*args, **kwargs) - except Exception: - pass + except Exception as retry_error: + if is_resent_failure(retry_error): + raise else: return result diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1683b4a41e7..f02e7762deb 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5608,9 +5608,10 @@ async def test_async_messages_with_num_retries_send_the_rejected_reasoning_only_ @pytest.mark.usefixtures("_chat_bridge_httpx") -def test_sync_messages_with_num_retries_send_the_rejected_reasoning_only_once( - respx_mock: respx.MockRouter, +def test_sync_messages_with_global_num_retries_send_the_rejected_reasoning_once_and_keep_the_setting( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] ) @@ -5623,10 +5624,62 @@ def test_sync_messages_with_num_retries_send_the_rejected_reasoning_only_once( max_tokens=64, tools=_RETRY_TOOLS, messages=_history_with_foreign_reasoning(), - num_retries=2, ) assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0] + assert litellm.num_retries == 2 + + +_SERVER_ERROR: Final = httpx.Response( + 500, json={"error": {"message": "The server had an error while processing your request.", "type": "server_error"}} +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] class _FailureLog(CustomLogger): diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 19de6ccd634..526ead24d26 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -747,6 +747,12 @@ def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]: def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool): + return _upstream_after(respx_mock, monkeypatch, (first,), stream) + + +def _upstream_after( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, failures: tuple[httpx.Response, ...], stream: bool +) -> respx.Route: monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() @@ -755,7 +761,7 @@ def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, fir if stream else httpx.Response(200, json=TEXT_RESPONSE) ) - return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served]) + return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[*failures, served]) @pytest.mark.asyncio @@ -817,9 +823,10 @@ async def test_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_on assert _replayed_reasoning_counts(upstream) == [1, 0] -def test_sync_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_once( +def test_sync_openai_turn_with_global_num_retries_sends_the_rejected_reasoning_once_and_keeps_the_setting( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) litellm.anthropic.messages.create( @@ -827,10 +834,60 @@ def test_sync_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_onc messages=_history(replayed_reasoning=True), model="openai/gpt-5.6-luna", tools=[dict(BASH_TOOL)], - num_retries=2, ) assert _replayed_reasoning_counts(upstream) == [1, 0] + assert litellm.num_retries == 2 + + +_SERVER_ERROR: Final = { + "error": {"message": "The server had an error while processing your request.", "type": "server_error"} +} + + +@pytest.mark.asyncio +async def test_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream_after( + respx_mock, + monkeypatch, + (httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)), + False, + ) + + response: Final = await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +def test_sync_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream_after( + respx_mock, + monkeypatch, + (httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)), + False, + ) + + response: Final = litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] class _FailureLog(CustomLogger): From 3eda243ea828153f4a539c477b7a269a7f55aa08 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 7 Nov 2026 10:31:40 -0800 Subject: [PATCH 19/28] chore: bump litellm 1.108.2 -> 1.108.3 litellm 1.108.2 already shipped, so the next release off this line is 1.108.3 --- pyproject.toml | 4 ++-- uv.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe156695894..84200cb91a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.108.2" +version = "1.108.3" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -396,7 +396,7 @@ litellm-enterprise = { workspace = true } members = ["enterprise", "litellm-proxy-extras"] [tool.commitizen] -version = "1.108.2" +version = "1.108.3" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index cc78423f359..0df2668e679 100644 --- a/uv.lock +++ b/uv.lock @@ -4594,7 +4594,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.108.2" +version = "1.108.3" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From edb48c1d09222b6d18899c5ea0f59b43bebb7ae4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:45:18 -0800 Subject: [PATCH 20/28] fix(anthropic): stop a cancelled SDK retry from starting another call The resent-failure retry predicate returned true for BaseException subclasses such as asyncio.CancelledError, so a cancelled acompletion_with_retries or aresponses_with_retries call could start another provider call. Retry only Exception subclasses again, as tenacity's default did, while still skipping a resent failure. --- litellm/main.py | 2 +- tests/unit/test_main.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 8626d64ebd7..586f42311b4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6244,7 +6244,7 @@ def completion( def _is_not_resent_failure(error: BaseException) -> bool: - return not (isinstance(error, Exception) and is_resent_failure(error)) + return isinstance(error, Exception) and not is_resent_failure(error) def completion_with_retries(*args, **kwargs): diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 6bb69def307..150f45e009d 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -7936,6 +7936,17 @@ async def test_responses_with_retries(sync_mode): assert mock_responses.call_args.kwargs["max_retries"] == 0 +@pytest.mark.parametrize("retry_function", [acompletion_with_retries, aresponses_with_retries]) +@pytest.mark.asyncio +async def test_async_retry_helpers_stop_when_the_call_is_cancelled(retry_function): + upstream: Final = AsyncMock(side_effect=[asyncio.CancelledError(), MagicMock()]) + + with pytest.raises(asyncio.CancelledError): + await retry_function(model="gpt-4o", num_retries=3, original_function=upstream) + + assert upstream.await_count == 1 + + def test_azure_embedding_exceptions(): with pytest.raises(Exception, match="Mock error") as exc_info: litellm.embedding( From d5ad71dd6584d9a942edf45a7fe0f346d3455440 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:54:21 -0800 Subject: [PATCH 21/28] test(anthropic): type the retry helper parameter in the cancellation test --- tests/unit/test_main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 150f45e009d..c832da5dbfc 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -8,7 +8,7 @@ import logging import os import threading import urllib.parse -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime @@ -7938,7 +7938,9 @@ async def test_responses_with_retries(sync_mode): @pytest.mark.parametrize("retry_function", [acompletion_with_retries, aresponses_with_retries]) @pytest.mark.asyncio -async def test_async_retry_helpers_stop_when_the_call_is_cancelled(retry_function): +async def test_async_retry_helpers_stop_when_the_call_is_cancelled( + retry_function: Callable[..., Awaitable[object]], +) -> None: upstream: Final = AsyncMock(side_effect=[asyncio.CancelledError(), MagicMock()]) with pytest.raises(asyncio.CancelledError): From 52f49b8e8628e5360aef3710848b7780a6d594cb Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 7 Nov 2026 14:34:39 -0800 Subject: [PATCH 22/28] test(integration): backport main's Gateway.rows, Scenario.model name and MCP ack fix to lts/1.108.x Backports test-only changes from main so the line's integration signal stops reporting stale helpers as failures. Gateway.database_url and Gateway.rows come from #47213 (fc27dfdc7d), and the name keyword on Scenario.model comes from #48917 (ebf84f8649). Only those hunks of tests/integration/_support/client.py are taken. The rest of #47213 gives each owned proxy its own database and Redis, which none of the line's tests need, and the workers field from a later main change is left out. The MCP progress and cancellation test wraps its upstream in _acknowledge_notifications_after_dispatch, from #49967 (b5952f243a). The helper is already on the line. --- tests/integration/_support/client.py | 14 ++++++++++---- tests/integration/mcp/test_mcp_transports.py | 6 ++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index 053318c2451..fe1db22987d 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -95,6 +95,10 @@ class Gateway: client: httpx.Client key: str upstream_url: str + database_url: str | None = None + + def rows(self, query: str, parameters: tuple[str, ...]) -> list[dict[str, JsonValue]]: + return read_rows(query, parameters, database_url=self.database_url) def request( self, @@ -267,12 +271,14 @@ class Scenario: ) assert read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (identity,)) == [] - def model(self, *, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue) -> str: - name: Final = f"integration-{uuid.uuid4().hex}" + def model( + self, *, name: str | None = None, model_info: Mapping[str, JsonValue] | None = None, **parameters: JsonValue + ) -> str: + model_name: Final = name if name is not None else f"integration-{uuid.uuid4().hex}" created: Final = self.gateway.post( "/model/new", { - "model_name": name, + "model_name": model_name, "litellm_params": { "model": "openai/gpt-4o-mini", "api_key": "integration-provider-key", @@ -284,7 +290,7 @@ class Scenario: ) identity: Final = string_value(object_value(created["model_info"])["id"]) self.cleanups.callback(self.delete_model, identity) - return name + return model_name @contextmanager diff --git a/tests/integration/mcp/test_mcp_transports.py b/tests/integration/mcp/test_mcp_transports.py index ef240cba570..be9e016e947 100644 --- a/tests/integration/mcp/test_mcp_transports.py +++ b/tests/integration/mcp/test_mcp_transports.py @@ -678,7 +678,7 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( from starlette.routing import Mount from integration._support.asgi import asgi_server - from integration._support.mcp import _drain_sse_streams + from integration._support.mcp import _acknowledge_notifications_after_dispatch, _drain_sse_streams from integration._support.process import owned_proxy upstream: Final = MCPServer("progress-cancellation-peer") @@ -770,7 +770,9 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( watchers = tuple(task for task in asyncio.all_tasks() if "_shutdown_watcher" in repr(task.get_coro())) await asyncio.wait_for(asyncio.gather(*watchers), 2) - hosted: Final = Starlette(routes=[Mount("/", app=app)], lifespan=lifespan) + hosted: Final = Starlette( + routes=[Mount("/", app=_acknowledge_notifications_after_dispatch(app))], lifespan=lifespan + ) with asgi_server(hosted) as upstream_url, httpx.Client() as client: config: Final = tmp_path / "proxy.yaml" config.write_text( From f4ba1bfafc682de903b04ce0cf2e81566977a2db Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 7 Nov 2026 16:45:18 -0800 Subject: [PATCH 23/28] fix(bedrock)!: backport #47429 deployment guardrailConfig on /bedrock pass-through routes to lts/1.108.x The router-model pass-through dropped a Bedrock deployment's guardrailConfig, so converse, converse-stream, invoke and invoke-with-response-stream reached Bedrock unguarded while /v1/chat/completions on the same deployment was blocked. The signer now sends it the way Bedrock reads it, in the body for Converse and as the X-Amzn-Bedrock-Guardrail* headers for Invoke, replacing any guardrail the caller sent Adapted to the line, which does not have #47040. The guardrail headers are merged through merge_bedrock_invoke_headers without the request metadata header, and the unit tests drop the request metadata assertions and the identity argument that only #47040 needs --- .../bedrock/passthrough/transformation.py | 46 +- litellm/llms/bedrock/request_metadata.py | 11 +- litellm/passthrough/main.py | 3 +- ...drock_passthrough_guardrail_config_wire.py | 531 ++++++++++++++++++ ...test_bedrock_passthrough_transformation.py | 195 ++++++- 5 files changed, 776 insertions(+), 10 deletions(-) create mode 100644 tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index d7d6341fe7a..49eea6f5daf 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -13,6 +13,10 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + bedrock_invoke_guardrail_headers, +) +from litellm.llms.bedrock.request_metadata import merge_bedrock_invoke_headers from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage from litellm.types.utils import ModelResponseStream @@ -42,6 +46,37 @@ def _is_converse_url(url: str) -> bool: return httpx.URL(url).path.rstrip("/").rsplit("/", 1)[-1] in _CONVERSE_ACTIONS +_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"}) +_BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset( + {"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"} +) + + +def _converse_guardrail_config(action: str, guardrail_config: object) -> object: + fields: Final = as_json_object(guardrail_config) + if action == "converse-stream" or fields is None: + return guardrail_config + return {key: value for key, value in fields.items() if key != "streamProcessingMode"} + + +def _with_deployment_guardrail( + action: str, + headers: Mapping[str, str], + body: Mapping[str, object], + guardrail_config: object, +) -> tuple[Mapping[str, str], Mapping[str, object]]: + if guardrail_config is None: + return headers, body + if action in _CONVERSE_ACTIONS: + return headers, {**body, "guardrailConfig": _converse_guardrail_config(action, guardrail_config)} + if action in _INVOKE_ACTIONS: + guardrail_headers: Final = tuple(bedrock_invoke_guardrail_headers(guardrail_config).items()) + return merge_bedrock_invoke_headers( + dict(headers), (), guardrail_headers, _BEDROCK_INVOKE_GUARDRAIL_HEADERS + ), body + return headers, body + + def _is_cache_point(entry: object) -> bool: fields: Final = as_json_object(entry) return fields is not None and "cachePoint" in fields @@ -363,13 +398,20 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD api_base: str, model: str | None = None, ) -> tuple[dict, bytes | None]: + action: Final = httpx.URL(api_base).path.rstrip("/").rsplit("/", 1)[-1] body: Final[Mapping[str, object]] = request_data or {} optional_params: Final = litellm_params.copy() + guarded_headers, guarded_body = _with_deployment_guardrail( + action, + headers, + _converse_body_to_send(body) if action in _CONVERSE_ACTIONS else body, + litellm_params.get("guardrailConfig"), + ) return self._sign_request( service_name="bedrock", - headers=headers, + headers=dict(guarded_headers), optional_params=optional_params, - request_data=dict(_converse_body_to_send(body) if _is_converse_url(api_base) else body), + request_data=dict(guarded_body), api_base=api_base, model=model, api_key=self.get_bedrock_bearer_token(optional_params), diff --git a/litellm/llms/bedrock/request_metadata.py b/litellm/llms/bedrock/request_metadata.py index 1f4e5886508..685b52e3b1c 100644 --- a/litellm/llms/bedrock/request_metadata.py +++ b/litellm/llms/bedrock/request_metadata.py @@ -179,12 +179,11 @@ def merge_bedrock_invoke_headers( """ Merge the ``X-Amzn-*`` headers the Invoke paths derive from params. - ``caller_owned`` (the guardrail headers) defers to a header the caller already set, which is - the long-standing behaviour for those. ``proxy_owned_names`` are dropped from the caller's - headers unconditionally and re-supplied only from ``proxy_owned``, because those names carry - proxy-authenticated identity into an AWS billing record that the caller must not be able to - write. Names are compared case-insensitively so a caller cannot leave a second spelling in - the dict and let the transport pick the winner. + ``caller_owned`` defers to a header the caller already set. ``proxy_owned_names`` are dropped + from the caller's headers unconditionally and re-supplied only from ``proxy_owned``, for names + the caller must not be able to write, such as proxy-authenticated identity headed for an AWS + billing record or a deployment's guardrail. Names are compared case-insensitively so a caller + cannot leave a second spelling in the dict and let the transport pick the winner. """ if not caller_owned and not proxy_owned and not proxy_owned_names: return headers diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 744f9e1aa4c..4e94b1bb910 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -437,6 +437,7 @@ def llm_passthrough_route( params: QueryParamTypes | None = None, cookies: CookieTypes | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, + guardrailConfig: object = None, **kwargs, ) -> ( httpx.Response @@ -547,7 +548,7 @@ def llm_passthrough_route( _request_data: dict | None = data if isinstance(data, dict) else (json if isinstance(json, dict) else None) headers, signed_json_body = provider_config.sign_request( headers=headers, - litellm_params=litellm_params_dict, + litellm_params={**litellm_params_dict, "guardrailConfig": guardrailConfig}, request_data=_request_data, api_base=str(updated_url), model=model, diff --git a/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py b/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py new file mode 100644 index 00000000000..f995a7ac83f --- /dev/null +++ b/tests/integration/providers/test_bedrock_passthrough_guardrail_config_wire.py @@ -0,0 +1,531 @@ +import base64 +import json +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import boto3 +import pytest +import yaml +from botocore.config import Config +from botocore.exceptions import ClientError +from integration._support.budget import case_timeout_seconds +from integration._support.client import Gateway +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.upstream import _aws_event_frame +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 180) + +_MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0" +_EVENT_STREAM: Final = "application/vnd.amazon.eventstream" +_BLOCKED_WORD: Final = "zzzforbidden" +_BLOCKED_TEXT: Final = "Blocked by guardrail" +_VERSIONLESS_GUARDRAIL_MESSAGE: Final = "guardrailIdentifier and guardrailVersion must be provided together." +_ACTIONS: Final = ("converse", "converse-stream", "invoke", "invoke-with-response-stream") +_DEPLOYMENT_GUARDRAIL: Final[Mapping[str, str]] = { + "guardrailIdentifier": "deployment-guardrail", + "guardrailVersion": "3", + "trace": "enabled", + "streamProcessingMode": "sync", +} +_CALLER_GUARDRAIL: Final[Mapping[str, str]] = { + "guardrailIdentifier": "caller-guardrail", + "guardrailVersion": "9", + "trace": "disabled", +} +_MANTLE_KEY: Final = "scripted-mantle-key" +_CALLER_INVOKE_HEADERS: Final[Mapping[str, str]] = { + "x-pass-x-amzn-bedrock-guardrailidentifier": "caller-guardrail", + "x-pass-x-amzn-bedrock-guardrailversion": "9", + "x-pass-x-amzn-bedrock-trace": "DISABLED", +} + + +@dataclass(frozen=True, slots=True) +class Applied: + """The guardrail a request asked Bedrock for, read from where Bedrock reads it for that action.""" + + identifier: str + version: str | None + trace: str | None + stream_processing_mode: str | None + + +@dataclass(frozen=True, slots=True) +class Outcome: + """What the AWS SDK caller reads back.""" + + text: str + intervened: bool + traced_guardrail: str | None + + +def _action(request: Request) -> str: + return request.target.partition("?")[0].rsplit("/", 1)[-1] + + +def _applied(request: Request) -> Applied | None: + if _action(request).startswith("converse"): + config: Final = json.loads(request.body).get("guardrailConfig") + if config is None: + return None + return Applied( + config["guardrailIdentifier"], + config.get("guardrailVersion"), + config.get("trace"), + config.get("streamProcessingMode"), + ) + identifier: Final = request.headers.get("x-amzn-bedrock-guardrailidentifier") + if identifier is None: + return None + return Applied( + identifier, + request.headers.get("x-amzn-bedrock-guardrailversion"), + request.headers.get("x-amzn-bedrock-trace"), + None, + ) + + +def _identifier(request: Request) -> str | None: + applied: Final = _applied(request) + return None if applied is None else applied.identifier + + +def _assessment(applied: Applied) -> Mapping[str, JsonValue]: + return { + applied.identifier: { + "wordPolicy": {"customWords": [{"match": _BLOCKED_WORD, "action": "BLOCKED", "detected": True}]}, + "appliedGuardrailDetails": {"guardrailId": applied.identifier, "guardrailVersion": applied.version}, + } + } + + +def _bedrock_with_guardrails(request: Request) -> Reply: + """Bedrock's runtime as the caller sees it: the guardrail the request names blocks a prompt holding the blocked + word, answering with its message and, when the request enabled tracing, a trace naming that guardrail. A guardrail + named without a version is refused with a 400 ValidationException, as real Bedrock in us-east-1 refused it on + every one of these actions on 2026-10-19.""" + applied: Final = _applied(request) + if applied is not None and applied.version is None: + return Reply( + status=400, + body=json.dumps({"message": _VERSIONLESS_GUARDRAIL_MESSAGE}).encode(), + headers={"x-amzn-errortype": "ValidationException"}, + ) + blocked: Final = applied is not None and _BLOCKED_WORD.encode() in request.body + traced: Final = blocked and applied is not None and (applied.trace or "").lower() == "enabled" + text: Final = _BLOCKED_TEXT if blocked else "ok" + stop_reason: Final = "guardrail_intervened" if blocked else "end_turn" + usage: Final = {"inputTokens": 5, "outputTokens": 1, "totalTokens": 6} + converse_trace: Final = {"trace": {"guardrail": {"inputAssessment": _assessment(applied)}}} if traced else {} + action: Final = _action(request) + if action == "converse": + return Reply( + body=json.dumps( + { + "output": {"message": {"role": "assistant", "content": [{"text": text}]}}, + "stopReason": stop_reason, + "usage": usage, + "metrics": {"latencyMs": 3}, + **converse_trace, + } + ).encode() + ) + if action == "converse-stream": + return Reply( + chunks=( + _aws_event_frame("messageStart", {"role": "assistant"}, "sc", "u"), + _aws_event_frame("contentBlockDelta", {"delta": {"text": text}, "contentBlockIndex": 0}, "sc", "u"), + _aws_event_frame("contentBlockStop", {"contentBlockIndex": 0}, "sc", "u"), + _aws_event_frame("messageStop", {"stopReason": stop_reason}, "sc", "u"), + _aws_event_frame( + "metadata", {"usage": usage, "metrics": {"latencyMs": 3}, **converse_trace}, "sc", "u" + ), + ), + content_type=_EVENT_STREAM, + ) + message: Final = { + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": "claude", + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, + **({"amazon-bedrock-guardrailAction": "INTERVENED"} if blocked else {}), + **({"amazon-bedrock-trace": {"guardrail": {"input": _assessment(applied)}}} if traced else {}), + } + if action == "invoke": + return Reply(body=json.dumps(message).encode()) + chunk: Final = {"bytes": base64.b64encode(json.dumps(message).encode()).decode()} + return Reply(chunks=(_aws_event_frame("chunk", chunk, "sc", "u"),), content_type=_EVENT_STREAM) + + +def _bedrock_outage(request: Request) -> Reply: + return Reply( + status=500, + body=json.dumps({"message": "scripted outage"}).encode(), + headers={"x-amzn-errortype": "InternalServerException"}, + ) + + +def _traced_guardrail(assessment: object) -> str | None: + return None if not isinstance(assessment, dict) else next(iter(assessment)) + + +def _send(runtime: object, action: str, model: str, prompt: str, caller_guardrail: bool) -> Outcome: + messages: Final = [{"role": "user", "content": [{"text": prompt}]}] + invoke_body: Final = json.dumps( + {"anthropic_version": "bedrock-2023-05-31", "max_tokens": 16, "messages": [{"role": "user", "content": prompt}]} + ) + invoke_guardrail: Final = ( + {"guardrailIdentifier": "caller-guardrail", "guardrailVersion": "9", "trace": "DISABLED"} + if caller_guardrail + else {} + ) + if action == "converse": + response: Final = getattr(runtime, "converse")( + modelId=model, + messages=messages, + **({"guardrailConfig": dict(_CALLER_GUARDRAIL)} if caller_guardrail else {}), + ) + return Outcome( + text=response["output"]["message"]["content"][0]["text"], + intervened=response["stopReason"] == "guardrail_intervened", + traced_guardrail=_traced_guardrail(response.get("trace", {}).get("guardrail", {}).get("inputAssessment")), + ) + if action == "converse-stream": + events: Final = list( + getattr(runtime, "converse_stream")( + modelId=model, + messages=messages, + **( + {"guardrailConfig": {**_CALLER_GUARDRAIL, "streamProcessingMode": "async"}} + if caller_guardrail + else {} + ), + )["stream"] + ) + (stop,) = tuple(event["messageStop"]["stopReason"] for event in events if "messageStop" in event) + (metadata,) = tuple(event["metadata"] for event in events if "metadata" in event) + return Outcome( + text="".join( + event["contentBlockDelta"]["delta"]["text"] for event in events if "contentBlockDelta" in event + ), + intervened=stop == "guardrail_intervened", + traced_guardrail=_traced_guardrail(metadata.get("trace", {}).get("guardrail", {}).get("inputAssessment")), + ) + if action == "invoke": + message: Final = json.loads( + getattr(runtime, "invoke_model")(modelId=model, body=invoke_body, **invoke_guardrail)["body"].read() + ) + else: + (message,) = tuple( + json.loads(event["chunk"]["bytes"]) + for event in getattr(runtime, "invoke_model_with_response_stream")( + modelId=model, body=invoke_body, **invoke_guardrail + )["body"] + ) + return Outcome( + text=message["content"][0]["text"], + intervened=message.get("amazon-bedrock-guardrailAction") == "INTERVENED", + traced_guardrail=_traced_guardrail(message.get("amazon-bedrock-trace", {}).get("guardrail", {}).get("input")), + ) + + +def _signed_header_names(request: Request) -> frozenset[str]: + return frozenset(request.headers["authorization"].split("SignedHeaders=")[1].split(",")[0].split(";")) + + +def _deployment(wire: Wire, guardrail_config: Mapping[str, str] | None = None) -> Mapping[str, JsonValue]: + return { + "model": f"bedrock/{_MODEL_ID}", + "api_base": wire.url, + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + "use_in_pass_through": True, + **({} if guardrail_config is None else {"guardrailConfig": dict(guardrail_config)}), + } + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + wire: Wire + guarded_model: str + unguarded_model: str + balanced_model: str + balanced_wires: Mapping[str, Wire] + failing_model: str + failing_wire: Wire + fallback_wire: Wire + mantle_model: str + versionless_model: str + key: str + + def runtime( # noqa: ANN201 # boto3 clients are untyped + self, monkeypatch: pytest.MonkeyPatch, extra_headers: Mapping[str, str] = MappingProxyType({}) + ): + monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", self.key) + client: Final = boto3.client( + "bedrock-runtime", + endpoint_url=f"{str(self.proxy.client.base_url).rstrip('/')}/bedrock", + region_name="us-east-1", + config=Config(read_timeout=case_timeout_seconds(), retries={"total_max_attempts": 1}), + ) + client.meta.events.register( + "before-send.bedrock-runtime.*", + lambda request, **_: request.headers.update(extra_headers), + ) + return client + + def received(self, wire: Wire, marker: str) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if marker.encode() in request.body) + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + suffix: Final = uuid.uuid4().hex[:12] + guarded_model: Final = f"bedrock-guarded-{suffix}" + unguarded_model: Final = f"bedrock-unguarded-{suffix}" + balanced_model: Final = f"bedrock-balanced-{suffix}" + failing_model: Final = f"bedrock-failing-{suffix}" + fallback_model: Final = f"bedrock-fallback-{suffix}" + mantle_model: Final = f"bedrock-mantle-{suffix}" + versionless_model: Final = f"bedrock-versionless-{suffix}" + directory: Final = tmp_path_factory.mktemp("bedrock-guardrail-config") + base_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + with ( + wire_server(_bedrock_with_guardrails) as wire, + wire_server(_bedrock_with_guardrails) as wire_a, + wire_server(_bedrock_with_guardrails) as wire_b, + wire_server(_bedrock_outage) as failing_wire, + wire_server(_bedrock_with_guardrails) as fallback_wire, + ): + config: Final = { + **base_config, + "model_list": [ + *base_config.get("model_list", []), + { + "model_name": guarded_model, + "litellm_params": _deployment(wire, _DEPLOYMENT_GUARDRAIL), + }, + {"model_name": unguarded_model, "litellm_params": _deployment(wire)}, + { + "model_name": balanced_model, + "litellm_params": _deployment( + wire_a, {"guardrailIdentifier": "guardrail-a", "guardrailVersion": "1"} + ), + }, + { + "model_name": balanced_model, + "litellm_params": _deployment( + wire_b, {"guardrailIdentifier": "guardrail-b", "guardrailVersion": "2"} + ), + }, + { + "model_name": failing_model, + "litellm_params": _deployment( + failing_wire, + {"guardrailIdentifier": "failing-guardrail", "guardrailVersion": "5", "trace": "enabled"}, + ), + }, + { + "model_name": fallback_model, + "litellm_params": _deployment( + fallback_wire, + {"guardrailIdentifier": "fallback-guardrail", "guardrailVersion": "6", "trace": "enabled"}, + ), + }, + { + "model_name": versionless_model, + "litellm_params": _deployment( + wire, {"guardrailIdentifier": "versionless-guardrail", "trace": "enabled"} + ), + }, + { + "model_name": mantle_model, + "litellm_params": { + "model": "bedrock_mantle/anthropic.claude-sonnet-5", + "api_base": wire.url, + "api_key": _MANTLE_KEY, + "aws_region_name": "us-east-1", + "use_in_pass_through": True, + "guardrailConfig": { + "guardrailIdentifier": "mantle-guardrail", + "guardrailVersion": "4", + "trace": "enabled", + }, + }, + }, + ], + "router_settings": { + **base_config.get("router_settings", {}), + "fallbacks": [{failing_model: [fallback_model]}], + }, + } + config_path: Final = directory / "bedrock-guardrail-config.yaml" + config_path.write_text(yaml.safe_dump(config)) + with ( + owned_proxy(session_gateway, directory, {}, config=config_path, workers=2) as proxy, + proxy.scenario() as scenario, + ): + yield Rig( + proxy=proxy, + wire=wire, + guarded_model=guarded_model, + unguarded_model=unguarded_model, + balanced_model=balanced_model, + balanced_wires={"guardrail-a": wire_a, "guardrail-b": wire_b}, + failing_model=failing_model, + failing_wire=failing_wire, + fallback_wire=fallback_wire, + mantle_model=mantle_model, + versionless_model=versionless_model, + key=scenario.key(key_alias=f"guardrail-key-{suffix}"), + ) + + +def _expected_on_the_wire(action: str, request: Request) -> Applied: + if action == "converse": + return Applied("deployment-guardrail", "3", "enabled", None) + if action == "converse-stream": + return Applied("deployment-guardrail", "3", "enabled", "sync") + assert {"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"} <= ( + _signed_header_names(request) + ), request.headers["authorization"] + return Applied("deployment-guardrail", "3", "ENABLED", None) + + +@pytest.mark.parametrize("action", _ACTIONS) +def test_passthrough_on_a_guardrailed_deployment_blocks_with_the_deployment_guardrail( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch) + + blocked: Final = _send(runtime, action, rig.guarded_model, f"{marker} say {_BLOCKED_WORD}", caller_guardrail=False) + allowed: Final = _send(runtime, action, rig.guarded_model, f"{marker} say hello", caller_guardrail=False) + + assert blocked == Outcome(_BLOCKED_TEXT, intervened=True, traced_guardrail="deployment-guardrail"), blocked + assert allowed == Outcome("ok", intervened=False, traced_guardrail=None), allowed + received: Final = rig.received(rig.wire, marker) + assert [_action(request) for request in received] == [action, action], received + assert [_applied(request) for request in received] == [ + _expected_on_the_wire(action, request) for request in received + ], [(request.headers, request.body) for request in received] + + +@pytest.mark.parametrize("action", _ACTIONS) +def test_caller_guardrail_is_replaced_by_the_deployment_guardrail( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch, _CALLER_INVOKE_HEADERS) + + outcome: Final = _send(runtime, action, rig.guarded_model, f"{marker} say {_BLOCKED_WORD}", caller_guardrail=True) + + assert outcome == Outcome(_BLOCKED_TEXT, intervened=True, traced_guardrail="deployment-guardrail"), outcome + (request,) = rig.received(rig.wire, marker) + assert _applied(request) == _expected_on_the_wire(action, request), (request.headers, request.body) + assert "caller-guardrail" not in json.dumps({**request.headers, "body": request.body.decode()}), request + + +@pytest.mark.parametrize("action", _ACTIONS) +def test_mantle_deployment_passthrough_sends_its_guardrail_with_bearer_auth( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch) + + outcome: Final = _send(runtime, action, rig.mantle_model, f"{marker} say {_BLOCKED_WORD}", caller_guardrail=False) + + assert outcome == Outcome(_BLOCKED_TEXT, intervened=True, traced_guardrail="mantle-guardrail"), outcome + (request,) = rig.received(rig.wire, marker) + assert request.headers["authorization"] == f"Bearer {_MANTLE_KEY}", request.headers + assert _applied(request) == Applied( + "mantle-guardrail", "4", "ENABLED" if action.startswith("invoke") else "enabled", None + ), (request.headers, request.body) + + +@pytest.mark.parametrize("action", _ACTIONS) +def test_deployment_guardrail_without_a_version_returns_the_bedrock_rejection( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch) + + with pytest.raises(ClientError) as raised: + _send(runtime, action, rig.versionless_model, f"{marker} say hello", caller_guardrail=False) + + error: Final = raised.value.response.get("Error", {}) + assert (error.get("Code"), error.get("Message")) == ("ValidationException", _VERSIONLESS_GUARDRAIL_MESSAGE), error + received: Final = rig.received(rig.wire, marker) + assert {_applied(request) for request in received} == { + Applied("versionless-guardrail", None, "ENABLED" if action.startswith("invoke") else "enabled", None) + }, [(request.headers, request.body) for request in received] + + +@pytest.mark.parametrize("action", _ACTIONS) +def test_deployment_without_a_guardrail_leaves_the_caller_guardrail_as_sent( + rig: Rig, monkeypatch: pytest.MonkeyPatch, action: str +) -> None: + if action.startswith("invoke"): + pytest.skip("BUG: the router-model invoke pass-through drops the caller's X-Amzn-Bedrock-Guardrail* headers") + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch) + + outcome: Final = _send(runtime, action, rig.unguarded_model, f"{marker} say {_BLOCKED_WORD}", caller_guardrail=True) + + assert outcome == Outcome(_BLOCKED_TEXT, intervened=True, traced_guardrail=None), outcome + (request,) = rig.received(rig.wire, marker) + assert _applied(request) == Applied( + "caller-guardrail", "9", "disabled", "async" if action == "converse-stream" else None + ), request.body + + +def test_each_load_balanced_request_carries_the_guardrail_of_the_deployment_it_reached( + rig: Rig, monkeypatch: pytest.MonkeyPatch +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch) + + outcomes: Final = tuple( + _send(runtime, action, rig.balanced_model, f"{marker}-{attempt} say {_BLOCKED_WORD}", caller_guardrail=False) + for attempt, action in enumerate(_ACTIONS * 6) + ) + + assert all(outcome.intervened for outcome in outcomes), outcomes + reached: Final = {name: rig.received(wire, marker) for name, wire in rig.balanced_wires.items()} + assert sum(len(requests) for requests in reached.values()) == len(outcomes), reached + assert all(reached.values()), {name: len(requests) for name, requests in reached.items()} + assert {name: {_identifier(request) for request in requests} for name, requests in reached.items()} == { + "guardrail-a": {"guardrail-a"}, + "guardrail-b": {"guardrail-b"}, + }, reached + + +def test_a_fallback_sends_the_guardrail_of_the_deployment_it_fell_back_to( + rig: Rig, monkeypatch: pytest.MonkeyPatch +) -> None: + marker: Final = uuid.uuid4().hex + runtime: Final = rig.runtime(monkeypatch) + + outcomes: Final = tuple( + _send(runtime, action, rig.failing_model, f"{marker} say {_BLOCKED_WORD}", caller_guardrail=False) + for action in _ACTIONS + ) + + assert outcomes == (Outcome(_BLOCKED_TEXT, intervened=True, traced_guardrail="fallback-guardrail"),) * 4, outcomes + failed: Final = rig.received(rig.failing_wire, marker) + served: Final = rig.received(rig.fallback_wire, marker) + assert [_action(request) for request in served] == list(_ACTIONS), served + assert {_identifier(request) for request in failed} == {"failing-guardrail"}, failed + assert {_identifier(request) for request in served} == {"fallback-guardrail"}, served diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 84d3160e609..4b7e7aad5e4 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -5,8 +5,9 @@ import struct import sys import types from binascii import crc32 -from collections.abc import Callable +from collections.abc import Callable, Mapping from datetime import datetime +from types import MappingProxyType from typing import Final from unittest.mock import patch @@ -17,6 +18,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector +from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.passthrough.transformation import ( BedrockPassthroughConfig, is_bedrock_streaming_endpoint, @@ -898,3 +900,194 @@ def test_stream_collector_logs_nothing_for_an_unrecognized_endpoint(): collector.add(_event_frame("messageStart", {"role": "assistant"})) assert collector.build_logged_response(_converse_stream_logging_obj()) is None + + +CONVERSE_BODY = {"messages": [{"role": "user", "content": [{"text": "hi"}]}]} +INVOKE_BODY = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 8, + "messages": [{"role": "user", "content": "hi"}], +} +DEPLOYMENT_GUARDRAIL: Final[Mapping[str, str]] = { + "guardrailIdentifier": "deployment-guardrail", + "guardrailVersion": "3", + "trace": "enabled", + "streamProcessingMode": "sync", +} +DEPLOYMENT_INVOKE_GUARDRAIL_HEADERS: Final[Mapping[str, str]] = { + "x-amzn-bedrock-guardrailidentifier": "deployment-guardrail", + "x-amzn-bedrock-guardrailversion": "3", + "x-amzn-bedrock-trace": "ENABLED", +} +CALLER_GUARDRAIL: Final[Mapping[str, str]] = { + "guardrailIdentifier": "caller-guardrail", + "guardrailVersion": "9", + "trace": "disabled", +} +CALLER_INVOKE_GUARDRAIL_HEADERS: Final[Mapping[str, str]] = { + "x-pass-X-Amzn-Bedrock-GuardrailIdentifier": "caller-guardrail", + "x-pass-X-Amzn-Bedrock-GuardrailVersion": "9", + "x-pass-X-Amzn-Bedrock-Trace": "DISABLED", +} + + +async def _sent_passthrough_request( + action: str, + body: dict[str, object], + deployment_params: Mapping[str, object] = MappingProxyType({}), + **kwargs: object, +) -> httpx.Request: + requests: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={"ok": True}, request=request) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "bedrock-pt", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "aws_access_key_id": "AKIAROUTERKEY", + "aws_secret_access_key": "router-secret", + "aws_region_name": "us-west-2", + **deployment_params, + }, + } + ] + ) + await router.allm_passthrough_route( + model="bedrock-pt", + method="POST", + endpoint=f"/model/bedrock-pt/{action}", + data=body, + client=AsyncHTTPHandler(transport=httpx.MockTransport(respond)), + **kwargs, + ) + return requests[0] + + +def _signed_header_names(request: httpx.Request) -> frozenset[str]: + return frozenset(request.headers["authorization"].split("SignedHeaders=")[1].split(",")[0].split(";")) + + +def _invoke_guardrail_headers(request: httpx.Request) -> dict[str, str]: + return { + name: value + for name, value in request.headers.items() + if name.lower().startswith("x-amzn-bedrock-guardrail") or name.lower() == "x-amzn-bedrock-trace" + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("action", "sent_guardrail"), + [ + ("converse", {"guardrailIdentifier": "deployment-guardrail", "guardrailVersion": "3", "trace": "enabled"}), + ("converse-stream", DEPLOYMENT_GUARDRAIL), + ], +) +async def test_converse_passthrough_sends_the_deployment_guardrail_in_the_body( + action: str, sent_guardrail: Mapping[str, str] +) -> None: + request: Final = await _sent_passthrough_request( + action, CONVERSE_BODY, deployment_params={"guardrailConfig": DEPLOYMENT_GUARDRAIL} + ) + + assert json.loads(request.content) == {**CONVERSE_BODY, "guardrailConfig": sent_guardrail} + assert _invoke_guardrail_headers(request) == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["invoke", "invoke-with-response-stream"]) +async def test_invoke_passthrough_sends_the_deployment_guardrail_as_signed_headers(action: str) -> None: + request: Final = await _sent_passthrough_request( + action, INVOKE_BODY, deployment_params={"guardrailConfig": DEPLOYMENT_GUARDRAIL} + ) + + assert _invoke_guardrail_headers(request) == DEPLOYMENT_INVOKE_GUARDRAIL_HEADERS + assert set(DEPLOYMENT_INVOKE_GUARDRAIL_HEADERS) <= _signed_header_names(request) + assert json.loads(request.content) == INVOKE_BODY + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("action", "caller_guardrail"), + [("converse", CALLER_GUARDRAIL), ("converse-stream", {**CALLER_GUARDRAIL, "streamProcessingMode": "async"})], +) +async def test_deployment_guardrail_replaces_the_caller_converse_guardrail( + action: str, caller_guardrail: Mapping[str, str] +) -> None: + request: Final = await _sent_passthrough_request( + action, + {**CONVERSE_BODY, "guardrailConfig": caller_guardrail}, + deployment_params={"guardrailConfig": {"guardrailIdentifier": "deployment-guardrail"}}, + ) + + assert json.loads(request.content) == { + **CONVERSE_BODY, + "guardrailConfig": {"guardrailIdentifier": "deployment-guardrail"}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["invoke", "invoke-with-response-stream"]) +async def test_deployment_guardrail_replaces_every_invoke_guardrail_header_the_caller_sent(action: str) -> None: + request: Final = await _sent_passthrough_request( + action, + INVOKE_BODY, + deployment_params={"guardrailConfig": {"guardrailIdentifier": "deployment-guardrail"}}, + request_headers=CALLER_INVOKE_GUARDRAIL_HEADERS, + ) + + assert _invoke_guardrail_headers(request) == {"x-amzn-bedrock-guardrailidentifier": "deployment-guardrail"} + assert json.loads(request.content) == INVOKE_BODY + + +@pytest.mark.asyncio +async def test_passthrough_without_a_deployment_guardrail_sends_the_caller_guardrail_as_is() -> None: + caller_body: Final = {**CONVERSE_BODY, "guardrailConfig": CALLER_GUARDRAIL} + + converse: Final = await _sent_passthrough_request("converse", caller_body) + invoke: Final = await _sent_passthrough_request("invoke", INVOKE_BODY, request_headers=CALLER_INVOKE_GUARDRAIL_HEADERS) + + assert json.loads(converse.content) == caller_body + assert _invoke_guardrail_headers(invoke) == { + "x-amzn-bedrock-guardrailidentifier": "caller-guardrail", + "x-amzn-bedrock-guardrailversion": "9", + "x-amzn-bedrock-trace": "DISABLED", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("action", ["invoke", "invoke-with-response-stream"]) +@pytest.mark.parametrize("guardrail_config", ["deployment-guardrail", {"guardrailVersion": "3"}]) +async def test_invoke_passthrough_refuses_a_deployment_guardrail_it_cannot_send( + action: str, guardrail_config: object +) -> None: + with pytest.raises(BedrockError, match="guardrailConfig") as rejected: + await _sent_passthrough_request(action, INVOKE_BODY, deployment_params={"guardrailConfig": guardrail_config}) + + assert rejected.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_converse_passthrough_forwards_a_non_object_deployment_guardrail_unchanged() -> None: + request: Final = await _sent_passthrough_request( + "converse", CONVERSE_BODY, deployment_params={"guardrailConfig": "deployment-guardrail"} + ) + + assert json.loads(request.content) == {**CONVERSE_BODY, "guardrailConfig": "deployment-guardrail"} + + +@pytest.mark.asyncio +async def test_passthrough_leaves_actions_without_a_guardrail_field_as_sent() -> None: + count_tokens_body: Final = {"input": {"converse": CONVERSE_BODY}} + + request: Final = await _sent_passthrough_request( + "count-tokens", count_tokens_body, deployment_params={"guardrailConfig": DEPLOYMENT_GUARDRAIL} + ) + + assert json.loads(request.content) == count_tokens_body + assert _invoke_guardrail_headers(request) == {} From 4fb2f026a03bb9d676f9fad1c214a36d0244a209 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 7 Nov 2026 16:58:22 -0800 Subject: [PATCH 24/28] refactor(bedrock): drop the unused converse URL helper and cover a trailing-slash converse pass-through --- litellm/llms/bedrock/passthrough/transformation.py | 4 ---- .../passthrough/test_bedrock_passthrough_transformation.py | 5 ++++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 49eea6f5daf..453e730e7fc 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -42,10 +42,6 @@ def is_bedrock_streaming_endpoint(endpoint: str) -> bool: _CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"}) -def _is_converse_url(url: str) -> bool: - return httpx.URL(url).path.rstrip("/").rsplit("/", 1)[-1] in _CONVERSE_ACTIONS - - _INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"}) _BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset( {"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"} diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index 4b7e7aad5e4..e418f6778ee 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -986,6 +986,7 @@ def _invoke_guardrail_headers(request: httpx.Request) -> dict[str, str]: [ ("converse", {"guardrailIdentifier": "deployment-guardrail", "guardrailVersion": "3", "trace": "enabled"}), ("converse-stream", DEPLOYMENT_GUARDRAIL), + ("converse/", {"guardrailIdentifier": "deployment-guardrail", "guardrailVersion": "3", "trace": "enabled"}), ], ) async def test_converse_passthrough_sends_the_deployment_guardrail_in_the_body( @@ -1050,7 +1051,9 @@ async def test_passthrough_without_a_deployment_guardrail_sends_the_caller_guard caller_body: Final = {**CONVERSE_BODY, "guardrailConfig": CALLER_GUARDRAIL} converse: Final = await _sent_passthrough_request("converse", caller_body) - invoke: Final = await _sent_passthrough_request("invoke", INVOKE_BODY, request_headers=CALLER_INVOKE_GUARDRAIL_HEADERS) + invoke: Final = await _sent_passthrough_request( + "invoke", INVOKE_BODY, request_headers=CALLER_INVOKE_GUARDRAIL_HEADERS + ) assert json.loads(converse.content) == caller_body assert _invoke_guardrail_headers(invoke) == { From d06f5a5021ed67982c6a19302470b9b790554b93 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 7 Nov 2026 17:07:56 -0800 Subject: [PATCH 25/28] test(e2e): repin the Together reasoning replay backend to MiniMax-M3.1-Flash MiniMax-M3 reached its 2026-11-06 deprecation date in the cost map, which names MiniMax-M3.1-Flash as its successor (cherry picked from commit 90f7ab6fe6837152b21617c8cb7dc73a541c42e9) --- tests/e2e/llm_translation/test_together_ai_e2e.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py index 2d5d5f2d93a..7775c5c6ed2 100644 --- a/tests/e2e/llm_translation/test_together_ai_e2e.py +++ b/tests/e2e/llm_translation/test_together_ai_e2e.py @@ -7,9 +7,9 @@ backends are pinned because the registry has no flag for what they prove: ``enab and the ``{"reasoning": {"enabled": false}}`` toggle that ``reasoning_effort="none"`` maps to are Qwen hybrid-model contracts, and the structured-output case runs on that hybrid model with reasoning off, since a reasoning-only model can spend the whole token budget -thinking and return no content. MiniMax-M3 is the serverless model whose +thinking and return no content. MiniMax-M3.1-Flash is the serverless model whose template renders a replayed ``reasoning_content`` back into the prompt (Qwen and -DeepSeek silently drop it). MiniMax-M3 honors that replayed field on nearly every call, not +DeepSeek silently drop it). MiniMax-M3.1-Flash honors that replayed field on nearly every call, not every call (one miss in dozens of otherwise identical calls), so the replay case asks up to ``REPLAY_ATTEMPTS`` times and fails only when no answer carries the secret, which a proxy that strips the field guarantees. Requires TOGETHER_API_KEY on the proxy; no @@ -56,7 +56,7 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e HYBRID_REASONING_BACKEND = "together_ai/Qwen/Qwen3.5-9B" -REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3" +REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3.1-Flash" SECRET_PROMPT = "Remember this for later and reply with just OK." SECRET_REASONING = "The user told me their favorite color is chartreuse. I must remember it." From cb738a569765fce2ebdd6c1f8a423f5611e2eff3 Mon Sep 17 00:00:00 2001 From: MFJG <59931828+MFJG-dev@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:44:54 -0400 Subject: [PATCH 26/28] fix(sdk): backport #45961 to lts/1.108.x completion() loaded the MCP gateway handler for any request with tools, and the handler pulls in mcp 2.x only code, so tool calls failed on installs with mcp 1.x (google-adk 1.36.1) or no mcp at all Cherry-picked from 783cf06f31. Only conflict was import context in litellm/responses/main.py (cherry picked from commit 783cf06f31e55e8f7ab84c2d7c708b0d90e68001) --- .../pass_through/messages/handler.py | 53 ++- litellm/main.py | 103 +++--- litellm/responses/main.py | 7 +- .../mcp/litellm_proxy_mcp_handler.py | 44 +-- litellm/responses/mcp/tool_precheck.py | 43 +++ .../base_sdk_tests/check_base_sdk_install.py | 24 ++ tests/base_sdk_tests/check_sdk_http.py | 55 ++- .../integration/mcp/test_mcp_llm_endpoints.py | 55 +++ .../unit/responses/mcp/test_tool_precheck.py | 318 ++++++++++++++++++ .../test_responses_prompt_management.py | 35 +- 10 files changed, 593 insertions(+), 144 deletions(-) create mode 100644 litellm/responses/mcp/tool_precheck.py create mode 100644 tests/unit/responses/mcp/test_tool_precheck.py diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index eeff6302ee8..949d81f3c1b 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -27,6 +27,7 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -569,38 +570,34 @@ def anthropic_messages_handler( # downstream path (native passthrough and both bridges) gets real tools rather than a # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider. skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) - if not skip_mcp_handler and tools: + if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.llms.anthropic.pass_through.messages.mcp_handler import ( anthropic_messages_with_mcp, ) - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - if LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools): - return anthropic_messages_with_mcp( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower - "str | None", system - ), - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - container=container, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, - ) + return anthropic_messages_with_mcp( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower + "str | None", system + ), + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + container=container, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) anthropic_messages_provider_config: Final = _native_messages_config( model, custom_llm_provider, kwargs.get("model_info") diff --git a/litellm/main.py b/litellm/main.py index 42d5ecc8f20..fa628cd9aff 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -135,6 +135,7 @@ from litellm.llms.vertex_ai.common_utils import ( get_vertex_ai_model_route, ) from litellm.realtime_api.main import realtime_health_check +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.completion import ( CompletionDispatchContext, @@ -5366,63 +5367,55 @@ def completion( control_options: Final = _resolve_control_options(kwargs, model) skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) - if not skip_mcp_handler and tools: + if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - from litellm.types.llms.openai import ToolParam - # Check if MCP tools are present (following responses pattern) - # Cast tools to Optional[Iterable[ToolParam]] for type checking - tools_for_mcp: Final = cast(Iterable[ToolParam] | None, tools) - if LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools_for_mcp): - return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it - model=model, - messages=messages, - functions=functions, - function_call=function_call, - timeout=timeout, - temperature=temperature, - top_p=top_p, - n=n, - stream=stream, - stream_options=stream_options, - stop=stop, - max_tokens=max_tokens, - max_completion_tokens=max_completion_tokens, - modalities=modalities, - prediction=prediction, - audio=audio, - presence_penalty=presence_penalty, - frequency_penalty=frequency_penalty, - logit_bias=logit_bias, - user=user, - response_format=response_format, - seed=seed, - tools=tools, - tool_choice=tool_choice, - parallel_tool_calls=parallel_tool_calls, - logprobs=logprobs, - top_logprobs=top_logprobs, - deployment_id=deployment_id, - reasoning_effort=reasoning_effort, - verbosity=verbosity, - safety_identifier=safety_identifier, - service_tier=service_tier, - store=store, - prompt_cache_key=prompt_cache_key, - base_url=base_url, - api_version=api_version, - api_key=api_key, - model_list=model_list, - extra_headers=extra_headers, - thinking=thinking, - web_search_options=web_search_options, - shared_session=shared_session, - enable_json_schema_validation=enable_json_schema_validation, - **kwargs, - ) + return acompletion_with_mcp( # pyright: ignore[reportReturnType] # MCP path returns a coroutine that acompletion() awaits; completion()'s sync return type omits it + model=model, + messages=messages, + functions=functions, + function_call=function_call, + timeout=timeout, + temperature=temperature, + top_p=top_p, + n=n, + stream=stream, + stream_options=stream_options, + stop=stop, + max_tokens=max_tokens, + max_completion_tokens=max_completion_tokens, + modalities=modalities, + prediction=prediction, + audio=audio, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + logit_bias=logit_bias, + user=user, + response_format=response_format, + seed=seed, + tools=tools, + tool_choice=tool_choice, + parallel_tool_calls=parallel_tool_calls, + logprobs=logprobs, + top_logprobs=top_logprobs, + deployment_id=deployment_id, + reasoning_effort=reasoning_effort, + verbosity=verbosity, + safety_identifier=safety_identifier, + service_tier=service_tier, + store=store, + prompt_cache_key=prompt_cache_key, + base_url=base_url, + api_version=api_version, + api_key=api_key, + model_list=model_list, + extra_headers=extra_headers, + thinking=thinking, + web_search_options=web_search_options, + shared_session=shared_session, + enable_json_schema_validation=enable_json_schema_validation, + **kwargs, + ) api_base = kwargs.get("api_base", None) mock_response: Final[MOCK_RESPONSE_TYPE | None] = kwargs.get("mock_response", None) mock_tool_calls: Final = kwargs.get("mock_tool_calls", None) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 1948597aaf1..a858b91c4c8 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -41,6 +41,7 @@ from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PromptObject, @@ -1112,11 +1113,7 @@ def _responses_try_dispatch_mcp_gateway( | None ): """Return a response when MCP gateway handles the call; otherwise None.""" - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, - ) - - if skip_mcp_handler or not LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway(tools=tools): + if skip_mcp_handler or not may_name_litellm_mcp_gateway(tools): return None mcp_call_kwargs: Final = { "model": model, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index 1dfa4b2abcb..e8a9213d4cf 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -1,4 +1,3 @@ -import re import traceback from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping, Sequence from datetime import datetime @@ -20,6 +19,12 @@ from litellm.proxy._experimental.mcp_server.utils import ( strip_known_server_prefix, ) from litellm.responses.main import aresponses # noqa: TID251 # inner call must skip the MCP gateway that invoked it +from litellm.responses.mcp.tool_precheck import ( + LITELLM_PROXY_MCP_SERVER_URL_PREFIX, + may_name_litellm_mcp_gateway, + names_gateway_explicitly, + proxy_path_mcp_name, +) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( ResponseInputParam, @@ -65,29 +70,6 @@ class _AresponsesOptions(ExtraItemsTypedDict, total=False, extra_items=object): pass -LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" -LITELLM_PROXY_MCP_SERVER_URL_PREFIX: Final = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/" - -_PROXY_MCP_PATH_RE: Final = re.compile(r"^https?://.+/mcp/([^/]+)$") - - -def _mcp_server_url(tool: ToolParam) -> str | None: - if not isinstance(tool, dict) or tool.get("type") != "mcp": - return None - server_url: Final = tool.get("server_url") - return server_url if isinstance(server_url, str) else None - - -def _names_gateway_explicitly(tool: ToolParam) -> bool: - return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL) - - -def _proxy_path_mcp_name(tool: ToolParam) -> str | None: - server_url: Final = _mcp_server_url(tool) - match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url) - return None if match is None else match.group(1) - - def _registered_mcp_servers() -> Collection[MCPServer]: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, @@ -133,7 +115,7 @@ async def _gateway_served_names( async def _served_mcp_path_names( tools: Collection[ToolParam], served_names: Callable[[Collection[str]], Awaitable[frozenset[str]]] ) -> frozenset[str]: - names: Final = frozenset(name for name in map(_proxy_path_mcp_name, tools) if name is not None) + names: Final = frozenset(name for name in map(proxy_path_mcp_name, tools) if name is not None) return await served_names(names) if names else frozenset[str]() @@ -162,17 +144,15 @@ class LiteLLM_Proxy_MCP_Handler: @staticmethod def should_use_litellm_mcp_gateway(tools: Iterable[ToolParam] | None) -> bool: - """True when a tool may name this gateway: server_url "litellm_proxy..." or an http(s) URL ending in - /mcp/. `_split_mcp_tools` then settles which of the latter the gateway actually serves.""" - return any(_names_gateway_explicitly(tool) or _proxy_path_mcp_name(tool) is not None for tool in tools or ()) + return may_name_litellm_mcp_gateway(tools) _should_use_litellm_mcp_gateway = should_use_litellm_mcp_gateway @staticmethod def _parse_mcp_tools(tools: Iterable[Mapping[str, object]] | None) -> SplitTools: items: Final = tuple(tools or ()) - gateway_tools: Final[list[ToolParam]] = [tool for tool in items if _names_gateway_explicitly(tool)] - other_tools: Final[list[ToolParam]] = [tool for tool in items if not _names_gateway_explicitly(tool)] + gateway_tools: Final[list[ToolParam]] = [tool for tool in items if names_gateway_explicitly(tool)] + other_tools: Final[list[ToolParam]] = [tool for tool in items if not names_gateway_explicitly(tool)] return gateway_tools, other_tools @staticmethod @@ -185,7 +165,7 @@ class LiteLLM_Proxy_MCP_Handler: return LiteLLM_Proxy_MCP_Handler._parse_mcp_tools( [ {**tool, "server_url": f"{LITELLM_PROXY_MCP_SERVER_URL_PREFIX}{name}"} - if (name := _proxy_path_mcp_name(tool)) in served + if (name := proxy_path_mcp_name(tool)) in served else tool for tool in items ] @@ -201,7 +181,7 @@ class LiteLLM_Proxy_MCP_Handler: ) -> tuple[bool, ...]: items: Final = tuple(tools or ()) served: Final = await _served_mcp_path_names(items, served_names) - return tuple(_names_gateway_explicitly(tool) or _proxy_path_mcp_name(tool) in served for tool in items) + return tuple(names_gateway_explicitly(tool) or proxy_path_mcp_name(tool) in served for tool in items) @staticmethod async def _apply_toolset_permissions( diff --git a/litellm/responses/mcp/tool_precheck.py b/litellm/responses/mcp/tool_precheck.py new file mode 100644 index 00000000000..b76f4066e6d --- /dev/null +++ b/litellm/responses/mcp/tool_precheck.py @@ -0,0 +1,43 @@ +import importlib +import re +from collections.abc import Iterable +from typing import Final + +LITELLM_PROXY_MCP_SERVER_URL: Final = "litellm_proxy" +LITELLM_PROXY_MCP_SERVER_URL_PREFIX: Final = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/" + +_PROXY_MCP_PATH_RE: Final = re.compile(r"^https?://.+/mcp/([^/]+)$") +_GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_server_manager" + + +def _mcp_server_url(tool: object) -> str | None: + match tool: + case {"type": "mcp", "server_url": str() as server_url} if isinstance(tool, dict): + return server_url + case _: + return None + + +def names_gateway_explicitly(tool: object) -> bool: + return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL) + + +def proxy_path_mcp_name(tool: object) -> str | None: + server_url: Final = _mcp_server_url(tool) + match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url) + return None if match is None else match.group(1) + + +def _gateway_registry_loads() -> bool: + try: + importlib.import_module(_GATEWAY_REGISTRY_MODULE) + except ImportError: + return False + return True + + +def may_name_litellm_mcp_gateway(tools: Iterable[object] | None) -> bool: + items: Final = tuple(tools or ()) + if any(map(names_gateway_explicitly, items)): + return True + return any(proxy_path_mcp_name(tool) is not None for tool in items) and _gateway_registry_loads() diff --git a/tests/base_sdk_tests/check_base_sdk_install.py b/tests/base_sdk_tests/check_base_sdk_install.py index 3abea2a9707..4abd71e39a6 100644 --- a/tests/base_sdk_tests/check_base_sdk_install.py +++ b/tests/base_sdk_tests/check_base_sdk_install.py @@ -102,6 +102,29 @@ def check_completion() -> str: return "mock completion round-trips" +def check_completion_with_function_tool() -> str: + import litellm + + response: Final = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "ping"}], + tools=[ + { + "type": "function", + "function": { + "name": "noop", + "description": "No operation", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + mock_response="pong", + ) + content: Final = response.choices[0].message.content + _require(content == "pong", f"mock completion with a function tool returned {content!r}") + return "mock completion with a function tool round-trips" + + def check_mcp_install_guidance() -> str: try: import litellm.experimental_mcp_client @@ -209,6 +232,7 @@ CHECKS: tuple[tuple[str, Callable[[], str]], ...] = ( ("import litellm", check_import), ("optional MCP installation guidance", check_mcp_install_guidance), ("chat completion", check_completion), + ("chat completion with a function tool", check_completion_with_function_tool), ("embedding", check_embedding), ("bundled model metadata", check_bundled_model_metadata), ("token counter", check_token_counter), diff --git a/tests/base_sdk_tests/check_sdk_http.py b/tests/base_sdk_tests/check_sdk_http.py index c93306cdced..f7a1e30ac99 100644 --- a/tests/base_sdk_tests/check_sdk_http.py +++ b/tests/base_sdk_tests/check_sdk_http.py @@ -89,7 +89,6 @@ def payload(response: object) -> Mapping[str, JsonValue]: return JSON_OBJECT.validate_json(response.model_dump_json(exclude_none=True)) -@skipIf(importlib.util.find_spec("mcp") is None, "BUG: LIT-9333 completion(tools=...) imports optional mcp") def check_tools(base: str, asynchronous: bool = False) -> None: tools: Final = [ { @@ -122,7 +121,6 @@ def check_tools(base: str, asynchronous: bool = False) -> None: assert json.loads(body) == {"model": "gpt-6", "messages": MESSAGES, "tools": tools} -@skipIf(importlib.util.find_spec("mcp") is None, "BUG: LIT-9333 responses() imports optional mcp") def check_responses(base: str, asynchronous: bool = False) -> None: output: Final[Sequence[JsonValue]] = [ { @@ -144,14 +142,62 @@ def check_responses(base: str, asynchronous: bool = False) -> None: "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, } ) - arguments: Final = dict(model="openai/gpt-6", input="ping", api_key="test-key", api_base=base + "/v1") + tools: Final = [ + {"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"}, + {"type": "mcp", "server_label": "repos", "server_url": "https://mcp.example.com/mcp/repos"}, + ] + arguments: Final = dict(model="openai/gpt-6", input="ping", tools=tools, api_key="test-key", api_base=base + "/v1") response: Final = asyncio.run(litellm.aresponses(**arguments)) if asynchronous else litellm.responses(**arguments) assert payload(response)["output"] == output assert payload(response)["usage"] == {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5} path, headers, body = REQUESTS.get(timeout=10) assert path == "/v1/responses" assert headers["authorization"] == "Bearer test-key" - assert json.loads(body) == {"model": "gpt-6", "input": "ping"} + assert json.loads(body) == {"model": "gpt-6", "input": "ping", "tools": tools} + + +def check_messages_tools(base: str) -> None: + tools: Final = [ + { + "name": "lookup", + "description": "Look up the forecast for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ] + content: Final = [{"type": "tool_use", "id": "toolu-http-check", "name": "lookup", "input": {"city": "Paris"}}] + enqueue( + { + "id": "msg-tools-http-check", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, + } + ) + + async def send() -> object: + create: Final = cast( # cast-ok: SDK facade exposes untyped kwargs + Callable[..., Awaitable[object]], litellm.anthropic.messages.acreate + ) + return await create( + model="anthropic/claude-sonnet-5-5", + messages=MESSAGES, + tools=tools, + max_tokens=16, + api_key="test-key", + api_base=base, + ) + + response: Final = JSON_OBJECT.validate_python(asyncio.run(send())) + assert response["content"] == content, response + path, headers, body = REQUESTS.get(timeout=10) + assert path == "/v1/messages" + assert headers["x-api-key"] == "test-key" + sent: Final = JSON_OBJECT.validate_json(body) + assert (sent["model"], sent["messages"], sent["tools"]) == ("claude-sonnet-5-5", MESSAGES, tools), sent @skipIf(importlib.util.find_spec("fastapi") is None, "BUG: LIT-9333 cache hits import proxy-only fastapi") @@ -392,6 +438,7 @@ def main() -> None: partial(check_tools, asynchronous=True), check_responses, partial(check_responses, asynchronous=True), + check_messages_tools, check_cache_hit, check_messages_stream_and_callback, ): diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index c3bccec1f8e..c1517a48c1c 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -468,6 +468,61 @@ def test_server_scoped_gateway_url_exposes_only_that_servers_tools(gateway: Gate assert [call["body"]["params"]["name"] for call in _peer_add_calls(rig.peer)] == ["add"] +@pytest.mark.parametrize("surface", ("chat", "responses", "messages")) +def test_proxy_path_url_of_a_served_server_is_listed_and_executed_by_the_gateway( + gateway: Gateway, surface: Surface +) -> None: + with _rig(gateway, surface) as rig: + server_url: Final = f"{str(gateway.client.base_url).rstrip('/')}/mcp/{rig.alias}" + response: Final = rig.send(_granted_key(rig), [{**AUTO, "server_url": server_url}]) + assert response.status_code == 200, response.text + requests: Final = rig.upstream_tools() + assert len(requests) == 2 and all(rig.tool in names for names in requests), requests + peer_calls: Final = tuple(JSON_OBJECT.validate_python(call["body"]) for call in _peer_add_calls(rig.peer)) + assert [object_value(call["params"])["name"] for call in peer_calls] == ["add"], peer_calls + assert rig.final_text(JSON_OBJECT.validate_json(response.content)) == ANSWER, response.text + + +LOOKUP_PARAMETERS: Final = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} +CALLER_FUNCTION_TOOLS: Final[Mapping[Surface, Mapping[str, object]]] = { + "chat": {"type": "function", "function": {"name": "lookup", "parameters": LOOKUP_PARAMETERS}}, + "responses": {"type": "function", "name": "lookup", "parameters": LOOKUP_PARAMETERS}, + "messages": {"name": "lookup", "input_schema": LOOKUP_PARAMETERS}, +} + + +@pytest.mark.parametrize("surface", ("chat", "responses", "messages")) +def test_tools_that_do_not_name_the_gateway_reach_the_provider_unchanged_and_never_the_peer( + gateway: Gateway, surface: Surface +) -> None: + with _rig(gateway, surface) as rig: + remote_mcp_tools: Final = [ + { + "type": "mcp", + "server_label": "docs", + "server_url": "https://mcp.example.com/sse", + "require_approval": "never", + }, + { + "type": "mcp", + "server_label": "repos", + "server_url": f"https://mcp.example.com/mcp/unserved{uuid.uuid4().hex[:8]}", + "require_approval": "never", + }, + ] + response: Final = rig.send(_granted_key(rig), [CALLER_FUNCTION_TOOLS[surface], *remote_mcp_tools]) + assert response.status_code == 200, response.text + bodies: Final = tuple( + JSON_OBJECT.validate_json(request.body) for request in rig.wire.drain() if request.method == "POST" + ) + assert len(bodies) == 1, bodies + forwarded: Final = bodies[0].get("tools") + assert isinstance(forwarded, list) and len(forwarded) == 3, forwarded + assert [tool for tool in forwarded if isinstance(tool, dict) and tool.get("type") == "mcp"] == remote_mcp_tools + assert "lookup" in json.dumps([tool for tool in forwarded if tool not in remote_mcp_tools]), forwarded + assert tool_calls(rig.peer.drain()) == (), "a tool that does not name the gateway reached the peer" + + def test_streaming_chat_executes_the_tool_once_and_streams_the_follow_up(gateway: Gateway) -> None: with _rig(gateway, "chat") as rig: key: Final = _granted_key(rig) diff --git a/tests/unit/responses/mcp/test_tool_precheck.py b/tests/unit/responses/mcp/test_tool_precheck.py new file mode 100644 index 00000000000..832151ce2a6 --- /dev/null +++ b/tests/unit/responses/mcp/test_tool_precheck.py @@ -0,0 +1,318 @@ +import json +import sys +from collections import deque +from collections.abc import Callable, Iterable, Mapping, Sequence +from importlib.abc import MetaPathFinder +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway + +GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy._experimental.mcp_server.mcp_server_manager" +GATEWAY_HANDLER_MODULES: Final = ( + "litellm.responses.mcp.litellm_proxy_mcp_handler", + "litellm.responses.mcp.chat_completions_handler", + "litellm.llms.anthropic.pass_through.messages.mcp_handler", +) + +FUNCTION_TOOL: Final = { + "type": "function", + "function": {"name": "noop", "description": "No operation", "parameters": {"type": "object", "properties": {}}}, +} +RESPONSES_FUNCTION_TOOL: Final = { + "type": "function", + "name": "noop", + "description": "No operation", + "parameters": {"type": "object", "properties": {}}, +} +ANTHROPIC_FUNCTION_TOOL: Final = { + "name": "noop", + "description": "No operation", + "input_schema": {"type": "object", "properties": {}}, +} +ANTHROPIC_MESSAGE_PAYLOAD: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} +GATEWAY_MCP_TOOL: Final = {"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"} +GATEWAY_SERVER_MCP_TOOL: Final = {"type": "mcp", "server_label": "x", "server_url": "litellm_proxy/mcp/github"} +GATEWAY_PATH_MCP_TOOL: Final = {"type": "mcp", "server_label": "x", "server_url": "https://gw.example.com/mcp/github"} +REMOTE_MCP_TOOL: Final = { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/sse", + "require_approval": "never", +} + +RESPONSES_PAYLOAD: Final = { + "id": "resp_1", + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + "tools": [], +} + + +@pytest.fixture +def gateway_handler_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + for module_name in GATEWAY_HANDLER_MODULES: + monkeypatch.setitem(sys.modules, module_name, None) + + +class _FailingImport(MetaPathFinder): + def __init__(self, module_name: str, error: ImportError) -> None: + self.module_name: Final = module_name + self.error: Final = error + + def find_spec( + self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = None + ) -> ModuleSpec | None: + if fullname == self.module_name: + raise self.error + return None + + +@pytest.fixture( + params=[ + ModuleNotFoundError("No module named 'mcp'"), + ImportError("cannot import name 'CacheableResult' from 'mcp.types'"), + ModuleNotFoundError("No module named 'fastapi'"), + ], + ids=["mcp-missing", "mcp-1x", "proxy-extras-missing"], +) +def gateway_registry_unavailable( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, gateway_handler_unavailable: None +) -> None: + monkeypatch.delitem(sys.modules, GATEWAY_REGISTRY_MODULE, raising=False) + monkeypatch.setattr(sys, "meta_path", [_FailingImport(GATEWAY_REGISTRY_MODULE, request.param), *sys.meta_path]) + + +class _ToolList(list[Mapping[str, object]]): + pass + + +@pytest.mark.parametrize( + "tools", + [[FUNCTION_TOOL], [REMOTE_MCP_TOOL], _ToolList([FUNCTION_TOOL]), (FUNCTION_TOOL, REMOTE_MCP_TOOL)], + ids=["function-tool", "remote-mcp-tool", "list-subclass", "tuple"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_completion_without_a_gateway_tool_does_not_import_the_gateway_handler(tools: Iterable[Mapping[str, object]]): + response: Final = litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=tools, + mock_response="ok", + ) + + assert response.choices[0].message.content == "ok" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gateway_handler_unavailable") +async def test_acompletion_with_function_tool_does_not_import_the_gateway_handler(): + response: Final = await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=[FUNCTION_TOOL], + mock_response="ok", + ) + + assert response.choices[0].message.content == "ok" + + +@pytest.mark.parametrize( + "tool", + [GATEWAY_MCP_TOOL, GATEWAY_SERVER_MCP_TOOL, GATEWAY_PATH_MCP_TOOL], + ids=["litellm-proxy", "litellm-proxy-server", "gateway-path-url"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_completion_with_gateway_mcp_tool_still_reaches_the_handler(tool: Mapping[str, object]): + with pytest.raises(ModuleNotFoundError, match=r"litellm_proxy_mcp_handler|chat_completions_handler"): + litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=[tool], + mock_response="ok", + ) + + +@pytest.mark.parametrize( + "tools", + [None, [RESPONSES_FUNCTION_TOOL], [REMOTE_MCP_TOOL]], + ids=["no-tools", "function-tool", "remote-mcp-tool"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_responses_without_a_gateway_tool_does_not_import_the_gateway_handler( + tools: list[Mapping[str, object]] | None, respx_mock: respx.MockRouter +): + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=RESPONSES_PAYLOAD) + ) + + response: Final = litellm.responses(model="openai/gpt-4o-mini", input="hi", tools=tools, api_key="sk-test") + + assert upstream.called + assert response.output[0].content[0].text == "Done." + + +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_responses_with_gateway_mcp_tool_still_reaches_the_handler(): + with pytest.raises(litellm.APIConnectionError, match=r"litellm_proxy_mcp_handler"): + litellm.responses(model="openai/gpt-4o-mini", input="hi", tools=[GATEWAY_MCP_TOOL], api_key="sk-test") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gateway_handler_unavailable") +async def test_anthropic_messages_with_function_tool_does_not_import_the_gateway_handler( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + return_value=httpx.Response(200, json=ANTHROPIC_MESSAGE_PAYLOAD) + ) + + response: Final = await litellm.anthropic_interface.messages.acreate( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + tools=[ANTHROPIC_FUNCTION_TOOL], + api_key="sk-test", + ) + + assert upstream.called + assert response["content"][0]["text"] == "ok" + + +@pytest.mark.parametrize( + "tools, expected", + [ + (None, False), + ([], False), + ([FUNCTION_TOOL], False), + ([RESPONSES_FUNCTION_TOOL], False), + ([{"type": "web_search"}], False), + (["not-a-mapping"], False), + ([REMOTE_MCP_TOOL], False), + ([{"type": "mcp", "server_label": "x"}], False), + ([{"type": "mcp", "server_label": "x", "server_url": None}], False), + ([{"server_url": "litellm_proxy"}], False), + ([FUNCTION_TOOL, GATEWAY_MCP_TOOL], True), + ([GATEWAY_SERVER_MCP_TOOL], True), + ([GATEWAY_PATH_MCP_TOOL], True), + ([REMOTE_MCP_TOOL, GATEWAY_PATH_MCP_TOOL], True), + ], + ids=[ + "none", + "empty", + "chat-function", + "responses-function", + "web-search", + "non-mapping", + "remote-mcp", + "mcp-without-url", + "mcp-null-url", + "gateway-url-without-mcp-type", + "gateway-among-functions", + "gateway-server", + "gateway-path-url", + "gateway-path-after-remote-mcp", + ], +) +def test_may_name_litellm_mcp_gateway_matches_only_gateway_mcp_tools(tools: list[object] | None, expected: bool): + assert may_name_litellm_mcp_gateway(tools) is expected + + +@pytest.mark.parametrize( + "tools, expected", + [ + ([GATEWAY_PATH_MCP_TOOL], False), + ([FUNCTION_TOOL, GATEWAY_PATH_MCP_TOOL], False), + ([GATEWAY_MCP_TOOL], True), + ([GATEWAY_SERVER_MCP_TOOL, GATEWAY_PATH_MCP_TOOL], True), + ], + ids=["gateway-path-url", "gateway-path-among-functions", "litellm-proxy", "litellm-proxy-server-and-path-url"], +) +@pytest.mark.usefixtures("gateway_registry_unavailable") +def test_proxy_path_url_names_the_gateway_only_when_its_registry_loads(tools: list[object], expected: bool): + assert may_name_litellm_mcp_gateway(tools) is expected + + +@pytest.mark.usefixtures("gateway_registry_unavailable") +def test_responses_sends_a_proxy_path_mcp_url_to_the_provider_without_the_gateway_registry( + respx_mock: respx.MockRouter, +): + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=RESPONSES_PAYLOAD) + ) + + response: Final = litellm.responses( + model="openai/gpt-4o-mini", input="hi", tools=[GATEWAY_PATH_MCP_TOOL], api_key="sk-test" + ) + + assert response.output[0].content[0].text == "Done." + assert json.loads(upstream.calls.last.request.content)["tools"] == [GATEWAY_PATH_MCP_TOOL] + + +@pytest.mark.usefixtures("gateway_registry_unavailable") +def test_completion_sends_a_proxy_path_mcp_url_to_the_provider_without_the_gateway_registry(): + response: Final = litellm.completion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + tools=[GATEWAY_PATH_MCP_TOOL], + mock_response="ok", + ) + + assert response.choices[0].message.content == "ok" + + +class _ToolTuple(tuple[Mapping[str, object], ...]): + pass + + +@pytest.mark.parametrize( + "make_tools", + [ + lambda items: (item for item in items), + lambda items: iter(items), + lambda items: deque(items), + lambda items: _ToolList(items), + lambda items: _ToolTuple(items), + ], + ids=["generator", "iterator", "deque", "list-subclass", "tuple-subclass"], +) +@pytest.mark.usefixtures("gateway_handler_unavailable") +def test_precheck_reads_every_iterable_shape_without_the_gateway_handler( + make_tools: Callable[[list[Mapping[str, object]]], Iterable[Mapping[str, object]]], +): + assert may_name_litellm_mcp_gateway(make_tools([FUNCTION_TOOL, REMOTE_MCP_TOOL])) is False + assert may_name_litellm_mcp_gateway(make_tools([FUNCTION_TOOL, GATEWAY_MCP_TOOL])) is True diff --git a/tests/unit/responses/test_responses_prompt_management.py b/tests/unit/responses/test_responses_prompt_management.py index 1e8880424ea..9af92e7f9ca 100644 --- a/tests/unit/responses/test_responses_prompt_management.py +++ b/tests/unit/responses/test_responses_prompt_management.py @@ -70,10 +70,6 @@ def _patch_responses_dispatch(): import_module("litellm.responses.main").litellm, "get_llm_provider", side_effect=_provider_by_model, ), - patch.object( - import_module("litellm.responses.mcp.litellm_proxy_mcp_handler").LiteLLM_Proxy_MCP_Handler, "should_use_litellm_mcp_gateway", - return_value=False, - ), patch.object( import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config", return_value=None, @@ -158,7 +154,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -193,7 +189,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -216,7 +212,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -247,7 +243,7 @@ class TestResponsesAPIPromptManagement: logging_obj.model_call_details = {} patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -273,7 +269,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -305,7 +301,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm litellm.responses( @@ -328,7 +324,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -368,7 +364,7 @@ class TestResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm litellm.responses( @@ -399,8 +395,7 @@ class TestResponsesAPIPromptManagement: side_effect=_provider_by_model, ), patches[1], - patches[2], - patches[3] as mock_handler, + patches[2] as mock_handler, ): import litellm @@ -437,7 +432,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm await litellm.aresponses( @@ -467,7 +462,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm await litellm.aresponses( @@ -498,7 +493,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3]: + with patches[0], patches[1], patches[2]: import litellm await litellm.aresponses( @@ -524,7 +519,7 @@ class TestAsyncResponsesAPIPromptManagement: ) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3] as mock_handler: + with patches[0], patches[1], patches[2] as mock_handler: import litellm await litellm.aresponses( @@ -657,7 +652,7 @@ async def test_async_guardrail_block_from_prompt_hook_reaches_caller_unwrapped() logging_obj.async_get_chat_completion_prompt = AsyncMock(side_effect=block) patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3], pytest.raises(HTTPException) as exc_info: + with patches[0], patches[1], patches[2], pytest.raises(HTTPException) as exc_info: await litellm.aresponses(input="Hi", model="gpt-4o", prompt_id="blocked", litellm_logging_obj=logging_obj) assert exc_info.value is block @@ -669,7 +664,7 @@ def test_sync_guardrail_block_from_prompt_hook_reaches_caller_unwrapped(): logging_obj.get_chat_completion_prompt.side_effect = block patches = _patch_responses_dispatch() - with patches[0], patches[1], patches[2], patches[3], pytest.raises(HTTPException) as exc_info: + with patches[0], patches[1], patches[2], pytest.raises(HTTPException) as exc_info: litellm.responses(input="Hi", model="gpt-4o", prompt_id="blocked", litellm_logging_obj=logging_obj) assert exc_info.value is block From ca4e6521e70b9ecdc13bf747b0c1fc48b174fcce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:47:28 +0000 Subject: [PATCH 27/28] fix(router): serve a team's wildcard deployment before a proxy-wide wildcard (#48490) Backport of #48490 to lts/1.108.x. Cherry-picked from 76d001a7c06d5dbf93f9191b247db0ef981b6c66 (main). The line has no wildcard_routed_models helper and no get_models_by_pattern, so only wildcard_route_deployments is changed. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 12 +++---- .../test_ptu_consumption.py | 16 +++------ tests/unit/test_router/test_router.py | 36 ++++++++++++++++++- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 244fac94d42..52b131362b1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12915,13 +12915,13 @@ class Router: return [{**DeploymentTypedDict(**m), "model_name": model_name} for m in matches] def wildcard_route_deployments(self, model: str, team_id: str | None) -> Sequence[Mapping[str, object]]: - """The deployments routing serves ``model`` from through a wildcard: the most specific - proxy-wide pattern matching it, else ``team_id``'s own pattern, never both.""" - proxy_wide: Final[Sequence[Mapping[str, object]]] = self.pattern_router.get_deployments_by_pattern(model=model) + """The deployments routing serves ``model`` from through a wildcard: ``team_id``'s own pattern + matching it, else the most specific proxy-wide pattern, never both.""" team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None - if proxy_wide or team_router is None: - return proxy_wide - return team_router.get_deployments_by_pattern(model=model) + team_owned: Final[Sequence[Mapping[str, object]]] = ( + team_router.get_deployments_by_pattern(model=model) if team_router is not None else () + ) + return team_owned or self.pattern_router.get_deployments_by_pattern(model=model) def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, diff --git a/tests/unit/proxy/management_endpoints/test_ptu_consumption.py b/tests/unit/proxy/management_endpoints/test_ptu_consumption.py index a446afbd809..065ec93dcd0 100644 --- a/tests/unit/proxy/management_endpoints/test_ptu_consumption.py +++ b/tests/unit/proxy/management_endpoints/test_ptu_consumption.py @@ -232,14 +232,10 @@ def test_rows_keyed_by_a_name_a_wildcard_route_serves_are_sized_by_its_shared_de assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx(1.0) -@pytest.mark.parametrize( - ("catch_all_routes", "expected_ptu_hours"), [(False, 1.0), (True, 0.0)], ids=["team-wildcard", "behind-catch-all"] -) -def test_a_teams_wildcard_reservation_reads_ptu_hours_only_while_the_router_serves_from_it( - monkeypatch, catch_all_routes, expected_ptu_hours -): - """The router tries the proxy-wide wildcards before a team's own, so a team's `ptu-*` - reservation behind an open `*` route never serves `ptu-chat`, and its row reads no PTU-hours.""" +@pytest.mark.parametrize("catch_all_routes", [False, True], ids=["team-wildcard", "beside-catch-all"]) +def test_a_teams_wildcard_reservation_reads_ptu_hours_even_beside_a_proxy_wide_catch_all(monkeypatch, catch_all_routes): + """The router tries a team's own wildcard before the proxy-wide ones, so a team's `ptu-*` + reservation serves `ptu-chat` whether or not an open `*` route exists, and its row reads its PTU-hours.""" monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "True") reservation: Final = { "model_name": "ptu-team-r", @@ -264,9 +260,7 @@ def test_a_teams_wildcard_reservation_reads_ptu_hours_only_while_the_router_serv attached: Final = with_ptu_consumption(_response(_day("2026-09-23", {"ptu-chat": one_hour})), router, "team-r") - assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx( - expected_ptu_hours - ) + assert attached.results[0].breakdown.model_groups["ptu-chat"].metrics.ptu_hours == pytest.approx(1.0) def _mixed_ptu_router() -> Router: diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 8cde45e7603..fe9541cbe21 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -7371,7 +7371,8 @@ def test_get_wildcard_deployment_usable_by_team_prefers_the_team_pattern(): pytest.param(False, "rwild-chat", "team-r", ("team-r-wildcard",), id="team-pattern-when-no-proxy-wide-match"), pytest.param(False, "rwild-chat", "team-b", (), id="another-teams-pattern-never-serves"), pytest.param(False, "rwild-chat", None, (), id="no-team-no-team-pattern"), - pytest.param(True, "rwild-chat", "team-r", ("catch-all",), id="proxy-wide-pattern-beats-the-team-pattern"), + pytest.param(True, "rwild-chat", "team-r", ("team-r-wildcard",), id="team-pattern-beats-the-proxy-wide-pattern"), + pytest.param(True, "rwild-chat", "team-b", ("catch-all",), id="another-team-falls-through-to-proxy-wide"), pytest.param(True, "ptu-chat", "team-r", ("ptu-wildcard",), id="only-the-most-specific-proxy-wide-pattern"), ], ) @@ -7407,6 +7408,39 @@ def test_wildcard_route_deployments_is_the_one_pattern_routing_serves_from( assert tuple(Deployment.model_validate(d).model_info.id for d in early_deployments) == expected_ids +@pytest.mark.parametrize( + ("team_id", "expected_id"), + [ + pytest.param("team-a", "team-a-byok", id="team-key-uses-its-own-byok-wildcard"), + pytest.param("team-b", "shared-openrouter", id="team-without-byok-uses-the-shared-wildcard"), + pytest.param(None, "shared-openrouter", id="key-without-team-uses-the-shared-wildcard"), + ], +) +def test_team_byok_wildcard_is_served_before_a_proxy_wide_wildcard_of_the_same_name( + team_id: str | None, expected_id: str +): + """Regression: a proxy-wide openrouter/* shadowed a team's own openrouter/* BYOK deployment for that team's keys.""" + router: Final = litellm.Router( + model_list=[ + { + "model_name": "model_name_team-a_internal", + "litellm_params": {"model": "openrouter/*", "api_key": "sk-team-a"}, + "model_info": {"id": "team-a-byok", "team_id": "team-a", "team_public_model_name": "openrouter/*"}, + }, + { + "model_name": "openrouter/*", + "litellm_params": {"model": "openrouter/*", "api_key": "sk-shared"}, + "model_info": {"id": "shared-openrouter"}, + }, + ] + ) + requested: Final = "openrouter/openai/gpt-5-nano" + + served: Final = router.wildcard_route_deployments(requested, team_id) + + assert tuple(Deployment.model_validate(d).model_info.id for d in served) == (expected_id,) + + def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint(): """ Test that get_deployment_credentials_with_provider correctly copies From 854937381d0dd5ac5a7abf8421b7220d153eba12 Mon Sep 17 00:00:00 2001 From: ekpog200 <62500055+ekpog200@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:42:59 +0300 Subject: [PATCH 28/28] fix(videos)!: keep model_id in proxy video ids so per-model keys resolve on status/content (#44614) Backport of #44614 to lts/1.108.x. Cherry-picked from ba61ecdf570c91a2409560138170ff9ab1fe7ad7 (main). The line has no splits_stream_timeout on _update_kwargs_with_deployment, so only the _video_call_pin wrapping is added around deployment selection. The line has no litellm/llms/base_llm/videos/async_task_utils.py, so its _id_safe_name dedupe is applied to the dashscope video transformation that holds the same helper on this line. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/dashscope/videos/transformation.py | 9 +- litellm/proxy/auth/auth_utils.py | 50 ++- litellm/proxy/video_endpoints/endpoints.py | 92 ++-- litellm/proxy/video_endpoints/utils.py | 115 ++++- litellm/router.py | 95 +++- litellm/router_utils/cooldown_handlers.py | 6 + .../router_utils/pattern_match_deployments.py | 2 +- litellm/types/videos/main.py | 2 + litellm/types/videos/utils.py | 111 +++-- .../router_code_coverage.py | 2 + .../routing/test_video_poll_routing_wire.py | 4 +- tests/unit/conftest.py | 10 +- tests/unit/proxy/auth/test_auth_utils.py | 412 ++++++++++++----- .../proxy/video_endpoints/test_endpoints.py | 162 ++++++- .../unit/proxy/video_endpoints/test_utils.py | 413 +++++++++++++++++- .../test_router_provider_endpoints.py | 247 +++++++++++ tests/unit/test_video_generation.py | 30 ++ 17 files changed, 1533 insertions(+), 229 deletions(-) diff --git a/litellm/llms/dashscope/videos/transformation.py b/litellm/llms/dashscope/videos/transformation.py index 17b685a0e1a..625ce3bab7e 100644 --- a/litellm/llms/dashscope/videos/transformation.py +++ b/litellm/llms/dashscope/videos/transformation.py @@ -35,6 +35,7 @@ from litellm.types.videos.utils import ( decode_video_id_with_provider, encode_video_id_with_provider, extract_original_video_id, + id_safe_name, ) from litellm.utils import get_model_info @@ -599,17 +600,13 @@ def _stamped_poll_models(metadata: object) -> tuple[str, str | None] | None: fields: Final = _PARAMETERS_ADAPTER.validate_python(metadata) except ValidationError: return None - requested: Final = _id_safe_name(fields.get("model_group")) - served: Final = _id_safe_name(fields.get("deployment_model_name")) + requested: Final = id_safe_name(fields.get("model_group")) + served: Final = id_safe_name(fields.get("deployment_model_name")) if requested is None: return None return requested, None if served is None or served == requested or "*" in served else served -def _id_safe_name(name: object) -> str | None: - return name if isinstance(name, str) and name and ";" not in name else None - - class DashScopeVideoConfig(BaseVideoConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseVideoConfig contract returns list return [ diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 98bb0985f4a..d012bbe51f3 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -4,6 +4,7 @@ import re import sys from collections.abc import Collection, Iterator, Mapping from functools import lru_cache +from itertools import accumulate, chain, repeat from logging import Logger from types import MappingProxyType from typing import Any, Final, Protocol @@ -39,6 +40,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, Deployment, server_owned_wif_fields_present from litellm.types.router import reject_server_owned_wif_params as _reject_server_owned_wif_params from litellm.types.utils import CustomPricingLiteLLMParams, oauth_token_exchange_litellm_params +from litellm.types.videos.main import DecodedVideoId def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: @@ -566,6 +568,31 @@ def fallback_target_model_name(target: object) -> str | None: return None +def _direct_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]: + from litellm.proxy.auth.model_checks import get_all_fallbacks + + return frozenset( + name for target in get_all_fallbacks(model, llm_router) if (name := fallback_target_model_name(target)) + ) + + +def _fallback_hop( + frontier_and_reached: tuple[frozenset[str], frozenset[str]], llm_router: Router +) -> tuple[frozenset[str], frozenset[str]]: + frontier, reached = frontier_and_reached + next_hop: Final = ( + frozenset(chain.from_iterable(_direct_fallback_model_names(model, llm_router) for model in frontier)) - reached + ) + return next_hop, reached | next_hop + + +def router_fallback_model_names(model: str, llm_router: Router) -> frozenset[str]: + """The groups a failure on `model` can reach through the router's fallbacks, chained and generic (`*`) ones.""" + hops: Final = accumulate(repeat(llm_router), _fallback_hop, initial=(frozenset({model}), frozenset({model}))) + _, reached = next(hop for hop in hops if not hop[0]) + return reached - {model} + + def request_fallback_model_names(request_body: Mapping[str, object]) -> tuple[str, ...]: return tuple( dict.fromkeys( @@ -1975,10 +2002,10 @@ def _extract_models_from_managed_resource_id( ) if resource_id_field == "video_id": - model_id = decode_video_id_with_provider(resource_id).get("model_id") + decoded_video: Final = decode_video_id_with_provider(resource_id) _append_model_candidates( candidates=candidates, - value=_resolve_model_id_with_router(model_id, llm_router), + value=_video_access_group(decoded_video, llm_router) or decoded_video.get("model_id"), ) else: model_id = decode_character_id_with_provider(resource_id).get("model_id") @@ -1992,6 +2019,25 @@ def _extract_models_from_managed_resource_id( return _dedupe_model_candidates(candidates) +def _video_access_group(decoded_video: DecodedVideoId, llm_router: Router | None) -> str | None: + """ + The group a key must be allowed to call to use a video: the creating deployment's group, or, while router + fallbacks skip model access checks, the group whose router fallback created it. + """ + from litellm.proxy.auth.fallback_model_access import router_fallback_access_check + from litellm.proxy.video_endpoints.utils import video_model_group + + creating_group: Final = video_model_group(decoded_video, llm_router) + requested_model: Final = decoded_video.get("requested_model") + if llm_router is None or creating_group is None or requested_model is None: + return creating_group + if router_fallback_access_check.is_enforced(): + return creating_group + if creating_group in router_fallback_model_names(requested_model, llm_router): + return requested_model + return creating_group + + def _resolve_model_id_with_router(model_id: str | None, llm_router: Router | None) -> str | None: if model_id is None or llm_router is None: return model_id diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index c4f33eae114..d4643994043 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -20,9 +20,14 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.image_endpoints.endpoints import batch_to_bytesio from litellm.proxy.video_endpoints.utils import ( + VideoRequestRoute, encode_character_id_in_response, + encode_video_id_in_response, extract_model_from_target_model_names, + fallback_requested_model, get_custom_provider_from_data, + routable_model_name, + video_model_group, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -84,6 +89,7 @@ async def video_generation( # Read request body data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) if input_reference is not None: input_reference_file: Final = await batch_to_bytesio([input_reference]) if input_reference_file: @@ -118,7 +124,13 @@ async def video_generation( version=version, ) else: - return generated + requested_model: Final = routable_model_name(requested.model, llm_router, requested.custom_llm_provider) + return encode_video_id_in_response( + response=generated, + fallback_provider=requested.custom_llm_provider, + fallback_model_id=requested_model, + requested_model=fallback_requested_model(requested_model, generated, llm_router), + ) @router.get( @@ -269,12 +281,9 @@ async def video_status( if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) @@ -305,7 +314,12 @@ async def video_status( version=version, ) else: - return status + return encode_video_id_in_response( + response=status, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded, + polled_video=decoded, + ) @router.get( @@ -358,7 +372,6 @@ async def video_content( decoded: Final = decode_video_id_with_provider(video_id) provider_from_id: Final = decoded.get("custom_llm_provider") - model_id_from_decoded: Final = decoded.get("model_id") custom_llm_provider: Final = ( get_custom_llm_provider_from_request_headers(request=request) @@ -369,12 +382,9 @@ async def video_content( if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) try: @@ -462,6 +472,7 @@ async def video_remix( ) data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) data["video_id"] = video_id decoded: Final = decode_video_id_with_provider(video_id) @@ -471,18 +482,15 @@ async def video_remix( custom_llm_provider: Final = ( get_custom_llm_provider_from_request_headers(request=request) or get_custom_llm_provider_from_request_query(request=request) - or data.get("custom_llm_provider") + or requested.custom_llm_provider or provider_from_id ) if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) @@ -513,7 +521,13 @@ async def video_remix( version=version, ) else: - return remixed + return encode_video_id_in_response( + response=remixed, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded + or routable_model_name(requested.model, llm_router, custom_llm_provider), + requested_model=decoded.get("requested_model"), + ) @router.post( @@ -771,6 +785,7 @@ async def video_edit( ) data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) uploaded_video: Final = data.pop("video", None) if isinstance(uploaded_video, StarletteUploadFile): video_files: Final = await batch_to_bytesio((uploaded_video,)) @@ -793,10 +808,9 @@ async def video_edit( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) try: @@ -826,7 +840,13 @@ async def video_edit( version=version, ) else: - return edited + return encode_video_id_in_response( + response=edited, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded + or routable_model_name(requested.model, llm_router, custom_llm_provider), + requested_model=decoded.get("requested_model"), + ) @router.post( @@ -875,6 +895,7 @@ async def video_extension( ) data: Final = await read_request_body(request=request) + requested: Final = VideoRequestRoute.model_validate(data) data["video_id"] = video_reference_to_id(data.pop("video", None)) decoded: Final = decode_video_id_with_provider(data["video_id"]) @@ -890,10 +911,9 @@ async def video_extension( ) data["custom_llm_provider"] = custom_llm_provider - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded, provider_from_id) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = video_model_group(decoded, llm_router) + if resolved_model: + data["model"] = resolved_model processor: Final = ProxyBaseLLMRequestProcessing(data=data, served_model=decoded.get("served_model")) try: @@ -923,4 +943,10 @@ async def video_extension( version=version, ) else: - return extended + return encode_video_id_in_response( + response=extended, + fallback_provider=custom_llm_provider, + fallback_model_id=model_id_from_decoded + or routable_model_name(requested.model, llm_router, custom_llm_provider), + requested_model=decoded.get("requested_model"), + ) diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 554f0f6ef53..a2be42f1b76 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -1,8 +1,18 @@ -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final import orjson +from pydantic import BaseModel, ConfigDict -from litellm.types.videos.utils import encode_character_id_with_provider +from litellm.types.videos.main import DecodedVideoId, VideoObject +from litellm.types.videos.utils import ( + decode_video_id_with_provider, + encode_character_id_with_provider, + encode_video_id_with_provider, + id_safe_name, +) + +if TYPE_CHECKING: + from litellm.router import Router def extract_model_from_target_model_names(target_model_names: Any) -> str | None: @@ -65,3 +75,104 @@ def encode_character_id_in_response(response: Any, custom_llm_provider: str, mod model_id=model_id, ) return response + + +class VideoRequestRoute(BaseModel): + model_config = ConfigDict(frozen=True) + + model: object = None + custom_llm_provider: object = None + + +def routable_model_name(model: object, llm_router: "Router | None", custom_llm_provider: object) -> str | None: + name: Final = id_safe_name(model) + if name is None or llm_router is None: + return name + provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None + return name if llm_router.resolve_model_name_from_model_id(name, provider) else None + + +def video_model_group(decoded: DecodedVideoId, llm_router: "Router | None") -> str | None: + """ + The group a video id names. A provider model that several groups share resolves to the group of the + deployment that created the video, under its team's public name for a team model, while a name the router + routes by itself (the id's model, bare or provider-prefixed), or one that routes to that deployment's group + (an alias of it), is kept as written. + """ + if llm_router is None: + return None + model_id: Final = decoded.get("model_id") + provider: Final = decoded.get("custom_llm_provider") + resolved: Final = llm_router.resolve_model_name_from_model_id(model_id, provider) if model_id else None + deployment_id: Final = decoded.get("deployment_id") + if not deployment_id or resolved in (model_id, f"{provider}/{model_id}"): + return resolved + deployment: Final = llm_router.get_deployment(model_id=deployment_id) + if deployment is None: + return resolved + creating_group: Final = deployment.model_info.team_public_model_name or deployment.model_name + if resolved is not None and llm_router.resolve_served_model_name(resolved, creating_group) == creating_group: + return resolved + return creating_group + + +def fallback_requested_model(requested_model: str | None, response: object, llm_router: "Router | None") -> str | None: + """The group a create asked for when one of that group's router fallbacks served it.""" + from litellm.proxy.auth.auth_utils import router_fallback_model_names + + if requested_model is None or llm_router is None or not isinstance(response, VideoObject): + return None + deployment_id: Final = id_safe_name(response.hidden_params.get("model_id")) + deployment: Final = llm_router.get_deployment(model_id=deployment_id) if deployment_id else None + if deployment is None: + return None + creating_group: Final = deployment.model_info.team_public_model_name or deployment.model_name + return requested_model if creating_group in router_fallback_model_names(requested_model, llm_router) else None + + +def encode_video_id_in_response( + response: object, + fallback_provider: object, + fallback_model_id: object, + polled_video: DecodedVideoId | None = None, + requested_model: str | None = None, +) -> object: + """ + Stamps the provider, model and the deployment that answered into a returned video id. A poll of an existing + video (`polled_video`) answered without a routed deployment keeps the deployment its id already named, and + every returned id keeps the group a router fallback was serving (`requested_model`). + """ + if not isinstance(response, VideoObject) or not response.id: + return response + + decoded: Final = decode_video_id_with_provider(response.id) + hidden_params: Final = response.hidden_params + routed_deployment_id: Final = id_safe_name(hidden_params.get("model_id")) + deployment_id: Final = routed_deployment_id or (polled_video or decoded).get("deployment_id") + kept_requested_model: Final = ( + id_safe_name(requested_model) or decoded.get("requested_model") or (polled_video or {}).get("requested_model") + ) + model_id: Final = decoded.get("model_id") or id_safe_name(fallback_model_id) or routed_deployment_id + provider: Final = ( + id_safe_name(decoded.get("custom_llm_provider")) + or id_safe_name(hidden_params.get("custom_llm_provider")) + or id_safe_name(fallback_provider) + ) + if not model_id or not provider: + return response + if ( + model_id == decoded.get("model_id") + and deployment_id == decoded.get("deployment_id") + and kept_requested_model == decoded.get("requested_model") + ): + return response + + restamped_id: Final = encode_video_id_with_provider( + video_id=decoded.get("video_id") or response.id, + provider=provider, + model_id=model_id, + served_model=decoded.get("served_model"), + deployment_id=deployment_id, + requested_model=kept_requested_model, + ) + return response.model_copy(update={"id": restamped_id}) diff --git a/litellm/router.py b/litellm/router.py index 52b131362b1..9880ca6fdd0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -34,6 +34,7 @@ from collections.abc import ( MutableMapping, Sequence, ) +from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache, partial @@ -223,6 +224,7 @@ from litellm.router_utils.cooldown_handlers import ( is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_video_lookup_not_found, set_cooldown_deployments, ) from litellm.router_utils.deployment_pricing import deployment_pricing_models @@ -370,6 +372,7 @@ from litellm.types.utils import ( shared_backend_model_info, ) from litellm.types.utils import ModelInfo as ModelMapInfo +from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES, decode_video_id_with_provider from litellm.utils import ( CustomStreamWrapper, EmbeddingResponse, @@ -488,6 +491,7 @@ def model_info_is_active_for_environment(model_info: Mapping[str, object] | None _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _CallbackT = TypeVar("_CallbackT") +_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object]) _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset({"model", "api_base", "api_key", "api_version"}) _ALIAS_MARKER_FORWARDED_PARAMS_KWARG: Final = "_alias_marker_forwarded_params" @@ -496,6 +500,7 @@ _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600 CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding" _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"}) +_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None) _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType( { "prompt_caching": PromptCachingDeploymentCheck, @@ -5717,17 +5722,19 @@ class Router: passthrough_on_no_deployment: Final = kwargs.pop("passthrough_on_no_deployment", False) function_name: Final = "_ageneric_api_call_with_fallbacks" + handler_name: Final[str | None] = getattr(original_generic_function, "__name__", None) deployment = None # rebind-ok: pre-init so the except block can stamp a failure with no deployment picked try: parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) try: - deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init above - model=model, - request_kwargs=kwargs, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - ) + with self._video_call_pin(handler_name, kwargs): + deployment = await self.async_get_available_deployment( # rebind-ok: set on success, see pre-init + model=model, + request_kwargs=kwargs, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + ) except Exception as e: if passthrough_on_no_deployment: return await original_generic_function(model=model, **kwargs) @@ -6409,6 +6416,37 @@ class Router: ) return await self._ageneric_api_call_with_fallbacks(original_function=original_function, **kwargs) + @staticmethod + @contextlib.contextmanager + def _video_call_pin(handler_name: str | None, request_kwargs: Mapping[str, object]) -> Generator[None]: + """ + Scopes the deployment a video id names to the routing of one call on that video (status, content, remix, + edit, extension). Any other call routes as if no id were sent, whatever its body carries. + """ + video_id: Final = request_kwargs.get("video_id") + deployment_id: Final = ( + decode_video_id_with_provider(video_id).get("deployment_id") + if handler_name in VIDEO_ID_CALL_TYPES and isinstance(video_id, str) + else None + ) + token: Final = _video_call_deployment_id.set(deployment_id) + try: + yield + finally: + _video_call_deployment_id.reset(token) + + @staticmethod + def _narrow_to_video_deployment(healthy_deployments: Sequence[_DeploymentT]) -> Sequence[_DeploymentT]: + """ + A video lives on the provider account of the deployment that created it, so a call on an id naming a + deployment that every routing filter kept goes to that deployment, else to whichever one routing picks. + """ + deployment_id: Final = _video_call_deployment_id.get() + creating: Final = tuple( + deployment for deployment in healthy_deployments if deployment_id in Router._deployment_ids((deployment,)) + ) + return creating or healthy_deployments + def _generic_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs): """ Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router @@ -6430,13 +6468,14 @@ class Router: kwargs=kwargs, metadata_variable_name=metadata_variable_name, ) - deployment: Final = self.get_available_deployment( - model=model, - messages=kwargs.get("messages", None), - input=kwargs.get("input", None), - specific_deployment=kwargs.pop("specific_deployment", None), - request_kwargs=kwargs, - ) + with self._video_call_pin(handler_name, kwargs): + deployment: Final = self.get_available_deployment( + model=model, + messages=kwargs.get("messages", None), + input=kwargs.get("input", None), + specific_deployment=kwargs.pop("specific_deployment", None), + request_kwargs=kwargs, + ) self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="generic_api_call") data: Final = deployment["litellm_params"].copy() @@ -8914,6 +8953,13 @@ class Router: ) return False + if is_video_lookup_not_found(exception, kwargs): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Provider 404 says the video is not on this deployment's account, not that the deployment is down." + ) + return False + exception_status: Final = getattr(exception, "status_code", "") if is_caller_timeout_408(kwargs, exception_status): @@ -12337,7 +12383,8 @@ class Router: 1. First, check if model_id directly matches a model_name or deployment ID 2. If not, search through router's model_list to find a match by litellm_params.model 3. If not, keep model_id when the router routes that name itself (an alias, a routing group, or a wildcard - whose deployment serves `custom_llm_provider`) + whose deployment serves `custom_llm_provider`), else name it `custom_llm_provider/model_id` when only that + form routes to such a wildcard 4. Return the model_name if found, None otherwise Args: @@ -12385,8 +12432,15 @@ class Router: if self.is_recognized_model(model_id): return model_id - if custom_llm_provider is not None and self.pattern_router.routes_to_provider(model_id, custom_llm_provider): + if custom_llm_provider is None: + return None + if self.pattern_router.routes_to_provider(model_id, custom_llm_provider): return model_id + provider_prefixed: Final = f"{custom_llm_provider}/{model_id}" + if not model_id.startswith(f"{custom_llm_provider}/") and self.pattern_router.routes_to_provider( + provider_prefixed, custom_llm_provider + ): + return provider_prefixed return None def resolve_served_model_name(self, model_name: str, served_model: str) -> str: @@ -13897,10 +13951,10 @@ class Router: def _filter_deployments_by_model_access_groups( self, model: str, - healthy_deployments: list, + healthy_deployments: list[DeploymentTypedDict], request_kwargs: Mapping[str, object] | None, request_team_id: str | None, - ) -> list: + ) -> list[DeploymentTypedDict]: """ Restrict candidate deployments to caller-authorized model access groups. @@ -14084,7 +14138,8 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) healthy_deployments = litellm.utils.get_order_filtered_deployments( - cast(list[dict], healthy_deployments), target_order=_target_order + list(self._narrow_to_video_deployment(cast(list[dict[str, object]], healthy_deployments))), + target_order=_target_order, ) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in @@ -15271,7 +15326,7 @@ class Router: ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) _target_order: Final = (request_kwargs or {}).pop("_target_order", None) healthy_deployments = litellm.utils.get_order_filtered_deployments( - healthy_deployments, target_order=_target_order + list(self._narrow_to_video_deployment(healthy_deployments)), target_order=_target_order ) ## WEIGHTED FAILOVER EXCLUSION ## -> drop deployments already tried in diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index cdca5009b0e..20ad8a6e163 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -35,6 +35,7 @@ from litellm.types.utils import ( BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, InternalCallOrigin, ) +from litellm.types.videos.utils import VIDEO_ID_CALL_TYPES from .router_callbacks.track_deployment_metrics import ( get_deployment_failures_for_current_minute, @@ -89,6 +90,11 @@ def is_background_response_poll_not_found(exception: Exception, litellm_params: ) +def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, object]) -> bool: + """Whether a call on an existing video id failed with a provider 404: the video is not on that account.""" + return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES + + _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first. (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"), diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 62ba5592f90..daa052a46e1 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -173,7 +173,7 @@ class PatternMatchRouter: return None # No matching pattern found def routes_to_provider(self, request: str, custom_llm_provider: str) -> bool: - deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.route(request) or ()) + deployments: Final = _ROUTED_DEPLOYMENTS.validate_python(self.get_pattern(request) or ()) return any( (deployment.litellm_params.custom_llm_provider or inferred_provider(deployment.litellm_params.model)) == custom_llm_provider diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index 05444b1e7ec..4d6f8fc6a30 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -113,6 +113,8 @@ class DecodedVideoId(TypedDict, total=False): model_id: str | None video_id: str served_model: ReadOnly[str | None] + deployment_id: ReadOnly[str | None] + requested_model: ReadOnly[str | None] class CharacterObject(LiteLLMBaseModel): diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index 9a07c9ba802..0fc2b8ede1b 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -6,6 +6,7 @@ Format: vid_{base64_encoded_string} """ import base64 +from collections.abc import Sequence from typing import Final from litellm._logging import verbose_logger @@ -14,6 +15,22 @@ from litellm.types.videos.main import DecodedVideoId VIDEO_ID_PREFIX: Final = "video_" VIDEO_ID_SERVED_MODEL_PREFIX: Final = "served_model:" +VIDEO_ID_DEPLOYMENT_PREFIX: Final = "deployment_id:" +VIDEO_ID_REQUESTED_MODEL_PREFIX: Final = "requested_model:" +VIDEO_ID_CALL_TYPES: Final = frozenset( + { + "avideo_status", + "video_status", + "avideo_content", + "video_content", + "avideo_remix", + "video_remix", + "avideo_edit", + "video_edit", + "avideo_extension", + "video_extension", + } +) CHARACTER_ID_PREFIX: Final = "character_" CHARACTER_ID_TEMPLATE: Final = "litellm:custom_llm_provider:{};model_id:{};character_id:{}" @@ -37,7 +54,12 @@ def _add_base64_padding(value: str) -> str: def encode_video_id_with_provider( - video_id: str, provider: str, model_id: str | None = None, served_model: str | None = None + video_id: str, + provider: str, + model_id: str | None = None, + served_model: str | None = None, + deployment_id: str | None = None, + requested_model: str | None = None, ) -> str: """Encode provider and model_id into video_id using base64.""" if not provider or not video_id: @@ -54,12 +76,24 @@ def encode_video_id_with_provider( # ID is not encoded (even if it starts with video_), so encode it assembled_id = str(SpecialEnums.LITELLM_MANAGED_VIDEO_COMPLETE_STR.value).format(provider, model_id or "", video_id) served_part: Final = f";{VIDEO_ID_SERVED_MODEL_PREFIX}{served_model}" if served_model else "" + deployment_part: Final = f";{VIDEO_ID_DEPLOYMENT_PREFIX}{deployment_id}" if deployment_id else "" + requested_part: Final = f";{VIDEO_ID_REQUESTED_MODEL_PREFIX}{requested_model}" if requested_model else "" - base64_encoded_id: Final[str] = base64.b64encode((assembled_id + served_part).encode("utf-8")).decode("utf-8") + base64_encoded_id: Final[str] = base64.b64encode( + (assembled_id + served_part + deployment_part + requested_part).encode("utf-8") + ).decode("utf-8") return f"{VIDEO_ID_PREFIX}{base64_encoded_id}" +def id_safe_name(name: object) -> str | None: + return name if isinstance(name, str) and name and ";" not in name else None + + +def _trailing_part(parts: Sequence[str], prefix: str) -> str | None: + return next((part.removeprefix(prefix) or None for part in parts[3:] if part.startswith(prefix)), None) + + def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: """Decode provider and model_id from encoded video_id.""" if not encoded_video_id: @@ -80,43 +114,6 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: cleaned_id = encoded_video_id.replace(VIDEO_ID_PREFIX, "") cleaned_id = _add_base64_padding(cleaned_id) decoded_id: Final = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8") - - if ";" not in decoded_id: - return DecodedVideoId( - custom_llm_provider=None, - model_id=None, - video_id=encoded_video_id, - ) - - parts: Final = decoded_id.split(";") - - custom_llm_provider = None - model_id = None - decoded_video_id = encoded_video_id - - if len(parts) >= 3: - custom_llm_provider_part: Final = parts[0] - model_id_part: Final = parts[1] - video_id_part: Final = parts[2] - - custom_llm_provider = custom_llm_provider_part.replace("litellm:custom_llm_provider:", "") - model_id = model_id_part.replace("model_id:", "") - decoded_video_id = video_id_part.replace("video_id:", "") - - served_model: Final = ( - parts[3].removeprefix(VIDEO_ID_SERVED_MODEL_PREFIX) or None - if len(parts) >= 4 and parts[3].startswith(VIDEO_ID_SERVED_MODEL_PREFIX) - else None - ) - - if served_model is None: - return DecodedVideoId(custom_llm_provider=custom_llm_provider, model_id=model_id, video_id=decoded_video_id) - return DecodedVideoId( - custom_llm_provider=custom_llm_provider, - model_id=model_id, - video_id=decoded_video_id, - served_model=served_model, - ) except Exception as e: verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) return DecodedVideoId( @@ -125,6 +122,42 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: video_id=encoded_video_id, ) + if ";" not in decoded_id: + return DecodedVideoId( + custom_llm_provider=None, + model_id=None, + video_id=encoded_video_id, + ) + + parts: Final = decoded_id.split(";") + + custom_llm_provider = None + model_id = None + decoded_video_id = encoded_video_id + + if len(parts) >= 3: + custom_llm_provider_part: Final = parts[0] + model_id_part: Final = parts[1] + video_id_part: Final = parts[2] + + custom_llm_provider = custom_llm_provider_part.replace("litellm:custom_llm_provider:", "") + model_id = model_id_part.replace("model_id:", "") + decoded_video_id = video_id_part.replace("video_id:", "") + + served_model: Final = _trailing_part(parts, VIDEO_ID_SERVED_MODEL_PREFIX) + deployment_id: Final = _trailing_part(parts, VIDEO_ID_DEPLOYMENT_PREFIX) + requested_model: Final = _trailing_part(parts, VIDEO_ID_REQUESTED_MODEL_PREFIX) + decoded_parts: Final = DecodedVideoId( + custom_llm_provider=custom_llm_provider, model_id=model_id, video_id=decoded_video_id + ) + with_served: Final[DecodedVideoId] = ( + decoded_parts if served_model is None else {**decoded_parts, "served_model": served_model} + ) + with_deployment: Final[DecodedVideoId] = ( + with_served if deployment_id is None else {**with_served, "deployment_id": deployment_id} + ) + return with_deployment if requested_model is None else {**with_deployment, "requested_model": requested_model} + def extract_original_video_id(encoded_video_id: str) -> str: """Extract original video ID without encoding.""" diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 9bd077ef3a5..8e84cf2e3e9 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -99,6 +99,8 @@ ignored_function_names = [ "_embedding", "_aembedding", "_stream_pre_content_error", # Tested through the non-retriable retry error tests in test_router.py + "_narrow_to_video_deployment", # Tested through video_status and avideo_status polls in test_router_provider_endpoints.py + "_video_call_pin", # Tested through the video_status pin tests and the chat video_id test in test_router_provider_endpoints.py "_deployment_num_retries", # Tested through the deployment num_retries mid-stream budget test in test_router.py "_request_fallback_list", # Tested through every mid-stream retry test in test_router.py "_request_model_group", # Tested through test_anthropic_messages_retry_budget_precedence_direct_call diff --git a/tests/integration/routing/test_video_poll_routing_wire.py b/tests/integration/routing/test_video_poll_routing_wire.py index e0d88a6b0ff..a3d767275a0 100644 --- a/tests/integration/routing/test_video_poll_routing_wire.py +++ b/tests/integration/routing/test_video_poll_routing_wire.py @@ -338,9 +338,9 @@ def test_an_id_without_a_served_group_still_polls_its_own_deployment(rig: _Rig) video_id: Final = created_id(create(rig.gateway, {"model": rig.member_a, "seconds": "5", "size": "1280x720"})) assert decoded(video_id).get("served_model") is None legacy: Final = forged_id("dashscope", rig.member_a, task_of(video_id), None) - assert legacy == video_id + assert {**decoded(legacy), "deployment_id": decoded(video_id).get("deployment_id")} == decoded(video_id) rig.drain() - _assert_served(_poll(rig, legacy), legacy) + _assert_served(_poll(rig, legacy), video_id) download: Final = _download(rig, legacy) assert download.status_code == 200 and download.content == MP4, download.text received: Final = rig.a.drain() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 8331fb1937b..9028b8461b0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -93,7 +93,15 @@ CALLBACK_LISTS: Final = ( "_async_failure_callback", "_async_input_callback", ) -RESET_TO_NONE_GLOBALS: Final = ("model_fallbacks", "cache") +RESET_TO_NONE_GLOBALS: Final = ( + "model_fallbacks", + "cache", + "max_fallbacks", + "default_fallbacks", + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", +) RESTORED_GLOBALS: Final = ( "disable_aiohttp_transport", "force_ipv4", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index eb76901db45..a8b2ff4f6cd 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -414,18 +414,11 @@ class TestLogOnceIfBudgetReservationDisabled: def test_logs_info_only_once_when_enabled(self, caplog): with caplog.at_level(logging.INFO, logger="LiteLLM Proxy"): log_once_if_budget_reservation_disabled(disabled=False) - assert not any( - "disable_budget_reservation is enabled" in record.message - for record in caplog.records - ) + assert not any("disable_budget_reservation is enabled" in record.message for record in caplog.records) for _ in range(3): log_once_if_budget_reservation_disabled(disabled=True) - records = [ - record - for record in caplog.records - if "disable_budget_reservation is enabled" in record.message - ] + records = [record for record in caplog.records if "disable_budget_reservation is enabled" in record.message] assert len(records) == 1 assert records[0].levelno == logging.INFO @@ -753,19 +746,28 @@ def test_get_model_from_request_no_request_extracts_model(): ) -@pytest.mark.parametrize("provider,model", [ - ("laya", "english"), ("laya", "multilingual"), ("laya", "typed-decisions"), - ("bespoke", "nimble-latest"), ("bespoke", "bespokelabs/Bespoke-Nimble-9B"), -]) +@pytest.mark.parametrize( + "provider,model", + [ + ("laya", "english"), + ("laya", "multilingual"), + ("laya", "typed-decisions"), + ("bespoke", "nimble-latest"), + ("bespoke", "bespokelabs/Bespoke-Nimble-9B"), + ], +) @pytest.mark.parametrize("suffix", ["", "/"]) def test_oss_native_model_uses_the_classifier_permission_identity(provider: str, model: str, suffix: str) -> None: - assert get_model_from_request( - request_data={"model": model}, route=f"/{provider}/v1/systemone{suffix}" - ) == f"{provider}/{model}" + assert ( + get_model_from_request(request_data={"model": model}, route=f"/{provider}/v1/systemone{suffix}") + == f"{provider}/{model}" + ) @pytest.mark.parametrize("provider", ["laya", "bespoke"]) -@pytest.mark.parametrize("model", [None, "", "auto", "laya/english", "bespoke/nimble-latest", "unknown", ["english"], 7]) +@pytest.mark.parametrize( + "model", [None, "", "auto", "laya/english", "bespoke/nimble-latest", "unknown", ["english"], 7] +) def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(provider: str, model: object) -> None: with pytest.raises(HTTPException) as denied: get_model_from_request(request_data={"model": model}, route=f"/{provider}/v1/systemone") @@ -780,38 +782,49 @@ def test_laya_model_normalization_does_not_change_other_provider_routes() -> Non def _cache_prediction_router(): from litellm.router import Router - return Router(model_list=[ - { - "model_name": group, - "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "test-provider-key"}, - "model_info": {"id": deployment_id, "team_id": team_id}, - } - for group, deployment_id, team_id in ( - ("current-group", "current-id", None), ("candidate-group", "candidate-id", None), - ("own-group", "own-id", "prediction-team"), ("foreign-group", "foreign-id", "foreign-team"), - ) - ]) + return Router( + model_list=[ + { + "model_name": group, + "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "test-provider-key"}, + "model_info": {"id": deployment_id, "team_id": team_id}, + } + for group, deployment_id, team_id in ( + ("current-group", "current-id", None), + ("candidate-group", "candidate-id", None), + ("own-group", "own-id", "prediction-team"), + ("foreign-group", "foreign-id", "foreign-team"), + ) + ] + ) -@pytest.mark.parametrize("candidate,team_id,expected", [ - ("candidate-id", None, ["current-group", "candidate-group"]), - ("current-id", None, "current-group"), - ("missing-id", None, None), - ("candidate-group", None, None), - ("own-id", None, None), - ("own-id", "prediction-team", ["current-group", "own-group"]), - ("foreign-id", "prediction-team", None), -]) +@pytest.mark.parametrize( + "candidate,team_id,expected", + [ + ("candidate-id", None, ["current-group", "candidate-group"]), + ("current-id", None, "current-group"), + ("missing-id", None, None), + ("candidate-group", None, None), + ("own-id", None, None), + ("own-id", "prediction-team", ["current-group", "own-group"]), + ("foreign-id", "prediction-team", None), + ], +) def test_cache_prediction_auth_resolves_only_exact_deployment_ids(candidate, team_id, expected): - assert get_model_from_request( - request_data={ - "current_deployment_id": "current-id", "candidate_deployment_id": candidate, - "request": {"model": "caller-controlled-provider-model"}, - }, - route="/cost/predict-cache", - llm_router=_cache_prediction_router(), - team_id=team_id, - ) == expected + assert ( + get_model_from_request( + request_data={ + "current_deployment_id": "current-id", + "candidate_deployment_id": candidate, + "request": {"model": "caller-controlled-provider-model"}, + }, + route="/cost/predict-cache", + llm_router=_cache_prediction_router(), + team_id=team_id, + ) + == expected + ) def _cache_prediction_auth_app( @@ -834,18 +847,29 @@ def _cache_prediction_auth_app( router = _cache_prediction_router() allowed_models = ["current-group", "candidate-group", "own-group"] token = UserAPIKeyAuth( - api_key="test-proxy-key-hash", user_id="prediction-user", user_role=LitellmUserRoles.INTERNAL_USER, - models=allowed_models if key_models is None else key_models, team_id=team_id, + api_key="test-proxy-key-hash", + user_id="prediction-user", + user_role=LitellmUserRoles.INTERNAL_USER, + models=allowed_models if key_models is None else key_models, + team_id=team_id, team_models=allowed_models if team_models is None else team_models, - allowed_routes=allowed_routes, metadata=metadata or {}, + allowed_routes=allowed_routes, + metadata=metadata or {}, ) user = LiteLLM_UserTable( - user_id=token.user_id, user_role=LitellmUserRoles.INTERNAL_USER.value, models=user_models, + user_id=token.user_id, + user_role=LitellmUserRoles.INTERNAL_USER.value, + models=user_models, ) + async def authenticate(request, request_data, **_headers): await auth._enforce_key_and_fallback_model_access( - valid_token=token, request_data=request_data, route=request.url.path, request=request, - llm_model_list=router.get_model_list(), llm_router=router, + valid_token=token, + request_data=request_data, + route=request.url.path, + request=request, + llm_model_list=router.get_model_list(), + llm_router=router, ) return token @@ -878,22 +902,36 @@ def _cache_prediction_auth_app( def _cache_prediction_payload(candidate="candidate-id", current="current-id"): return { - "current_deployment_id": current, "candidate_deployment_id": candidate, - "request": {"messages": [{"role": "user", "content": [{ - "type": "text", "text": "Stable cached context", - "cache_control": {"type": "ephemeral"}, - }]}]}, + "current_deployment_id": current, + "candidate_deployment_id": candidate, + "request": { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Stable cached context", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + }, } @pytest.mark.asyncio -@pytest.mark.parametrize("allowed_routes,user_models,candidate,status_code", [ - (["/chat/completions"], ["current-group", "candidate-group"], "candidate-id", 403), - (["/cost/predict-cache"], ["current-group"], "candidate-id", 403), - (["/cost/*"], ["current-group", "candidate-group"], "candidate-id", 200), - (["/cost/predict-cache"], ["current-group"], "current-id", 200), - (["/cost/predict-cache"], ["current-group"], "missing-id", 404), -]) +@pytest.mark.parametrize( + "allowed_routes,user_models,candidate,status_code", + [ + (["/chat/completions"], ["current-group", "candidate-group"], "candidate-id", 403), + (["/cost/predict-cache"], ["current-group"], "candidate-id", 403), + (["/cost/*"], ["current-group", "candidate-group"], "candidate-id", 200), + (["/cost/predict-cache"], ["current-group"], "current-id", 200), + (["/cost/predict-cache"], ["current-group"], "missing-id", 404), + ], +) async def test_cache_prediction_authorizes_route_and_personal_models_before_provider_counts( monkeypatch, allowed_routes, user_models, candidate, status_code ): @@ -912,20 +950,27 @@ async def test_cache_prediction_authorizes_route_and_personal_models_before_prov @pytest.mark.asyncio @pytest.mark.parametrize("arm", ["current_deployment_id", "candidate_deployment_id"]) -@pytest.mark.parametrize("team_id,key_models,user_models,team_models", [ - (None, ["*"], ["*"], None), - (None, ["current-group", "candidate-group"], ["*"], None), - (None, ["*"], ["current-group", "candidate-group"], None), - ("prediction-team", ["*"], ["*"], ["current-group", "candidate-group"]), -]) +@pytest.mark.parametrize( + "team_id,key_models,user_models,team_models", + [ + (None, ["*"], ["*"], None), + (None, ["current-group", "candidate-group"], ["*"], None), + (None, ["*"], ["current-group", "candidate-group"], None), + ("prediction-team", ["*"], ["*"], ["current-group", "candidate-group"]), + ], +) async def test_cache_prediction_hides_foreign_and_missing_ids_before_model_authorization( monkeypatch, arm, team_id, key_models, user_models, team_models ): import httpx app, counts = _cache_prediction_auth_app( - monkeypatch, ["/cost/predict-cache"], user_models, - team_id=team_id, key_models=key_models, team_models=team_models, + monkeypatch, + ["/cost/predict-cache"], + user_models, + team_id=team_id, + key_models=key_models, + team_models=team_models, ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: missing = await client.post("/cost/predict-cache", json={**_cache_prediction_payload(), arm: "missing-id"}) @@ -938,19 +983,26 @@ async def test_cache_prediction_hides_foreign_and_missing_ids_before_model_autho @pytest.mark.asyncio @pytest.mark.parametrize("arm", ["current_deployment_id", "candidate_deployment_id"]) -@pytest.mark.parametrize("key_models,team_models,status_code", [ - (["*"], ["*"], 200), - (["current-group", "candidate-group"], ["*"], 403), - (["*"], ["current-group", "candidate-group"], 403), -]) +@pytest.mark.parametrize( + "key_models,team_models,status_code", + [ + (["*"], ["*"], 200), + (["current-group", "candidate-group"], ["*"], 403), + (["*"], ["current-group", "candidate-group"], 403), + ], +) async def test_cache_prediction_checks_each_visible_team_deployment_model( monkeypatch, arm, key_models, team_models, status_code ): import httpx app, counts = _cache_prediction_auth_app( - monkeypatch, ["/cost/predict-cache"], ["*"], - team_id="prediction-team", key_models=key_models, team_models=team_models, + monkeypatch, + ["/cost/predict-cache"], + ["*"], + team_id="prediction-team", + key_models=key_models, + team_models=team_models, ) async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client: response = await client.post("/cost/predict-cache", json={**_cache_prediction_payload(), arm: "own-id"}) @@ -976,19 +1028,24 @@ async def test_cache_prediction_checks_each_visible_personal_deployment_model(mo @pytest.mark.asyncio -@pytest.mark.parametrize("header_tag,key_tags,limit,status_code,provider_calls", [ - ("limited", [], 1, 429, 1), - (None, ["limited"], 1, 429, 1), - ("limited", ["limited"], 4, 200, 4), - ("unlimited", [], 1, 200, 4), -]) +@pytest.mark.parametrize( + "header_tag,key_tags,limit,status_code,provider_calls", + [ + ("limited", [], 1, 429, 1), + (None, ["limited"], 1, 429, 1), + ("limited", ["limited"], 4, 200, 4), + ("unlimited", [], 1, 200, 4), + ], +) async def test_cache_prediction_preserves_authenticated_header_and_key_tag_rpm( monkeypatch, header_tag, key_tags, limit, status_code, provider_calls ): import httpx app, counts = _cache_prediction_auth_app( - monkeypatch, ["/cost/predict-cache"], ["current-group", "candidate-group"], + monkeypatch, + ["/cost/predict-cache"], + ["current-group", "candidate-group"], metadata={"tag_rpm_limit": {"limited": limit}, "tags": key_tags}, ) headers = {"x-litellm-tags": header_tag} if header_tag else {} @@ -1142,7 +1199,11 @@ def _azure_relay_router(): model_list=[ { "model_name": "gpt", - "litellm_params": {"model": "azure_ai/gpt-5.4-mini", "api_base": "https://a.services.ai.azure.com", "api_key": "k"}, + "litellm_params": { + "model": "azure_ai/gpt-5.4-mini", + "api_base": "https://a.services.ai.azure.com", + "api_key": "k", + }, }, { "model_name": "other-group", @@ -1364,8 +1425,14 @@ def test_get_model_from_request_resolves_video_id_model_with_router(): provider="vertex_ai", model_id="veo-3.1-generate-001", ) - llm_router = MagicMock() - llm_router.resolve_model_name_from_model_id.return_value = "gcp/google/veo-3.1-generate-001" + llm_router = Router( + model_list=[ + { + "model_name": "gcp/google/veo-3.1-generate-001", + "litellm_params": {"model": "vertex_ai/veo-3.1-generate-001"}, + } + ] + ) assert ( get_model_from_request( @@ -1375,7 +1442,148 @@ def test_get_model_from_request_resolves_video_id_model_with_router(): ) == "gcp/google/veo-3.1-generate-001" ) - llm_router.resolve_model_name_from_model_id.assert_called_once_with("veo-3.1-generate-001") + + +def test_get_model_from_request_checks_the_group_of_the_deployment_that_created_the_video() -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + llm_router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ] + ) + video_id: Final = encode_video_id_with_provider( + video_id="video_raw", provider="azure", model_id="sora-2", deployment_id="dep-sora-shared" + ) + + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=llm_router, + ) + == "sora-shared" + ) + + +def test_get_model_from_request_checks_a_video_created_through_a_provider_wildcard_by_its_own_name() -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + llm_router: Final = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "wildcard-key"}, + "model_info": {"id": "dep-wildcard"}, + } + ] + ) + video_id: Final = encode_video_id_with_provider( + video_id="video_raw", provider="openai", model_id="sora-2", deployment_id="dep-wildcard" + ) + routed_ids: Final = [ + deployment["model_info"]["id"] for deployment in llm_router.pattern_router.get_deployments_by_pattern("sora-2") + ] + + assert routed_ids == ["dep-wildcard"] + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=llm_router, + ) + == "sora-2" + ) + + +def test_get_model_from_request_checks_a_wildcard_video_whose_bare_name_does_not_route_by_its_prefixed_name() -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + llm_router: Final = Router( + model_list=[ + { + "model_name": "gemini/*", + "litellm_params": {"model": "gemini/*", "api_key": "wildcard-key"}, + "model_info": {"id": "dep-wildcard"}, + } + ] + ) + video_id: Final = encode_video_id_with_provider( + video_id="operations/op_1", provider="gemini", model_id="unlisted-video-model", deployment_id="dep-wildcard" + ) + + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=llm_router, + ) + == "gemini/unlisted-video-model" + ) + + +def _sora_fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-pair", "sora-backup", "sora-last", "sora-other") + ], + fallbacks=[{"sora-pair": ["sora-backup"]}, {"sora-backup": ["sora-last"]}], + ) + + +@pytest.mark.parametrize( + ("general_settings", "requested_model", "creating_group", "expected"), + [ + ({"enforce_fallback_model_access": False}, "sora-pair", "sora-backup", "sora-pair"), + ({"enforce_fallback_model_access": False}, "sora-pair", "sora-last", "sora-pair"), + ({}, "sora-pair", "sora-backup", "sora-backup"), + ({"enforce_fallback_model_access": False}, "sora-other", "sora-backup", "sora-backup"), + ({"enforce_fallback_model_access": False}, None, "sora-backup", "sora-backup"), + ], + ids=[ + "opt-out-fallback", + "opt-out-chained-fallback", + "enforced", + "opt-out-no-fallback-relation", + "opt-out-no-requested-model", + ], +) +def test_get_model_from_request_checks_a_fallback_served_video_by_the_requested_group_only_under_the_opt_out( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, object], + requested_model: str | None, + creating_group: str, + expected: str, +) -> None: + from litellm.types.videos.utils import encode_video_id_with_provider + + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings) + video_id: Final = encode_video_id_with_provider( + video_id="video_raw", + provider="azure", + model_id="sora-2", + deployment_id=f"dep-{creating_group}", + requested_model=requested_model, + ) + + assert ( + get_model_from_request( + request_data={"video_id": video_id}, + route="/v1/videos/{video_id}", + llm_router=_sora_fallback_router(), + ) + == expected + ) _BATCH_DEPLOYMENT_ID = "8d0eaa7e6c6f54a425dfd0062cb6b0dc" @@ -4463,9 +4671,7 @@ class TestIsRequestBodySafeBlocksAwsIdentitySelectors: ), ], ) -def test_configurable_clientside_parameters( - allowed_param, input_value, should_return_true -): +def test_configurable_clientside_parameters(allowed_param, input_value, should_return_true): router = Router( model_list=[ { @@ -4499,9 +4705,7 @@ def test_get_customer_user_header_from_mapping_returns_customer_header_with_mixe def test_get_customer_user_header_from_mapping_no_customer_returns_none(): from litellm.proxy.auth.auth_utils import get_customer_user_header_from_mapping - mappings = [ - {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"} - ] + mappings = [{"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}] result = get_customer_user_header_from_mapping(mappings) assert result is None @@ -4529,17 +4733,13 @@ def test_get_internal_user_header_from_mapping_returns_internal_header(): def test_get_internal_user_header_from_mapping_no_internal_returns_none(): from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - mappings = [ - {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"} - ] + mappings = [{"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}] result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(mappings) assert result is None # Also support single mapping dict single_mapping = {"header_name": "X-Only-Customer", "litellm_user_role": "customer"} - result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping( - single_mapping - ) + result = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(single_mapping) assert result is None @@ -4561,9 +4761,7 @@ def test_get_internal_user_header_from_mapping_no_internal_returns_none(): def test_get_model_from_request(request_data, expected_model): from litellm.proxy.auth.auth_utils import get_model_from_request - request_data = { - "target_model_names": "gpt-3.5-turbo, gpt-4o-mini-general-deployment" - } + request_data = {"target_model_names": "gpt-3.5-turbo, gpt-4o-mini-general-deployment"} route = "/openai/deployments/gpt-3.5-turbo" model = get_model_from_request(request_data, "/v1/files") assert model == ["gpt-3.5-turbo", "gpt-4o-mini-general-deployment"] @@ -4625,9 +4823,7 @@ def test_get_model_from_request(request_data, expected_model): ), ], ) -def test_get_model_from_request_vertex_ai_passthrough( - request_data, route, expected_model -): +def test_get_model_from_request_vertex_ai_passthrough(request_data, route, expected_model): """Test that get_model_from_request correctly extracts Vertex AI model from URL""" from litellm.proxy.auth.auth_utils import get_model_from_request diff --git a/tests/unit/proxy/video_endpoints/test_endpoints.py b/tests/unit/proxy/video_endpoints/test_endpoints.py index 5052d75f9f9..9809d26826b 100644 --- a/tests/unit/proxy/video_endpoints/test_endpoints.py +++ b/tests/unit/proxy/video_endpoints/test_endpoints.py @@ -28,7 +28,8 @@ mock's call args), and a brand-new kwarg added to this layer surfaces as a failu from contextlib import ExitStack from dataclasses import dataclass -from typing import Any, Dict, Optional +from collections.abc import Awaitable, Callable +from typing import Any, Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch import orjson @@ -41,7 +42,9 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import ProxyLogging from litellm.router import Router +from litellm.types.videos.main import VideoObject from litellm.types.videos.utils import ( + decode_video_id_with_provider, encode_character_id_with_provider, encode_video_id_with_provider, ) @@ -70,6 +73,29 @@ RESOLVED_MODELS: Dict[str, str] = {VIDEO_MODEL_ID: "azure-sora"} SENTINEL = object() +def _video_with_empty_model_id( + hidden_params: dict[str, object] | None = None, +) -> VideoObject: + video: Final[VideoObject] = VideoObject( + id=encode_video_id_with_provider("video_new456", "azure", ""), + object="video", + status="queued", + ) + video.hidden_params = dict(hidden_params or {}) + return video + + +def _assert_id_resolves_to_deployment(response: object, *, deployment_id: str | None = None) -> None: + assert isinstance(response, VideoObject) + expected_deployment: Final = {} if deployment_id is None else {"deployment_id": deployment_id} + assert decode_video_id_with_provider(response.id) == { + "custom_llm_provider": "azure", + "model_id": VIDEO_MODEL_ID, + "video_id": "video_new456", + **expected_deployment, + } + + class FakeRequest: """Minimal stand-in. headers/query_params are read by the provider readers (mocked) and on the edit path the raw body is parsed for real via orjson.""" @@ -105,6 +131,9 @@ class Harness: assert self.base_process.call_count == 1 return dict(self.base_process.call_args.args[0].data) + def respond_with(self, response: object) -> None: + self.base_process.return_value = response # pyright: ignore[reportAny] # MagicMock subclasses Any in typeshed + def route_type(self) -> str: return self.base_process.call_args.kwargs["route_type"] @@ -118,6 +147,7 @@ def harness(): side_effect=lambda model_id, custom_llm_provider=None: RESOLVED_MODELS.get(model_id) ) router.resolve_model_name_from_model_id = resolve_model + router.get_deployment.return_value = None read_body = AsyncMock(return_value={}) batch_to_bytesio = AsyncMock(return_value=[b"filebytes"]) @@ -244,6 +274,36 @@ async def test_generation__input_reference_attached(harness): } +@pytest.mark.asyncio +async def test_generation__restamps_id_with_asked_for_model_over_router_deployment_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id(hidden_params={"model_id": "deployment-uuid"})) + + resp: Final = await call_generation(harness, body={"model": VIDEO_MODEL_ID, "prompt": "x"}) + + _assert_id_resolves_to_deployment(resp, deployment_id="deployment-uuid") + + +@pytest.mark.asyncio +async def test_generation__restamps_id_with_request_model_when_no_hidden_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_generation(harness, body={"model": VIDEO_MODEL_ID, "prompt": "x"}) + + _assert_id_resolves_to_deployment(resp) + + +@pytest.mark.asyncio +async def test_generation__unroutable_request_model_falls_back_to_router_deployment_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id(hidden_params={"model_id": "deployment-uuid"})) + + resp: Final = await call_generation(harness, body={"model": "team-public-sora", "prompt": "x"}) + + assert isinstance(resp, VideoObject) + decoded: Final = decode_video_id_with_provider(resp.id) + assert decoded.get("model_id") == "deployment-uuid" + assert decoded.get("deployment_id") == "deployment-uuid" + + @pytest.mark.asyncio async def test_generation__exception_routed_through_handler(harness): harness.base_process.side_effect = ValueError("provider boom") @@ -284,6 +344,24 @@ async def test_status__model_encoded_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_status__restamps_response_id_with_decoded_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_status(harness, AZURE_VIDEO_ID) + + _assert_id_resolves_to_deployment(resp) + + +@pytest.mark.asyncio +async def test_status__id_without_a_deployment_takes_the_deployment_that_answered_the_poll(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id(hidden_params={"model_id": "deployment-uuid"})) + + resp: Final = await call_status(harness, AZURE_VIDEO_ID) + + _assert_id_resolves_to_deployment(resp, deployment_id="deployment-uuid") + + @pytest.mark.asyncio async def test_status__plain_id_defaults_to_openai(harness): await call_status(harness, "video_plain") @@ -387,6 +465,30 @@ async def test_status_and_content__hand_the_served_member_to_the_request_process assert harness.base_process.call_args.args[0].served_model == "wan3-b" +@pytest.mark.asyncio +@pytest.mark.parametrize("call", [call_status, call_content]) +async def test_status_and_content__shared_provider_model_routes_to_the_creating_deployments_group( + harness: Harness, call: Callable[[Harness, str], Awaitable[object]] +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ] + ) + harness.base_process.return_value = b"x" + video_id: Final = encode_video_id_with_provider("video_raw", "azure", "sora-2", deployment_id="dep-sora-shared") + + with patch.object(proxy_server, "llm_router", router): + await call(harness, video_id) + + assert harness.processor_data()["model"] == "sora-shared" + + # =========================================================================== # # POST /v1/videos/edits - video_edit # # =========================================================================== # @@ -422,6 +524,15 @@ async def test_edit__extracts_nested_video_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_edit__restamps_new_video_id_with_source_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_edit(harness, body={"prompt": "brighter", "video": {"id": AZURE_VIDEO_ID}}) + + _assert_id_resolves_to_deployment(resp) + + @pytest.mark.asyncio async def test_edit__provider_from_body_data_for_plain_id(harness): """For a plain id, get_custom_provider_from_data (run for real) pulls the @@ -473,6 +584,23 @@ async def test_edit__json_string_video_reference_from_form_field(harness): assert harness.processor_data()["video_id"] == "video_plain" +@pytest.mark.asyncio +async def test_edit__uploaded_video_restamps_new_id_with_requested_model(harness: Harness) -> None: + import io + + upload: Final = StarletteUploadFile(file=io.BytesIO(b"rawmp4"), filename="clip.mp4") + harness.read_body.return_value = {"model": VIDEO_MODEL_ID, "prompt": "make it nighttime", "video": upload} + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await endpoints.video_edit( + request=FakeRequest(raw_body=b"multipart"), + fastapi_response=Response(), + user_api_key_dict=_user(), + ) + + _assert_id_resolves_to_deployment(resp) + + @pytest.mark.asyncio async def test_edit__uploaded_video_file_is_forwarded_not_dropped(harness): """A multipart-uploaded source video must be converted to bytes and attached @@ -565,6 +693,15 @@ async def test_remix__model_encoded_id_full_contract(harness): } +@pytest.mark.asyncio +async def test_remix__restamps_new_video_id_with_source_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_remix(harness, AZURE_VIDEO_ID, body={"prompt": "new colors"}) + + _assert_id_resolves_to_deployment(resp) + + @pytest.mark.asyncio async def test_remix__provider_from_body_data_not_request_body_reader(harness): """remix resolves the provider from data.get('custom_llm_provider'), never @@ -694,6 +831,20 @@ async def test_get_character__plain_id_defaults_openai_no_encode(harness): assert resp["id"] == "char_raw3" +@pytest.mark.asyncio +async def test_get_character__model_served_only_by_a_provider_wildcard_routes_through_it(harness: Harness) -> None: + router: Final = Router( + model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": "key-w"}}] + ) + harness.base_process.return_value = {"id": "char_raw4"} + character_id: Final = encode_character_id_with_provider("char_orig", "openai", "sora-2") + + with patch.object(proxy_server, "llm_router", router): + await call_get_character(harness, character_id) + + assert harness.processor_data()["model"] == "sora-2" + + # =========================================================================== # # POST /v1/videos/extensions - video_extension # # =========================================================================== # @@ -723,3 +874,12 @@ async def test_extension__extracts_nested_video_id_full_contract(harness): "custom_llm_provider": "azure", "model": "azure-sora", } + + +@pytest.mark.asyncio +async def test_extension__restamps_new_video_id_with_source_model_id(harness: Harness) -> None: + harness.respond_with(_video_with_empty_model_id()) + + resp: Final = await call_extension(harness, body={"prompt": "continue", "video": {"id": AZURE_VIDEO_ID}}) + + _assert_id_resolves_to_deployment(resp) diff --git a/tests/unit/proxy/video_endpoints/test_utils.py b/tests/unit/proxy/video_endpoints/test_utils.py index 9a2c208c075..01a2b7c2840 100644 --- a/tests/unit/proxy/video_endpoints/test_utils.py +++ b/tests/unit/proxy/video_endpoints/test_utils.py @@ -1,11 +1,12 @@ """ Pure-logic contract tests for litellm/proxy/video_endpoints/utils.py -Four helpers the video proxy endpoints lean on: +Five helpers the video proxy endpoints lean on: - extract_model_from_target_model_names: first model from a comma string / list - video_reference_to_id: normalize a video reference (dict / bare id / JSON string) to an id - get_custom_provider_from_data: provider precedence (top-level > extra_body) - encode_character_id_in_response: re-encode a response id in place + - encode_video_id_in_response: re-stamp a VideoObject id with the resolved model_id Every test asserts the exact result (or identity), so a mutation that flips a branch, drops a strip/filter, or changes precedence fails. The only collaborator @@ -13,19 +14,29 @@ is encode_character_id_with_provider, which runs for real; encoding assertions are checked by the genuine decode round-trip. """ +from typing import Final import pytest +from litellm.router import Router +from litellm.types.videos.main import DecodedVideoId from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, + encode_video_id_in_response, extract_model_from_target_model_names, + fallback_requested_model, get_custom_provider_from_data, + routable_model_name, + video_model_group, video_reference_to_id, ) +from litellm.types.videos.main import VideoObject from litellm.types.videos.utils import ( decode_character_id_with_provider, + decode_video_id_with_provider, encode_character_id_with_provider, + encode_video_id_with_provider, ) # =========================================================================== # @@ -103,12 +114,7 @@ def test_provider__falsy_top_level_falls_through_to_extra_body(falsy): def test_provider__from_extra_body_dict(): - assert ( - get_custom_provider_from_data( - {"extra_body": {"custom_llm_provider": "bedrock"}} - ) - == "bedrock" - ) + assert get_custom_provider_from_data({"extra_body": {"custom_llm_provider": "bedrock"}}) == "bedrock" def test_provider__from_extra_body_json_string(): @@ -126,10 +132,7 @@ def test_provider__json_string_parsing_to_non_dict_is_none(): def test_provider__extra_body_provider_not_a_string_is_none(): - assert ( - get_custom_provider_from_data({"extra_body": {"custom_llm_provider": 123}}) - is None - ) + assert get_custom_provider_from_data({"extra_body": {"custom_llm_provider": 123}}) is None @pytest.mark.parametrize( @@ -161,9 +164,7 @@ def test_encode__dict_with_id_mutates_in_place_and_preserves_other_keys(): assert out is response # same dict, mutated in place assert out["object"] == "character" and out["name"] == "hero" - assert out["id"] == encode_character_id_with_provider( - "char_raw", "azure", "model-1" - ) + assert out["id"] == encode_character_id_with_provider("char_raw", "azure", "model-1") decoded = decode_character_id_with_provider(out["id"]) assert decoded["custom_llm_provider"] == "azure" assert decoded["model_id"] == "model-1" @@ -206,3 +207,387 @@ def test_encode__object_without_id_attr_returned_unchanged(): out = encode_character_id_in_response(resp, "azure", "model-1") assert out is resp assert not hasattr(resp, "id") + + +# =========================================================================== # +# encode_video_id_in_response +# =========================================================================== # + + +def _video(video_id: str, hidden_params: dict[str, object] | None = None) -> VideoObject: + video: Final[VideoObject] = VideoObject(id=video_id, object="video", status="queued") + video.hidden_params = dict(hidden_params or {}) + return video + + +def test_encode_video__fills_empty_model_id_and_deployment_from_hidden_params() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-1"}) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id=None) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "openai", + "model_id": "deployment-1", + "video_id": "video_raw", + "deployment_id": "deployment-1", + } + assert out is not video + assert video.id == original_id + assert out.hidden_params == {"model_id": "deployment-1"} + assert out.status == "queued" + + +def test_encode_video__asked_for_model_wins_over_hidden_deployment_id() -> None: + video: Final = _video( + encode_video_id_with_provider("video_raw", "openai", ""), hidden_params={"model_id": "deployment-1"} + ) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora-alias") + + assert isinstance(out, VideoObject) + decoded: Final = decode_video_id_with_provider(out.id) + assert decoded.get("model_id") == "sora-alias" + assert decoded.get("deployment_id") == "deployment-1" + + +def test_encode_video__stamped_id_keeps_model_and_served_group_and_gains_routed_deployment() -> None: + original_id: Final = encode_video_id_with_provider("task_1", "volcengine", "seedance-group", "seedance-east") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-1"}) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="other") + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "volcengine", + "model_id": "seedance-group", + "video_id": "task_1", + "served_model": "seedance-east", + "deployment_id": "deployment-1", + } + + +def test_encode_video__routed_deployment_replaces_the_one_in_the_id() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id="deployment-old") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-new"}) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id=None) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id).get("deployment_id") == "deployment-new" + + +@pytest.mark.parametrize( + ("polled_deployment", "answering_deployment", "returned_deployment"), + [ + (None, "deployment-routed", "deployment-routed"), + ("deployment-gone", "deployment-routed", "deployment-routed"), + ("deployment-creator", None, "deployment-creator"), + ], +) +def test_encode_video__a_poll_takes_the_deployment_that_answered_else_keeps_the_one_its_id_named( + polled_deployment: str | None, + answering_deployment: str | None, + returned_deployment: str, +) -> None: + polled_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id=polled_deployment) + video: Final = _video( + encode_video_id_with_provider("video_raw", "openai", ""), + hidden_params={} if answering_deployment is None else {"model_id": answering_deployment}, + ) + + out: Final = encode_video_id_in_response( + video, + fallback_provider=None, + fallback_model_id="sora", + polled_video=decode_video_id_with_provider(polled_id), + ) + + assert isinstance(out, VideoObject) + assert out.id == encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id=returned_deployment) + + +def test_encode_video__a_poll_keeps_the_group_a_fallback_create_was_serving() -> None: + polled_id: Final = encode_video_id_with_provider( + "video_raw", "azure", "sora-2", deployment_id="dep-backup", requested_model="sora-pair" + ) + video: Final = _video(encode_video_id_with_provider("video_raw", "azure", "sora-2")) + + out: Final = encode_video_id_in_response( + video, fallback_provider=None, fallback_model_id=None, polled_video=decode_video_id_with_provider(polled_id) + ) + + assert isinstance(out, VideoObject) + assert out.id == polled_id + + +def test_encode_video__a_remix_of_a_fallback_video_keeps_its_requested_group() -> None: + video: Final = _video( + encode_video_id_with_provider("video_new", "azure", "sora-2"), hidden_params={"model_id": "dep-backup"} + ) + + out: Final = encode_video_id_in_response( + video, fallback_provider=None, fallback_model_id=None, requested_model="sora-pair" + ) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id).get("requested_model") == "sora-pair" + + +def test_encode_video__id_already_naming_the_routed_deployment_is_unchanged() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id="deployment-1") + video: Final = _video(original_id, hidden_params={"model_id": "deployment-1"}) + + assert encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="other") is video + + +def test_encode_video__unrouted_response_keeps_the_deployment_in_its_id() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "openai", "", deployment_id="deployment-1") + + out: Final = encode_video_id_in_response(_video(original_id), fallback_provider=None, fallback_model_id="sora") + + assert isinstance(out, VideoObject) + decoded: Final = decode_video_id_with_provider(out.id) + assert decoded.get("model_id") == "sora" + assert decoded.get("deployment_id") == "deployment-1" + + +def test_encode_video__name_with_id_separator_is_never_written_into_the_id() -> None: + video: Final = _video( + encode_video_id_with_provider("video_raw", "openai", ""), hidden_params={"model_id": "deployment-1"} + ) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora;model_id:evil") + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "openai", + "model_id": "deployment-1", + "video_id": "video_raw", + "deployment_id": "deployment-1", + } + + +def test_encode_video__falls_back_to_given_model_id() -> None: + video: Final = _video(encode_video_id_with_provider("video_raw", "openai", "")) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora-2") + + assert isinstance(out, VideoObject) + decoded: Final = decode_video_id_with_provider(out.id) + assert decoded.get("model_id") == "sora-2" + assert decoded.get("video_id") == "video_raw" + + +def test_encode_video__keeps_existing_model_id_over_fallback() -> None: + original_id: Final = encode_video_id_with_provider("video_raw", "azure", "deployment-1") + + out: Final = encode_video_id_in_response(_video(original_id), fallback_provider="openai", fallback_model_id="other") + + assert isinstance(out, VideoObject) + assert out.id == original_id + + +def test_encode_video__plain_id_encoded_with_fallback_provider() -> None: + video: Final = _video("video_raw", hidden_params={"model_id": "deployment-1"}) + + out: Final = encode_video_id_in_response(video, fallback_provider="openai", fallback_model_id=None) + + assert isinstance(out, VideoObject) + assert decode_video_id_with_provider(out.id) == { + "custom_llm_provider": "openai", + "model_id": "deployment-1", + "video_id": "video_raw", + "deployment_id": "deployment-1", + } + + +def test_encode_video__no_model_id_anywhere_unchanged() -> None: + video: Final = _video(encode_video_id_with_provider("video_raw", "openai", "")) + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id=None) + + assert out is video + + +def test_encode_video__no_provider_anywhere_unchanged() -> None: + video: Final = _video("video_raw") + + out: Final = encode_video_id_in_response(video, fallback_provider=None, fallback_model_id="sora-2") + + assert out is video + + +@pytest.mark.parametrize("response", [b"video-bytes", {"id": "video_raw"}, None]) +def test_encode_video__non_video_object_passthrough(response: object) -> None: + assert encode_video_id_in_response(response, "openai", "sora-2") is response + + +def _router_with_groups(*names: str) -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": "openai/sora-2", "api_key": "group-key"}} for name in names + ] + ) + + +def test_routable_model_name__keeps_a_name_the_router_resolves() -> None: + assert routable_model_name("sora-alias", _router_with_groups("sora-alias"), "openai") == "sora-alias" + + +def test_routable_model_name__drops_a_name_the_router_cannot_route() -> None: + assert routable_model_name("team-public-name", _router_with_groups("sora-alias"), "openai") is None + + +def test_routable_model_name__keeps_the_name_without_a_router() -> None: + assert routable_model_name("sora-2", None, None) == "sora-2" + + +@pytest.mark.parametrize("model", [None, "", 7, "sora;model_id:evil"]) +def test_routable_model_name__never_returns_a_name_unsafe_for_the_id(model: object) -> None: + assert routable_model_name(model, _router_with_groups("sora;model_id:evil"), "openai") is None + + +def _two_groups_sharing_sora() -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ] + ) + + +def _decoded(model_id: str, deployment_id: str | None) -> DecodedVideoId: + base: Final = DecodedVideoId(custom_llm_provider="azure", model_id=model_id, video_id="video_raw") + return base if deployment_id is None else {**base, "deployment_id": deployment_id} + + +@pytest.mark.parametrize("deployment_id", ["dep-sora-team", "dep-sora-shared"]) +def test_video_model_group__shared_provider_model_resolves_to_the_creating_deployments_group( + deployment_id: str, +) -> None: + router: Final = _two_groups_sharing_sora() + + assert video_model_group(_decoded("sora-2", deployment_id), router) == deployment_id.removeprefix("dep-") + + +@pytest.mark.parametrize("deployment_id", [None, "dep-gone"]) +def test_video_model_group__without_a_known_deployment_matches_plain_resolution(deployment_id: str | None) -> None: + router: Final = _two_groups_sharing_sora() + + assert video_model_group(_decoded("sora-2", deployment_id), router) == router.resolve_model_name_from_model_id( + "sora-2", "azure" + ) + + +def test_video_model_group__keeps_a_group_named_in_the_id_over_a_deployment_of_another_group() -> None: + assert video_model_group(_decoded("sora-team", "dep-sora-shared"), _two_groups_sharing_sora()) == "sora-team" + + +@pytest.mark.parametrize("deployment_id", ["dep-sora-team", "dep-sora-shared"]) +def test_video_model_group__keeps_an_alias_of_the_creating_deployments_group(deployment_id: str) -> None: + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"openai/sora-{name}", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-team", "sora-shared") + ], + model_group_alias={"sora-alias": deployment_id.removeprefix("dep-")}, + ) + provider_model: Final = f"sora-{deployment_id.removeprefix('dep-')}" + + assert video_model_group(_decoded(provider_model, deployment_id), router) == "sora-alias" + + +def test_video_model_group__names_a_team_model_by_its_public_name() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "model_name_team-1_internal", + "litellm_params": {"model": "azure/sora-2", "api_key": "team-key"}, + "model_info": {"id": "dep-team", "team_id": "team-1", "team_public_model_name": "team-sora"}, + } + ] + ) + + assert video_model_group(_decoded("sora-2", "dep-team"), router) == "team-sora" + + +def _fallback_router() -> Router: + return Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "azure/sora-2", "api_base": f"https://{name}.example.com", "api_key": name}, + "model_info": {"id": f"dep-{name}"}, + } + for name in ("sora-pair", "sora-backup", "sora-last") + ], + model_group_alias={"sora-alias": "sora-pair"}, + fallbacks=[{"sora-pair": ["sora-backup"]}, {"sora-backup": ["sora-last"]}], + ) + + +def _wildcard_router() -> Router: + return Router( + model_list=[ + { + "model_name": "gemini/*", + "litellm_params": {"model": "gemini/*", "api_key": "wildcard-key"}, + "model_info": {"id": "dep-wildcard"}, + } + ] + ) + + +@pytest.mark.parametrize( + ("requested_model", "answering_deployment", "expected"), + [ + ("sora-pair", "dep-sora-backup", "sora-pair"), + ("sora-pair", "dep-sora-last", "sora-pair"), + ("sora-backup", "dep-sora-pair", None), + ("sora-pair", "dep-sora-pair", None), + ("sora-alias", "dep-sora-pair", None), + ("sora-pair", None, None), + (None, "dep-sora-backup", None), + ], + ids=[ + "fallback-served", + "chained-fallback-served", + "served-by-a-group-it-never-falls-back-to", + "served-by-own-group", + "served-by-alias-target", + "unrouted", + "no-requested-model", + ], +) +def test_fallback_requested_model__names_the_asked_group_only_when_another_group_served_the_create( + requested_model: str | None, answering_deployment: str | None, expected: str | None +) -> None: + video: Final = _video( + encode_video_id_with_provider("video_raw", "azure", "sora-2"), + hidden_params={} if answering_deployment is None else {"model_id": answering_deployment}, + ) + + assert fallback_requested_model(requested_model, video, _fallback_router()) == expected + + +def test_fallback_requested_model__is_none_for_a_create_a_provider_wildcard_served() -> None: + video: Final = _video( + encode_video_id_with_provider("operations/op_1", "gemini", "veo-3.1-fast-generate-preview"), + hidden_params={"model_id": "dep-wildcard"}, + ) + + assert fallback_requested_model("gemini/veo-3.1-fast-generate-preview", video, _wildcard_router()) is None + + +def test_video_model_group__is_none_without_a_router() -> None: + assert video_model_group(_decoded("sora-2", "dep-sora-shared"), None) is None diff --git a/tests/unit/test_router/test_router_provider_endpoints.py b/tests/unit/test_router/test_router_provider_endpoints.py index 708322f4fec..9312204f471 100644 --- a/tests/unit/test_router/test_router_provider_endpoints.py +++ b/tests/unit/test_router/test_router_provider_endpoints.py @@ -16,6 +16,7 @@ from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.router_callbacks.track_deployment_metrics import ( get_deployment_successes_for_current_minute, ) @@ -23,6 +24,7 @@ from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import HttpxBinaryResponseContent from litellm.types.router import RoutingStrategy from litellm.types.utils import ImageResponse, ModelResponse, RerankResponse +from litellm.types.videos.utils import encode_video_id_with_provider from openai import AsyncAzureOpenAI, AsyncOpenAI EVENT_TIMEOUT_SECONDS: Final = 5 @@ -730,3 +732,248 @@ async def test_router_moderation_endpoint_factory_invokes_default_model( assert route.call_count == 1 assert json.loads(route.calls[0].request.content) == {"input": "hello"} assert response.id == "modr-2" + + +def _sora_deployment( + model_name: str, api_key: str, deployment_id: str, access_groups: tuple[str, ...] = () +) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": "openai/sora-2", "api_key": api_key}, + "model_info": {"id": deployment_id, "access_groups": list(access_groups)}, + } + + +def _two_account_sora_router() -> Router: + return Router( + model_list=[ + _sora_deployment("sora", "key-a", "dep-a"), + _sora_deployment("sora", "key-b", "dep-b"), + _sora_deployment("other", "key-x", "dep-x"), + ], + model_group_alias={"sora-alias": "sora"}, + ) + + +async def _poll_status_keys( + router: Router, + model: str, + deployment_id: str, + respx_mock: respx.MockRouter, + caller: UserAPIKeyAuth | None = None, +) -> set[str]: + route: Final = respx_mock.get("https://api.openai.com/v1/videos/video_raw").respond( + 200, json={"id": "video_raw", "object": "video", "status": "in_progress"} + ) + video_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id=deployment_id) + metadata: Final = {} if caller is None else {"user_api_key_auth": caller} + for _ in range(10): + await router.avideo_status(model=model, video_id=video_id, litellm_metadata=dict(metadata)) + return {call.request.headers["authorization"] for call in route.calls} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["sora", "sora-alias"]) +async def test_video_status_polls_the_deployment_that_created_the_video( + model: str, respx_mock: respx.MockRouter +) -> None: + keys: Final = await _poll_status_keys(_two_account_sora_router(), model, "dep-b", respx_mock) + + assert keys == {"Bearer key-b"} + + +def test_sync_video_status_polls_the_deployment_that_created_the_video(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.get("https://api.openai.com/v1/videos/video_raw").respond( + 200, json={"id": "video_raw", "object": "video", "status": "in_progress"} + ) + router: Final = _two_account_sora_router() + video_id: Final = encode_video_id_with_provider("video_raw", "openai", "sora", deployment_id="dep-b") + + for _ in range(10): + router.video_status(model="sora", video_id=video_id) + + assert {call.request.headers["authorization"] for call in route.calls} == {"Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_on_a_wildcard_polls_the_deployment_that_created_the_video( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": key}, + "model_info": {"id": dep}, + } + for dep, key in (("wc-a", "key-a"), ("wc-b", "key-b")) + ] + ) + + keys: Final = await _poll_status_keys(router, "openai/sora-2", "wc-b", respx_mock) + + assert keys == {"Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_never_pins_to_a_deployment_in_cooldown(respx_mock: respx.MockRouter) -> None: + router: Final = _two_account_sora_router() + router.cooldown_cache.add_deployment_to_cooldown( + model_id="dep-b", + original_exception=litellm.RateLimitError(message="upstream 429", llm_provider="openai", model="sora-2"), + exception_status=429, + cooldown_time=60, + ) + + keys: Final = await _poll_status_keys(router, "sora", "dep-b", respx_mock) + + assert keys == {"Bearer key-a"} + + +@pytest.mark.asyncio +async def test_video_status_on_a_routing_group_polls_the_member_deployment_that_created_the_video( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router( + model_list=[_sora_deployment("sora-a", "key-a", "dep-a"), _sora_deployment("sora-b", "key-b", "dep-b")], + routing_groups=[ + {"group_name": "sora-group", "models": ["sora-a", "sora-b"], "routing_strategy": "simple-shuffle"} + ], + ) + + keys: Final = await _poll_status_keys(router, "sora-group", "dep-b", respx_mock) + + assert keys == {"Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_never_pins_to_a_deployment_outside_the_requested_group( + respx_mock: respx.MockRouter, +) -> None: + keys: Final = await _poll_status_keys(_two_account_sora_router(), "sora", "dep-x", respx_mock) + + assert keys <= {"Bearer key-a", "Bearer key-b"} + + +@pytest.mark.asyncio +async def test_video_status_with_a_removed_deployment_routes_across_the_group(respx_mock: respx.MockRouter) -> None: + keys: Final = await _poll_status_keys(_two_account_sora_router(), "sora", "dep-gone", respx_mock) + + assert keys <= {"Bearer key-a", "Bearer key-b"} + assert keys + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("key_access_group", "expected_key"), [("ag-a", "Bearer key-a"), ("ag-b", "Bearer key-b")]) +async def test_video_status_pins_only_to_a_deployment_in_the_callers_access_groups( + key_access_group: str, expected_key: str, respx_mock: respx.MockRouter +) -> None: + router: Final = Router( + model_list=[ + _sora_deployment("sora", "key-a", "dep-a", access_groups=("ag-a",)), + _sora_deployment("sora", "key-b", "dep-b", access_groups=("ag-b",)), + ] + ) + + keys: Final = await _poll_status_keys( + router, "sora", "dep-b", respx_mock, caller=UserAPIKeyAuth(models=[key_access_group]) + ) + + assert keys == {expected_key} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["sora", "sora-alias"]) +async def test_pinned_video_status_logs_the_requested_model_group( + model: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + capture: Final = _RouterLoggingCapture("dep-b") + monkeypatch.setattr(litellm, "callbacks", [capture]) + + keys: Final = await _poll_status_keys(_two_account_sora_router(), model, "dep-b", respx_mock) + _, standard_logging_object = await capture.next_event() + + assert keys == {"Bearer key-b"} + assert isinstance(standard_logging_object, dict) + assert standard_logging_object["model_group"] == model + + +def _ordered_chat_router() -> Router: + return Router( + model_list=[ + { + "model_name": "chat-ordered", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": key, "order": order}, + "model_info": {"id": deployment_id}, + } + for deployment_id, key, order in (("dep-primary", "key-a", 1), ("dep-backup", "key-b", 2)) + ] + ) + + +@pytest.mark.asyncio +async def test_a_video_id_in_a_chat_body_never_overrides_deployment_order() -> None: + router: Final = _ordered_chat_router() + video_id: Final = encode_video_id_with_provider("video_raw", "openai", "chat-ordered", deployment_id="dep-backup") + + chats: Final = [ + await router.acompletion( + model="chat-ordered", messages=list(ROUTING_MESSAGES), mock_response="ok", video_id=video_id + ) + for _ in range(4) + ] + responses: Final = [ + await router.aresponses(model="chat-ordered", input="route this request", mock_response="ok", video_id=video_id) + for _ in range(4) + ] + + assert {chat._hidden_params["model_id"] for chat in chats} == {"dep-primary"} + assert {response._hidden_params["model_id"] for response in responses} == {"dep-primary"} + + +@pytest.mark.asyncio +async def test_video_lookup_404_never_cools_down_the_deployment_that_answered(respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://api.openai.com/v1/videos/video_gone").respond( + 404, json={"error": {"code": "NotFound", "message": "Video not found"}} + ) + router: Final = Router( + model_list=[_sora_deployment("sora", "key-a", "dep-a"), _sora_deployment("sora", "key-b", "dep-b")], + num_retries=0, + allowed_fails=1, + ) + video_id: Final = encode_video_id_with_provider("video_gone", "openai", "sora", deployment_id="dep-b") + + for _ in range(3): + with pytest.raises(litellm.NotFoundError): + await router.avideo_status(model="sora", video_id=video_id) + + assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == [] + + +@pytest.mark.asyncio +async def test_chat_404_still_cools_down_the_deployment(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.openai.com/v1/chat/completions").respond( + 404, json={"error": {"code": "NotFound", "message": "Deployment not found"}} + ) + router: Final = Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "key-a"}, + "model_info": {"id": "dep-chat"}, + }, + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "key-b"}, + "model_info": {"id": "dep-chat-b"}, + }, + ], + num_retries=0, + allowed_fails=1, + ) + + for _ in range(3): + with contextlib.suppress(litellm.NotFoundError, litellm.RateLimitError): + await router.acompletion(model="chat", messages=list(ROUTING_MESSAGES)) + + assert await router.cooldown_cache.async_get_active_cooldowns(["dep-chat", "dep-chat-b"], parent_otel_span=None) diff --git a/tests/unit/test_video_generation.py b/tests/unit/test_video_generation.py index 5c4bc8ea652..df7e9c68b90 100644 --- a/tests/unit/test_video_generation.py +++ b/tests/unit/test_video_generation.py @@ -2,6 +2,7 @@ import asyncio import io import json import os +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -1269,6 +1270,35 @@ def test_encode_video_id_with_provider_handles_azure_video_prefix(): assert encoded_twice == encoded_id # Should return the same encoded ID +@pytest.mark.parametrize( + "served_model, deployment_id, expected_extras", + [ + ("seedance-east", "dep-1", {"served_model": "seedance-east", "deployment_id": "dep-1"}), + (None, "dep-1", {"deployment_id": "dep-1"}), + ("seedance-east", None, {"served_model": "seedance-east"}), + (None, None, {}), + ], +) +def test_video_id_round_trips_served_model_and_deployment_id( + served_model: str | None, deployment_id: str | None, expected_extras: dict[str, str] +) -> None: + from litellm.types.videos.utils import ( + decode_video_id_with_provider, + encode_video_id_with_provider, + ) + + encoded: Final = encode_video_id_with_provider( + "task_1", "volcengine", "seedance-group", served_model=served_model, deployment_id=deployment_id + ) + + assert decode_video_id_with_provider(encoded) == { + "custom_llm_provider": "volcengine", + "model_id": "seedance-group", + "video_id": "task_1", + **expected_extras, + } + + class TestVideoListTransformation: """Tests for video list request/response transformation with provider ID encoding."""