diff --git a/litellm/constants.py b/litellm/constants.py index 6c889a317b8..334ef8d48a4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -419,9 +419,6 @@ CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0 AUDIO_SPEECH_CHUNK_SIZE = int( os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192) ) # chunk_size for audio speech streaming. Balance between latency and memory usage -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( - os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512) -) DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000)) #### Networking settings #### # Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 8dfaa8b1425..a1bf65141c9 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -220,23 +220,57 @@ def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs): safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role) +def _safe_get(obj, key, default=None): + """Read ``key`` from a dict-like or Pydantic-model-like object. + + The arize/langfuse_otel logger receives ``usage`` objects from many sources: + plain dicts, litellm ``Usage`` (which exposes ``.get``), and raw OpenAI + Pydantic models (e.g. ``openai.types.completion_usage.CompletionUsage`` and + nested ``CompletionTokensDetails`` / ``OutputTokensDetails``) which do NOT + expose ``.get``. Calling ``.get`` on the latter raised ``AttributeError`` — + see https://github.com/BerriAI/litellm/issues/13672. + """ + if obj is None: + return default + getter = getattr(obj, "get", None) + if callable(getter): + try: + return getter(key, default) + except TypeError: + # Some objects expose `.get` with a different signature + pass + return getattr(obj, key, default) + + def _set_usage_outputs(span: "Span", response_obj, span_attrs): usage = response_obj and response_obj.get("usage") if not usage: return safe_set_attribute( - span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens") + span, span_attrs.LLM_TOKEN_COUNT_TOTAL, _safe_get(usage, "total_tokens") + ) + completion_tokens = _safe_get(usage, "completion_tokens") or _safe_get( + usage, "output_tokens" ) - completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens") if completion_tokens: safe_set_attribute( span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens ) - prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens") + prompt_tokens = _safe_get(usage, "prompt_tokens") or _safe_get( + usage, "input_tokens" + ) if prompt_tokens: safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens) - reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens") + + # Reasoning tokens live in `completion_tokens_details` for Chat Completions + # API (Usage) and in `output_tokens_details` for Responses API + # (ResponseAPIUsage). Both nested objects may be plain Pydantic models + # without `.get`. + token_details = _safe_get(usage, "completion_tokens_details") or _safe_get( + usage, "output_tokens_details" + ) + reasoning_tokens = _safe_get(token_details, "reasoning_tokens") if reasoning_tokens: safe_set_attribute( span, diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 829ce14d69d..8ed6126d2eb 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -27,6 +27,16 @@ from litellm.utils import get_model_info if TYPE_CHECKING: pass + +# Anthropic-only fields that the translator above already maps into the +# OpenAI-format completion_kwargs (output_config → reasoning_effort / +# response_format, etc.). They must be filtered out of the raw +# extra_kwargs re-merge below or non-Anthropic backends reject the call +# with 400 "Extra inputs are not permitted". Add new entries here when +# extending AnthropicMessagesRequestOptionalParams with another Anthropic- +# specific key. +ANTHROPIC_ONLY_REQUEST_KEYS: frozenset[str] = frozenset({"output_config"}) + ######################################################## # init adapter ANTHROPIC_ADAPTER = AnthropicAdapter() @@ -202,8 +212,12 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data["output_format"] = output_format # Extract output_config from extra_kwargs so the translator can use it - # (e.g. output_config.effort for adaptive thinking → reasoning_effort) - extra_kwargs = extra_kwargs or {} + # (e.g. output_config.effort for adaptive thinking → reasoning_effort, + # output_config.format → response_format for structured outputs). + # Use explicit None check rather than `or {}` so an explicit empty dict + # caller-passed argument is preserved (matters for tests that drive + # the fallback inference path). + extra_kwargs = extra_kwargs if extra_kwargs is not None else {} if "output_config" in extra_kwargs: request_data["output_config"] = extra_kwargs["output_config"] @@ -225,8 +239,23 @@ class LiteLLMMessagesToCompletionTransformationHandler: "include_usage": True, } - excluded_keys = {"anthropic_messages"} - extra_kwargs = extra_kwargs or {} + # Keys that must NOT be forwarded as raw extras into the OpenAI-format + # ``completion_kwargs`` after translation. The translator above has + # already consumed the meaningful parts of these inputs (e.g. + # ``output_config.format`` → ``response_format``, ``output_config.effort`` + # → ``reasoning_effort`` for non-Claude targets). Re-adding the raw + # Anthropic-shaped key here causes 400 "Extra inputs are not permitted" + # on non-Anthropic backends (Azure OpenAI, Fireworks, Bedrock Nova, + # etc.) and is silently lossy on Anthropic-family targets, which would + # see the translated key ``response_format`` AND a duplicate, conflicting + # ``output_config``. + # + # Maintainability: when adding a new Anthropic-only request param to + # ``AnthropicMessagesRequestOptionalParams``, also extend + # ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak. + excluded_keys = ANTHROPIC_ONLY_REQUEST_KEYS | {"anthropic_messages"} + # NOTE: extra_kwargs was already coerced from None to {} at the top of + # this method (line ~220). It is guaranteed to be a dict here. for key, value in extra_kwargs.items(): if ( key == "litellm_logging_obj" diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 08797889192..fe8e694efe5 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -667,7 +667,7 @@ class LiteLLMAnthropicMessagesAdapter: @staticmethod def translate_anthropic_thinking_to_reasoning_effort( - thinking: Dict[str, Any] + thinking: Dict[str, Any], ) -> Optional[str]: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. @@ -1084,10 +1084,23 @@ class LiteLLMAnthropicMessagesAdapter: anthropic_message_request: AnthropicMessagesRequest, new_kwargs: ChatCompletionRequest, ) -> None: - """Translate output_format to response_format when applicable.""" - if "output_format" not in anthropic_message_request: - return - output_format = anthropic_message_request["output_format"] + """Translate Anthropic structured-output config to OpenAI ``response_format``. + + Accepts either the legacy top-level ``output_format`` field OR the + newer ``output_config.format`` (sub-key on ``output_config``) so that + both shapes flow through to non-Anthropic backends as + ``response_format``. Without the ``output_config.format`` branch, + callers using the new Anthropic Structured Outputs API would have + their schema silently dropped on the adapter path — only the legacy + top-level ``output_format`` was being mapped. + + ``output_format`` takes precedence when both are provided. + """ + output_format: Any = anthropic_message_request.get("output_format") + if not output_format: + output_config = anthropic_message_request.get("output_config") + if isinstance(output_config, dict): + output_format = output_config.get("format") if not output_format: return response_format = self.translate_anthropic_output_format_to_openai( diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index b9e219f5cbc..66e253f304d 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -149,9 +149,9 @@ class CloudflareChatConfig(BaseConfig): ) -> ModelResponse: completion_response = raw_response.json() - model_response.choices[0].message.content = completion_response["result"][ # type: ignore - "response" - ] + # Support both "response" and "response_text" keys (newer models like Nemotron use "response_text") + result = completion_response["result"] + model_response.choices[0].message.content = result.get("response") if result.get("response") is not None else result.get("response_text", "") # type: ignore prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model) completion_tokens = len( @@ -201,8 +201,10 @@ class CloudflareChatResponseIterator(BaseModelResponseIterator): index = int(chunk.get("index", 0)) - if "response" in chunk: + if "response" in chunk and chunk["response"] is not None: text = chunk["response"] + elif "response_text" in chunk and chunk["response_text"] is not None: + text = chunk["response_text"] returned_chunk = GenericStreamingChunk( text=text, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 5c3bbf61ee2..d450f7a4635 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -13,6 +13,7 @@ from litellm.types.llms.vertex_ai import VertexPartnerProvider from litellm.types.router import GenericLiteLLMParams from ....vertex_llm_base import VertexBase +from ..output_params_utils import sanitize_vertex_anthropic_output_params class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase): @@ -158,12 +159,10 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert "model", None ) # do not pass model in request body to vertex ai - anthropic_messages_request.pop( - "output_format", None - ) # do not pass output_format in request body to vertex ai - vertex ai does not support output_format as yet - - anthropic_messages_request.pop( - "output_config", None - ) # do not pass output_config in request body to vertex ai - vertex ai does not support output_config + # Vertex AI Claude accepts ``output_config.format`` (structured outputs) + # and ``output_format``, but rejects ``output_config.effort`` with 400 + # "Extra inputs are not permitted". Sanitize in place so the supported + # bits flow through. + sanitize_vertex_anthropic_output_params(anthropic_messages_request) return anthropic_messages_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py new file mode 100644 index 00000000000..982d8edbf20 --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -0,0 +1,50 @@ +""" +Shared sanitization for ``output_config`` / ``output_format`` on Vertex AI +Claude. Lives in its own module so both the chat-completion transformation +(``transformation.py``) and the Messages pass-through transformation +(``experimental_pass_through/transformation.py``) can import it without +forming a cycle through the parent module's heavier imports. + +CodeQL flagged the ``..transformation`` import path as a potential cyclic +import; extracting the helper into a leaf module resolves the warning and +keeps the parent module's import surface narrow. +""" + +# Keys inside ``output_config`` that Vertex AI Claude does not accept. +# Today only ``effort`` triggers "Extra inputs are not permitted"; add new +# entries here as Vertex parity drifts. Keep this list narrow — anything +# Vertex DOES accept (e.g. ``format`` for structured outputs) must be +# preserved so callers can rely on Anthropic-native features. +VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS: frozenset = frozenset({"effort"}) + + +def sanitize_vertex_anthropic_output_params(data: dict) -> None: + """ + Strip Vertex-unsupported keys from ``output_config`` / + ``output_format`` in-place; forward whatever remains. + + Behavior: + * ``output_config`` containing only unsupported keys (e.g. ``effort`` + alone) is removed entirely so the request body has no empty dict. + * ``output_config`` containing a mix of supported + unsupported keys + has the unsupported subset filtered out and the rest forwarded. + * ``output_config`` that is supported in full passes through unchanged. + * ``output_format`` is forwarded as-is (Vertex AI Claude accepts it). + * Non-dict values for ``output_config`` are dropped to avoid sending + malformed payloads downstream. + """ + output_config = data.get("output_config") + if output_config is None: + return + if not isinstance(output_config, dict): + data.pop("output_config", None) + return + sanitized = { + k: v + for k, v in output_config.items() + if k not in VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS + } + if sanitized: + data["output_config"] = sanitized + else: + data.pop("output_config", None) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 504914c4796..914c7e92e5e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -10,6 +10,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from ....anthropic.chat.transformation import AnthropicConfig +from .output_params_utils import sanitize_vertex_anthropic_output_params class VertexAIError(Exception): @@ -105,11 +106,12 @@ class VertexAIAnthropicConfig(AnthropicConfig): data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter - # VertexAI doesn't support output_format parameter, remove it if present - data.pop("output_format", None) - - # VertexAI doesn't support output_config parameter, remove it if present - data.pop("output_config", None) + # Vertex AI Claude accepts ``output_config.format`` (structured outputs / + # JSON Schema) but NOT ``output_config.effort`` — sending ``effort`` to + # Vertex returns 400 "Extra inputs are not permitted". Sanitize in place: + # forward the structured-output bits, drop the unsupported keys. + # Same treatment for the legacy top-level ``output_format`` field. + sanitize_vertex_anthropic_output_params(data) tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7893673ec5f..3db13e8ada6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6249,10 +6249,15 @@ async def initialize( # noqa: PLR0915 if litellm_log_setting.upper() == "INFO": import logging - from litellm._logging import verbose_proxy_logger, verbose_router_logger + from litellm._logging import ( + verbose_logger, + verbose_proxy_logger, + verbose_router_logger, + ) # this must ALWAYS remain logging.INFO, DO NOT MODIFY THIS + verbose_logger.setLevel(level=logging.INFO) # set package log to info verbose_router_logger.setLevel( level=logging.INFO ) # set router logs to info diff --git a/tests/llm_translation/test_cloudflare.py b/tests/llm_translation/test_cloudflare.py index 0c799b4f399..5a6a0008398 100644 --- a/tests/llm_translation/test_cloudflare.py +++ b/tests/llm_translation/test_cloudflare.py @@ -43,6 +43,14 @@ def _streaming_chunks() -> list[str]: ] +def _streaming_chunks_response_text() -> list[str]: + return [ + json.dumps({"response_text": "I am"}), + json.dumps({"response_text": " a language"}), + json.dumps({"response_text": " model."}), + ] + + @pytest.mark.parametrize("sync_mode", [True, False]) def test_completion_cloudflare(sync_mode): messages = [{"role": "user", "content": "what llm are you"}] @@ -145,3 +153,76 @@ def test_completion_cloudflare_stream(sync_mode): if c.choices[0].delta.content ) assert "language" in content.lower() + + +@pytest.mark.parametrize("sync_mode", [True, False]) +def test_completion_cloudflare_stream_response_text(sync_mode): + """Newer Cloudflare Workers AI models (e.g. Nemotron) emit `response_text` + instead of `response` in streamed chunks. The iterator must surface that + text so streaming output is not silently empty. + """ + messages = [{"role": "user", "content": "what llm are you"}] + raw_chunks = _streaming_chunks_response_text() + + if sync_mode: + + def _iter_lines(): + for chunk in raw_chunks: + yield f"data: {chunk}" + yield "data: [DONE]" + + mock_resp = MagicMock() + mock_resp.iter_lines.return_value = _iter_lines() + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/event-stream"} + + with patch.object(HTTPHandler, "post", return_value=mock_resp) as mock_post: + response = completion( + model="cloudflare/@cf/nvidia/nemotron-mini-4b-instruct", + messages=messages, + max_tokens=15, + stream=True, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + chunks_received = list(response) + mock_post.assert_called_once() + else: + + async def _aiter_lines(): + for chunk in raw_chunks: + yield f"data: {chunk}" + yield "data: [DONE]" + + mock_resp = MagicMock() + mock_resp.aiter_lines.return_value = _aiter_lines() + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/event-stream"} + + async def _run(): + with patch.object( + AsyncHTTPHandler, "post", new_callable=AsyncMock, return_value=mock_resp + ) as mock_post: + resp = await acompletion( + model="cloudflare/@cf/nvidia/nemotron-mini-4b-instruct", + messages=messages, + max_tokens=15, + stream=True, + api_base=FAKE_API_BASE, + api_key=FAKE_API_KEY, + ) + received = [] + async for chunk in resp: + received.append(chunk) + mock_post.assert_called_once() + return received + + chunks_received = asyncio.run(_run()) + + assert len(chunks_received) > 0 + content = "".join( + c.choices[0].delta.content + for c in chunks_received + if c.choices[0].delta.content + ) + assert "language" in content.lower() diff --git a/tests/test_litellm/integrations/arize/test_arize_utils.py b/tests/test_litellm/integrations/arize/test_arize_utils.py index a87a4167899..86c5448d468 100644 --- a/tests/test_litellm/integrations/arize/test_arize_utils.py +++ b/tests/test_litellm/integrations/arize/test_arize_utils.py @@ -273,6 +273,91 @@ def test_arize_set_attributes_responses_api(): ) +def test_set_usage_outputs_pydantic_completion_usage(): + """ + Regression test for https://github.com/BerriAI/litellm/issues/13672 + + `_set_usage_outputs` previously called `usage.get(...)` which crashes when + `usage` is a plain Pydantic model (e.g. openai.types.completion_usage.CompletionUsage) + that does not implement dict-style `.get()`. Same crash for nested + `output_tokens_details` / `completion_tokens_details`. + + The function must: + 1. Read total/prompt/completion tokens from a Pydantic usage without `.get`. + 2. Read reasoning_tokens from `completion_tokens_details` (chat completions API) + OR `output_tokens_details` (responses API), even when those nested objects + are Pydantic models without `.get`. + 3. Not raise AttributeError; not call span.record_exception. + """ + from unittest.mock import MagicMock + + from openai.types.completion_usage import ( + CompletionTokensDetails, + CompletionUsage, + ) + + from litellm.integrations.arize._utils import _set_usage_outputs + + span = MagicMock() + + # Plain OpenAI Pydantic model — has no `.get()` + usage = CompletionUsage( + completion_tokens=60, + prompt_tokens=40, + total_tokens=100, + completion_tokens_details=CompletionTokensDetails(reasoning_tokens=25), + ) + assert not hasattr(usage, "get"), "precondition: CompletionUsage must lack .get" + + response_obj = {"usage": usage} + + # Must not raise + _set_usage_outputs(span, response_obj, SpanAttributes) + + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_TOTAL, 100) + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 40) + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, 60) + # reasoning_tokens for chat completions live in completion_tokens_details + span.set_attribute.assert_any_call( + SpanAttributes.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, 25 + ) + + +def test_set_usage_outputs_pydantic_response_api_usage(): + """ + Same crash also affects Responses API with `output_tokens_details` as a + Pydantic model that lacks `.get()`. Verifies the responses-API path. + """ + from unittest.mock import MagicMock + + from litellm.integrations.arize._utils import _set_usage_outputs + from litellm.types.llms.openai import OutputTokensDetails + + # Build an object that mimics openai ResponsesAPI usage but lacks `.get` + # (uses a plain class — not BaseLiteLLMOpenAIResponseObject) + class PlainResponsesUsage: + def __init__(self): + self.total_tokens = 370 + self.input_tokens = 120 + self.output_tokens = 250 + self.output_tokens_details = OutputTokensDetails(reasoning_tokens=180) + + usage = PlainResponsesUsage() + assert not hasattr(usage, "get") + + span = MagicMock() + response_obj = {"usage": usage} + + _set_usage_outputs(span, response_obj, SpanAttributes) + + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_TOTAL, 370) + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 120) + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, 250) + span.set_attribute.assert_any_call( + SpanAttributes.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING, 180 + ) + + class TestArizeLogger(CustomLogger): """ Custom logger implementation to capture standard_callback_dynamic_params. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py new file mode 100644 index 00000000000..615dc5cfebc --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_handler_output_config_passthrough.py @@ -0,0 +1,232 @@ +""" +Regression tests for output_config passthrough through the Anthropic +``/v1/messages`` → ``/chat/completions`` adapter. + +Background — what was broken: +* When a client sent ``output_config`` to ``/v1/messages`` and the request + was routed to a non-Anthropic backend (Azure OpenAI, Fireworks, Bedrock + Nova, etc.), the adapter forwarded the raw Anthropic-shaped ``output_config`` + field as-is into the OpenAI-format ``completion_kwargs``. The non-Anthropic + backend then rejected the request with 400 "Extra inputs are not permitted". +* The translator above the re-merge already extracts the meaningful parts of + ``output_config`` (``format`` → ``response_format``, ``effort`` → + ``reasoning_effort`` for non-Claude targets), so re-adding the raw key was + always either redundant (Anthropic-family) or harmful (non-Anthropic). + +Tests cover (consolidating PRs #23706 and #22727): +1. ``output_config`` is excluded from the post-translation re-merge. +2. ``ANTHROPIC_ONLY_REQUEST_KEYS`` constant is exported and contains + ``output_config`` so future maintainers know where to extend it. +3. The translator-extracted fields (``response_format`` / ``reasoning_effort``) + are still present after the strip — the strip removes only the raw + Anthropic-shaped duplicate. +4. Helper-level coverage for empty ``extra_kwargs`` (PR #22727 Greptile P2 — + the original ``or {}`` pattern silently substituted a default and prevented + the fallback inference path from being exercised). +""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +# Anchor sys.path to this file's location — not the working-directory-relative +# pattern Greptile flagged on PR #23706. Resolves correctly regardless of +# where pytest is invoked from. +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")) +) + +from litellm.llms.anthropic.experimental_pass_through.adapters.handler import ( + ANTHROPIC_ONLY_REQUEST_KEYS, + LiteLLMMessagesToCompletionTransformationHandler, +) + +MESSAGES = [{"role": "user", "content": "hello"}] + + +def _call_prepare(extra_kwargs, model="gpt-4o", output_format=None, **overrides): + """ + Drive ``_prepare_completion_kwargs`` with the minimum scaffolding needed. + + ``output_format`` is a top-level parameter on the function, so callers + pass it explicitly here rather than tucking it into ``extra_kwargs``. + + Uses an explicit-None check on ``extra_kwargs`` so callers can test the + falsy-empty-dict path. The fallback ``or {}`` pattern PR #22727 used here + masked the no-extra-kwargs case from ever exercising the test's intent. + """ + return LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=overrides.get("max_tokens", 1024), + messages=overrides.get("messages", MESSAGES), + model=model, + metadata=None, + stop_sequences=None, + stream=False, + system=None, + temperature=None, + thinking=None, + tool_choice=None, + tools=None, + top_k=None, + top_p=None, + output_format=output_format, + extra_kwargs=extra_kwargs, + ) + + +class TestAnthropicOnlyRequestKeysExport: + """The exclusion list must be a public, named constant for maintainability — + Greptile P2 on PR #23706: ``excluded_keys`` was silently growing as a + point-fix pattern. A named module-level constant gives reviewers a single + grep target when extending Anthropic-only fields.""" + + def test_constant_exposed(self): + assert isinstance(ANTHROPIC_ONLY_REQUEST_KEYS, frozenset) + + def test_contains_output_config(self): + assert "output_config" in ANTHROPIC_ONLY_REQUEST_KEYS + + +class TestOutputConfigStrippedFromCompletionKwargs: + """``output_config`` must not survive the post-translation re-merge into + ``completion_kwargs`` regardless of the target provider — the translator + has already consumed its meaningful parts.""" + + def test_output_config_with_effort_is_stripped(self): + extra_kwargs = { + "custom_llm_provider": "azure", + "output_config": {"effort": "high"}, + } + + result = _call_prepare(extra_kwargs=extra_kwargs) + + # Returns (completion_kwargs, original_messages, ...) — first element + # is the dict we care about. + completion_kwargs = result[0] if isinstance(result, tuple) else result + assert "output_config" not in completion_kwargs, ( + "Raw output_config must not be forwarded — non-Anthropic backends " + "reject it with 400 'Extra inputs are not permitted'" + ) + + def test_output_config_format_translated_to_response_format(self): + """When ``output_config`` carries structured-output ``format``, the + translator now maps it to OpenAI's ``response_format`` so non-Anthropic + backends see the schema in their native shape. The raw + ``output_config`` key is still stripped from ``completion_kwargs`` — + only the translated ``response_format`` survives. + + Before this PR, only the legacy top-level ``output_format`` was + translated; ``output_config.format`` was silently dropped on the + adapter path even when the schema was correctly supplied (issue + flagged by Greptile review of the initial fix). + """ + schema = { + "type": "object", + "additionalProperties": False, + "properties": {"name": {"type": "string"}}, + } + extra_kwargs = { + "custom_llm_provider": "azure", + "output_config": {"format": {"type": "json_schema", "schema": schema}}, + } + + result = _call_prepare(extra_kwargs=extra_kwargs) + completion_kwargs = result[0] if isinstance(result, tuple) else result + + # Raw Anthropic-shaped key is gone (would 400 on non-Anthropic backends). + assert "output_config" not in completion_kwargs + # Translated OpenAI-shaped key is present so the schema actually + # reaches the downstream backend. + assert "response_format" in completion_kwargs, ( + "output_config.format must be translated to response_format — " + "without this, structured-output schemas are silently dropped on " + "the adapter path" + ) + + def test_output_format_top_level_still_translates(self): + """Regression guard: the legacy top-level ``output_format`` field must + continue to translate to ``response_format``. The new + ``output_config.format`` path must not break this existing behavior.""" + schema = {"type": "object", "properties": {"name": {"type": "string"}}} + result = _call_prepare( + extra_kwargs={"custom_llm_provider": "azure"}, + output_format={"type": "json_schema", "schema": schema}, + ) + completion_kwargs = result[0] if isinstance(result, tuple) else result + + assert "response_format" in completion_kwargs + + def test_output_format_takes_precedence_over_output_config_format(self): + """When both top-level ``output_format`` and ``output_config.format`` + are present, the legacy top-level ``output_format`` wins. Documents + which one the translator picks rather than leaving it implementation- + defined.""" + winning_schema = { + "type": "object", + "properties": {"top_level": {"type": "string"}}, + } + losing_schema = { + "type": "object", + "properties": {"nested": {"type": "string"}}, + } + result = _call_prepare( + extra_kwargs={ + "custom_llm_provider": "azure", + "output_config": { + "format": {"type": "json_schema", "schema": losing_schema} + }, + }, + output_format={"type": "json_schema", "schema": winning_schema}, + ) + completion_kwargs = result[0] if isinstance(result, tuple) else result + + assert "response_format" in completion_kwargs + # Verify the winning_schema (top-level output_format) was used, + # not the losing one nested under output_config. + rendered = str(completion_kwargs["response_format"]) + assert "top_level" in rendered + assert "nested" not in rendered + + def test_other_extra_kwargs_still_passed_through(self): + """Regression guard: the strip must be narrow. Unrelated fields like + ``api_key`` / ``timeout`` continue to flow through.""" + extra_kwargs = { + "custom_llm_provider": "azure", + "output_config": {"effort": "high"}, + "timeout": 30, + "user": "end-user-123", + } + + result = _call_prepare(extra_kwargs=extra_kwargs) + completion_kwargs = result[0] if isinstance(result, tuple) else result + + assert "output_config" not in completion_kwargs + assert completion_kwargs.get("timeout") == 30 + assert completion_kwargs.get("user") == "end-user-123" + + +class TestEmptyExtraKwargsPath: + """Greptile P2 on PR #22727: ``extra_kwargs or {default}`` substitutes a + default for an explicitly-passed empty dict, hiding the no-extra-kwargs + path. The new explicit-None pattern lets ``extra_kwargs={}`` reach the + code under test as written.""" + + def test_explicit_empty_dict_does_not_substitute_default(self): + # Explicit empty dict must be honored — not silently replaced with a + # default that adds back a custom_llm_provider this test wants absent. + result = _call_prepare(extra_kwargs={}) + completion_kwargs = result[0] if isinstance(result, tuple) else result + + # No output_config because nothing supplied it. + assert "output_config" not in completion_kwargs + + def test_none_extra_kwargs_handled_safely(self): + """The signature documents ``extra_kwargs: Optional[Dict] = None``; + passing None must not crash with KeyError or AttributeError.""" + result = _call_prepare(extra_kwargs=None) + # Just exercising the path; assert no exception and we get back a + # dict-like result. + completion_kwargs = result[0] if isinstance(result, tuple) else result + assert isinstance(completion_kwargs, dict) diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 79fc66a74b8..d0be476d72e 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -203,12 +203,15 @@ def test_vertex_ai_anthropic_structured_output_header_not_added(): def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): """ Test fix for issue #18625: Claude Sonnet 4.5 on VertexAI should use tool-based - structured outputs instead of output_format parameter. + structured outputs when ``response_format`` is supplied via the OpenAI-compat + interface (``map_openai_params``). This test verifies that: - 1. Claude Sonnet 4.5 uses tool-based structured outputs on VertexAI - 2. output_format parameter is removed from the final request - 3. The fix prevents "Extra inputs are not permitted" error + 1. Claude Sonnet 4.5 uses tool-based structured outputs when ``response_format`` + is given to the OpenAI-compat path (the path that triggered #18625). + 2. ``output_format`` is forwarded to Vertex AI when present — Vertex now + accepts the field; the prior blanket-strip behavior was the silent drop + of Anthropic Structured Outputs that this PR fixes. """ config = VertexAIAnthropicConfig() @@ -294,11 +297,15 @@ def test_vertex_ai_claude_sonnet_4_5_structured_output_fix(): headers={}, ) - # Verify that output_format was removed (fixes the "Extra inputs are not permitted" error) + # output_format is now forwarded to Vertex (Vertex parity has shifted — + # it accepts the field and uses it to enforce the JSON schema). The + # prior behavior silently stripped it, hiding Structured Outputs from + # callers who explicitly requested them. + assert "output_format" in final_data + assert final_data["output_format"]["type"] == "json_schema" assert ( - "output_format" not in final_data - ), "output_format should be removed for VertexAI" - assert "model" not in final_data, "model should be removed for VertexAI" + "model" not in final_data + ), "model is still stripped (Vertex routes by URL)" assert "tools" in final_data, "tools should still be present" assert "tool_choice" in final_data, "tool_choice should still be present" @@ -491,28 +498,22 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea ), "Header should be removed if no supported values remain" -def test_vertex_ai_anthropic_output_config_dropped(): +def test_vertex_ai_anthropic_output_config_effort_only_dropped(): """ - Test that output_config parameter is dropped from Vertex AI Anthropic requests. - - Vertex AI does not support the output_config parameter (used for effort settings - in Anthropic API). This test ensures it's properly removed to prevent - "Extra inputs are not permitted" errors. + ``output_config`` containing only ``effort`` (an Anthropic-only key Vertex + rejects with "Extra inputs are not permitted") is dropped entirely so the + request body has no empty dict. """ config = VertexAIAnthropicConfig() messages = [{"role": "user", "content": "What is 2+2?"}] - headers = {} + headers: dict = {} - # Simulate optional_params with output_config that would be passed in optional_params = { "max_tokens": 1024, - "output_config": { - "effort": "high" # This is Anthropic-specific and not supported by Vertex AI - }, + "output_config": {"effort": "high"}, } - # Call transform_request which should drop output_config result = config.transform_request( model="claude-3-5-sonnet-20241022", messages=messages, @@ -521,54 +522,144 @@ def test_vertex_ai_anthropic_output_config_dropped(): headers=headers, ) - # Verify output_config was removed assert ( "output_config" not in result - ), "output_config should be dropped from Vertex AI Anthropic requests" - - # Verify other parameters are preserved - assert result["max_tokens"] == 1024, "max_tokens should be preserved" - assert "messages" in result, "messages should be present" + ), "output_config containing only effort must be dropped" + assert result["max_tokens"] == 1024 + assert "messages" in result -def test_vertex_ai_anthropic_output_format_and_output_config_both_dropped(): +def test_vertex_ai_anthropic_output_config_format_passes_through(): """ - Test that both output_format and output_config are dropped from Vertex AI requests. - - This ensures that even if both parameters somehow make it to the transform_request, - they are properly cleaned up before sending to Vertex AI. + ``output_config`` containing structured-output ``format`` is FORWARDED to + Vertex AI Claude — Vertex now accepts it and uses it for JSON Schema + enforcement. Previously the entire field was being silently stripped, so + Anthropic Structured Outputs never engaged on Vertex even when callers + requested it. """ config = VertexAIAnthropicConfig() + messages = [{"role": "user", "content": "Return a person object."}] + output_config = { + "format": { + "type": "json_schema", + "schema": { + "type": "object", + "additionalProperties": False, + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + }, + } + } + optional_params = {"max_tokens": 1024, "output_config": output_config} + + result = config.transform_request( + model="claude-3-5-sonnet-20241022", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["output_config"] == output_config + + +def test_vertex_ai_anthropic_output_config_format_plus_effort_strips_only_effort(): + """ + Greptile P1 on PR #23396: when ``output_config`` contains BOTH ``format`` + and ``effort``, the prior conditional-passthrough logic forwarded the + full dict including the unsupported ``effort`` key, reproducing the + 400 error the fix was meant to resolve. Only ``effort`` (and any future + Vertex-unsupported keys) should be filtered; ``format`` must survive. + """ + config = VertexAIAnthropicConfig() + messages = [{"role": "user", "content": "Return a person object."}] + + output_config = { + "format": { + "type": "json_schema", + "schema": { + "type": "object", + "additionalProperties": False, + "properties": {"name": {"type": "string"}}, + }, + }, + "effort": "high", + } + optional_params = {"max_tokens": 1024, "output_config": output_config} + + result = config.transform_request( + model="claude-3-5-sonnet-20241022", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "output_config" in result + assert ( + "effort" not in result["output_config"] + ), "effort must be stripped — Vertex returns 400 on unknown keys" + assert result["output_config"]["format"] == output_config["format"] + + +def test_vertex_ai_anthropic_output_config_non_dict_dropped(): + """Defensive: if ``output_config`` is somehow not a dict, drop it rather + than forwarding malformed data downstream.""" + config = VertexAIAnthropicConfig() + messages = [{"role": "user", "content": "hi"}] + optional_params = {"max_tokens": 64, "output_config": "not-a-dict"} + + result = config.transform_request( + model="claude-3-5-sonnet-20241022", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "output_config" not in result + + +def test_vertex_ai_anthropic_output_format_preserved_output_config_effort_dropped(): + """ + When the request carries both ``output_format`` (top-level structured + outputs) AND an ``output_config`` whose only useful key for Vertex is + ``effort``: ``output_format`` must be forwarded (Vertex accepts it), + while ``output_config`` is dropped because Vertex returns 400 on + ``effort``. This replaces the old "drop both" behavior, which was the + silent strip the bug report flagged. + """ + config = VertexAIAnthropicConfig() messages = [{"role": "user", "content": "Extract structured data"}] - headers = {} + + output_format = { + "type": "json_schema", + "json_schema": { + "name": "data", + "schema": { + "type": "object", + "properties": {"result": {"type": "string"}}, + }, + }, + } optional_params = { "max_tokens": 2048, - "output_format": { - "type": "json_schema", - "json_schema": { - "name": "data", - "schema": { - "type": "object", - "properties": {"result": {"type": "string"}}, - }, - }, - }, + "output_format": output_format, "output_config": {"effort": "high"}, } - # Simulate parent class creating test_data with both parameters - # (as if the parent transform_request added them) test_data = { "model": "claude-3-5-sonnet-20241022", "messages": messages, "max_tokens": 2048, - "output_format": optional_params["output_format"], - "output_config": optional_params["output_config"], + "output_format": output_format, + "output_config": {"effort": "high"}, } - # Mock the parent transform_request to return data with both parameters original_transform = config.__class__.__bases__[0].transform_request def mock_transform_request( @@ -584,22 +675,50 @@ def test_vertex_ai_anthropic_output_format_and_output_config_both_dropped(): messages=messages, optional_params=optional_params, litellm_params={}, - headers=headers, + headers={}, ) - # Verify both were removed - assert ( - "output_format" not in result - ), "output_format should be dropped from Vertex AI requests" - assert ( - "output_config" not in result - ), "output_config should be dropped from Vertex AI requests" - - # Verify essential params are preserved - assert result["max_tokens"] == 2048, "max_tokens should be preserved" - assert "messages" in result, "messages should be present" - assert "model" not in result, "model should also be dropped for Vertex AI" - + # output_format flows through unchanged — Vertex AI Claude accepts it. + assert result["output_format"] == output_format + # output_config containing only ``effort`` is dropped to avoid the + # 400 "Extra inputs are not permitted" the silent strip used to mask. + assert "output_config" not in result + assert result["max_tokens"] == 2048 + assert "model" not in result, "model is still stripped (Vertex routes by URL)" finally: - # Restore original method config.__class__.__bases__[0].transform_request = original_transform + + +def test_sanitize_vertex_anthropic_output_params_unit(): + """Direct unit coverage for the helper itself (used by both Vertex + Anthropic transformation paths). Mirrors the integration assertions + above without going through the full ``transform_request`` stack.""" + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.output_params_utils import ( + sanitize_vertex_anthropic_output_params, + ) + + # No-op when output_config absent. + data: dict = {"max_tokens": 8} + sanitize_vertex_anthropic_output_params(data) + assert data == {"max_tokens": 8} + + # Effort-only → dropped entirely. + data = {"output_config": {"effort": "high"}} + sanitize_vertex_anthropic_output_params(data) + assert "output_config" not in data + + # Format-only → preserved unchanged. + fmt = {"format": {"type": "json_schema", "schema": {"type": "object"}}} + data = {"output_config": dict(fmt)} + sanitize_vertex_anthropic_output_params(data) + assert data["output_config"] == fmt + + # Mixed → effort filtered, format kept. + data = {"output_config": {"format": fmt["format"], "effort": "high"}} + sanitize_vertex_anthropic_output_params(data) + assert data["output_config"] == fmt + + # Non-dict → dropped defensively. + data = {"output_config": "garbage"} + sanitize_vertex_anthropic_output_params(data) + assert "output_config" not in data diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index a8021f94d84..fa014de4e62 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -68,6 +68,16 @@ describe("provider_info_helpers", () => { expect(result.logo).toBe(providerLogoMap[Providers.OpenAI]); }); + it("should resolve the zai (Z.AI) provider value to the Z.AI display name", () => { + // Regression test for https://github.com/BerriAI/litellm/issues/25482 — + // the backend already returns `zai` from /public/providers and the docs + // have a dedicated page, but the UI dropdown was missing an entry, so + // `getProviderLogoAndName("zai")` previously returned the raw value as + // the display name (no mapping). + const result = getProviderLogoAndName("zai"); + expect(result.displayName).toBe(Providers.ZAI); + }); + it("should return provider value as display name when no mapping exists", () => { const unknownProvider = "unknown_provider"; const result = getProviderLogoAndName(unknownProvider); @@ -156,6 +166,10 @@ describe("provider_info_helpers", () => { expect(getPlaceholder(Providers.WATSONX)).toBe("watsonx/ibm/granite-3-3-8b-instruct"); }); + it("should return zai/glm-4.5 placeholder for Z.AI provider", () => { + expect(getPlaceholder(Providers.ZAI)).toBe("zai/glm-4.5"); + }); + it("should return default gpt-3.5-turbo placeholder for unknown provider", () => { expect(getPlaceholder("UnknownProvider" as any)).toBe("gpt-3.5-turbo"); }); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index e833d0eb4fb..62c0633d117 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -103,6 +103,7 @@ export enum Providers { WATSONX_TEXT = "Watsonx Text", xAI = "xAI", XINFERENCE = "Xinference", + ZAI = "Z.AI (Zhipu AI)", } export const provider_map: Record = { @@ -210,6 +211,7 @@ export const provider_map: Record = { WATSONX_TEXT: "watsonx_text", xAI: "xai", XINFERENCE: "xinference", + ZAI: "zai", }; const asset_logos_folder = "../ui/assets/logos/"; @@ -366,6 +368,8 @@ export const getPlaceholder = (selectedProvider: string): string => { return "watsonx/ibm/granite-3-3-8b-instruct"; } else if (selectedProvider === Providers.Cursor) { return "cursor/claude-4-sonnet"; + } else if (selectedProvider === Providers.ZAI) { + return "zai/glm-4.5"; } else { return "gpt-3.5-turbo"; }