diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 3438e835faf..c8f94b575ad 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -10,6 +10,7 @@ from openai import ( AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, + BadRequestError, OpenAI, ) @@ -37,6 +38,10 @@ from litellm.utils import ( from ...types.llms.openai import HttpxBinaryResponseContent from ..base import BaseLLM +from ..openai.common_utils import ( + build_output_token_limit_response, + is_output_token_limit_error, +) from .common_utils import ( AzureOpenAIError, BaseAzureLLM, @@ -147,6 +152,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): headers: Final = dict(raw_response.headers) response: Final = raw_response.parse() return headers, response + except BadRequestError as e: + if not is_output_token_limit_error(e): + raise + return build_output_token_limit_response(e=e, data=data, is_async=False) except Exception as e: raise e @@ -175,6 +184,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): time_delta: Final = round(end_time - start_time, 2) e.message += f" - timeout value={timeout}, time taken={time_delta} seconds" raise e + except BadRequestError as e: + if not is_output_token_limit_error(e): + raise + return build_output_token_limit_response(e=e, data=data, is_async=True) except Exception as e: raise e diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 82ebee3962e..1b1ab80e85d 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -7,16 +7,25 @@ import inspect import json import os import ssl +import time +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional import httpx import openai from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage +from openai.types.chat.chat_completion import Choice +from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice +from openai.types.chat.chat_completion_chunk import ChoiceDelta +from openai.types.completion_usage import CompletionUsage if TYPE_CHECKING: from aiohttp import ClientSession import litellm +from litellm.litellm_core_utils.token_counter import token_counter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, @@ -111,6 +120,79 @@ def drop_params_from_unprocessable_entity_error( return new_data +_OUTPUT_TOKEN_LIMIT_ERROR_MARKER: Final[str] = ( + "could not finish the message because max_tokens or model output limit was reached" +) + + +def is_output_token_limit_error(e: openai.BadRequestError) -> bool: + """ + True when OpenAI/Azure rejected a chat request because the output budget could not fit a single visible token. + + GPT-5.x turns that case into a 400 while returning a length-truncated 200 for marginally larger budgets, so the + match has to stay pinned to the full provider sentence to avoid swallowing genuine bad requests. + """ + return _OUTPUT_TOKEN_LIMIT_ERROR_MARKER in e.message.lower() + + +def _output_token_limit_completion(model: str, prompt_tokens: int) -> ChatCompletion: + return ChatCompletion( + id=f"chatcmpl-{uuid.uuid4()}", + choices=( + Choice( + index=0, + finish_reason="length", + message=ChatCompletionMessage(role="assistant", content=""), + ), + ), + created=int(time.time()), + model=model, + object="chat.completion", + usage=CompletionUsage(completion_tokens=0, prompt_tokens=prompt_tokens, total_tokens=prompt_tokens), + ) + + +def _output_token_limit_chunk(model: str) -> ChatCompletionChunk: + return ChatCompletionChunk( + id=f"chatcmpl-{uuid.uuid4()}", + choices=( + ChunkChoice( + index=0, + finish_reason="length", + delta=ChoiceDelta(role="assistant", content=""), + ), + ), + created=int(time.time()), + model=model, + object="chat.completion.chunk", + ) + + +def _iter_once(chunk: ChatCompletionChunk) -> Iterator[ChatCompletionChunk]: + yield chunk + + +async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletionChunk]: + yield chunk + + +def build_output_token_limit_response( + e: openai.BadRequestError, data: Mapping[str, object], is_async: bool +) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]: + """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets. + + The provider billed the prompt it processed but sends no usage object with the 400, so the prompt is estimated + the way every other usage-less path estimates it: reporting zero would spend input tokens against no budget. + """ + model: Final[str] = str(data.get("model", "")) + messages: Final = data.get("messages") + prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0 + if not data.get("stream"): + return e.response.headers, _output_token_limit_completion(model, prompt_tokens) + chunk: Final = _output_token_limit_chunk(model) + return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk)) + + class BaseOpenAILLM: """ Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index e96b61d8204..4fc6655ca54 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -46,7 +46,9 @@ from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, OpenAIError, + build_output_token_limit_response, drop_params_from_unprocessable_entity_error, + is_output_token_limit_error, ) openaiOSeriesConfig: Final = OpenAIOSeriesConfig() @@ -436,6 +438,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): time_delta: Final = round(end_time - start_time, 2) e.message += f" - timeout value={timeout}, time taken={time_delta} seconds" raise e + except openai.BadRequestError as e: + if not is_output_token_limit_error(e): + raise + return build_output_token_limit_response(e=e, data=data, is_async=True) except Exception as e: raise e @@ -469,6 +475,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): return headers, response except OpenAIError: raise + except openai.BadRequestError as e: + if not is_output_token_limit_error(e): + raise + return build_output_token_limit_response(e=e, data=data, is_async=False) except Exception as e: if raw_response is not None: raise Exception( diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index a099b5c659f..a28e133700e 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -2,6 +2,8 @@ import os import sys from unittest.mock import MagicMock, call, patch +import httpx +import openai import pytest sys.path.insert( @@ -9,6 +11,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import litellm +from litellm.litellm_core_utils.token_counter import token_counter from litellm.llms.openai.common_utils import BaseOpenAILLM # Test parameters for different API functions @@ -247,3 +250,145 @@ def test_a_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypa closer.reap() assert wrapper.is_closed() is True + + +OUTPUT_LIMIT_400_MESSAGE = ( + "Could not finish the message because max_tokens or model output limit was reached. " + "Please try again with higher max_tokens." +) +GENUINE_400_MESSAGE = "Invalid value for 'max_tokens': integer above maximum value. Expected <= 128000, got 999999999." +LONG_PROMPT = "please summarise the following notes for me: " + ("token " * 200) + +CALL_KWARGS_BY_PROVIDER = { + "openai": {"model": "gpt-5.6-sol", "api_key": "sk-not-a-real-key"}, + "azure": { + "model": "azure/gpt-5.6-sol", + "api_key": "not-a-real-key", + "api_base": "https://not-a-real-resource.openai.azure.com", + "api_version": "2024-10-21", + }, +} + + +def _transport(message: str) -> httpx.MockTransport: + def _handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}}) + + return httpx.MockTransport(_handler) + + +def _sync_client_raising(provider: str, message: str): + http_client = httpx.Client(transport=_transport(message)) + if provider == "azure": + return openai.AzureOpenAI( + api_key="not-a-real-key", + azure_endpoint="https://not-a-real-resource.openai.azure.com", + api_version="2024-10-21", + http_client=http_client, + ) + return openai.OpenAI(api_key="sk-not-a-real-key", http_client=http_client) + + +def _async_client_raising(provider: str, message: str): + http_client = httpx.AsyncClient(transport=_transport(message)) + if provider == "azure": + return openai.AsyncAzureOpenAI( + api_key="not-a-real-key", + azure_endpoint="https://not-a-real-resource.openai.azure.com", + api_version="2024-10-21", + http_client=http_client, + ) + return openai.AsyncOpenAI(api_key="sk-not-a-real-key", http_client=http_client) + + +def _completion_kwargs(provider: str, client, **overrides) -> dict: + return { + **CALL_KWARGS_BY_PROVIDER[provider], + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 1, + "client": client, + **overrides, + } + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +def test_sync_output_limit_400_maps_to_length_truncated_response(provider): + response = litellm.completion( + **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE)) + ) + + assert response.choices[0].finish_reason == "length" + assert response.choices[0].message.content == "" + assert response.usage.completion_tokens == 0 + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +def test_mapped_response_still_bills_the_prompt_the_provider_processed(provider): + messages = [{"role": "user", "content": LONG_PROMPT}] + expected_prompt_tokens = token_counter(model="gpt-5.6-sol", messages=messages) + assert expected_prompt_tokens > 100, "the fixture prompt must be big enough for a zeroed count to stand out" + + response = litellm.completion( + **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), messages=messages) + ) + + assert response.usage.prompt_tokens == expected_prompt_tokens + assert response.usage.completion_tokens == 0 + assert litellm.completion_cost(completion_response=response) > 0 + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +@pytest.mark.asyncio +async def test_async_output_limit_400_maps_to_length_truncated_response(provider): + response = await litellm.acompletion( + **_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE)) + ) + + assert response.choices[0].finish_reason == "length" + assert response.choices[0].message.content == "" + assert response.usage.completion_tokens == 0 + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +def test_sync_streaming_output_limit_400_maps_to_length_truncated_stream(provider): + stream = litellm.completion( + **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), stream=True) + ) + chunks = list(stream) + + assert [c.choices[0].finish_reason for c in chunks].count("length") == 1 + assert all(not c.choices[0].delta.content for c in chunks) + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +@pytest.mark.asyncio +async def test_async_streaming_output_limit_400_maps_to_length_truncated_stream(provider): + stream = await litellm.acompletion( + **_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), stream=True) + ) + chunks = [chunk async for chunk in stream] + + assert [c.choices[0].finish_reason for c in chunks].count("length") == 1 + assert all(not c.choices[0].delta.content for c in chunks) + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +@pytest.mark.parametrize("stream", [False, True]) +def test_sync_genuine_bad_request_still_raises(provider, stream): + with pytest.raises(litellm.BadRequestError): + result = litellm.completion( + **_completion_kwargs(provider, _sync_client_raising(provider, GENUINE_400_MESSAGE), stream=stream) + ) + list(result) + + +@pytest.mark.parametrize("provider", ["openai", "azure"]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_async_genuine_bad_request_still_raises(provider, stream): + with pytest.raises(litellm.BadRequestError): + result = await litellm.acompletion( + **_completion_kwargs(provider, _async_client_raising(provider, GENUINE_400_MESSAGE), stream=stream) + ) + async for _ in result: + pass