fix(openai,azure): return a length-truncated 200 when the output budget fits no token (#36859)

OpenAI and Azure GPT-5.x answer a chat request whose output budget cannot fit a
single visible token with a 400, while the same models return a length-truncated
200 one or two tokens higher. Agents that probe a model with a hardcoded
max_tokens of 1 read that 400 as "model unavailable".

The four chat request helpers now recognise the provider's own sentence and hand
back the length-truncated response the provider gives at a slightly larger
budget: finish_reason "length", empty content, zero completion tokens. Any other
400 still raises. Streaming is covered by the same seam, and the caller's budget
is never raised on their behalf.

The provider bills the prompt it processed but sends no usage object with the
400, so the prompt tokens are estimated with the same token_counter every other
usage-less path uses. Reporting zero would let a caller send an arbitrarily
large prompt with max_tokens 1 and be charged nothing.

litellm/llms/azure/azure.py

@@ -10,6 +10,7 @@ from openai import (
     AsyncAzureOpenAI,
     AsyncOpenAI,
     AzureOpenAI,
+    BadRequestError,
     OpenAI,
 )
 
@@ -37,6 +38,10 @@ from litellm.utils import (
 
 from ...types.llms.openai import HttpxBinaryResponseContent
 from ..base import BaseLLM
+from ..openai.common_utils import (
+    build_output_token_limit_response,
+    is_output_token_limit_error,
+)
 from .common_utils import (
     AzureOpenAIError,
     BaseAzureLLM,
@@ -147,6 +152,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             headers: Final = dict(raw_response.headers)
             response: Final = raw_response.parse()
             return headers, response
+        except BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=False)
         except Exception as e:
             raise e
 
@@ -175,6 +184,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             time_delta: Final = round(end_time - start_time, 2)
             e.message += f" - timeout value={timeout}, time taken={time_delta} seconds"
             raise e
+        except BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=True)
         except Exception as e:
             raise e
 

litellm/llms/openai/common_utils.py

@@ -7,16 +7,25 @@ import inspect
 import json
 import os
 import ssl
+import time
+import uuid
+from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional
 
 import httpx
 import openai
 from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
+from openai.types.chat.chat_completion import Choice
+from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
+from openai.types.chat.chat_completion_chunk import ChoiceDelta
+from openai.types.completion_usage import CompletionUsage
 
 if TYPE_CHECKING:
     from aiohttp import ClientSession
 
 import litellm
+from litellm.litellm_core_utils.token_counter import token_counter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
@@ -111,6 +120,79 @@ def drop_params_from_unprocessable_entity_error(
     return new_data
 
 
+_OUTPUT_TOKEN_LIMIT_ERROR_MARKER: Final[str] = (
+    "could not finish the message because max_tokens or model output limit was reached"
+)
+
+
+def is_output_token_limit_error(e: openai.BadRequestError) -> bool:
+    """
+    True when OpenAI/Azure rejected a chat request because the output budget could not fit a single visible token.
+
+    GPT-5.x turns that case into a 400 while returning a length-truncated 200 for marginally larger budgets, so the
+    match has to stay pinned to the full provider sentence to avoid swallowing genuine bad requests.
+    """
+    return _OUTPUT_TOKEN_LIMIT_ERROR_MARKER in e.message.lower()
+
+
+def _output_token_limit_completion(model: str, prompt_tokens: int) -> ChatCompletion:
+    return ChatCompletion(
+        id=f"chatcmpl-{uuid.uuid4()}",
+        choices=(
+            Choice(
+                index=0,
+                finish_reason="length",
+                message=ChatCompletionMessage(role="assistant", content=""),
+            ),
+        ),
+        created=int(time.time()),
+        model=model,
+        object="chat.completion",
+        usage=CompletionUsage(completion_tokens=0, prompt_tokens=prompt_tokens, total_tokens=prompt_tokens),
+    )
+
+
+def _output_token_limit_chunk(model: str) -> ChatCompletionChunk:
+    return ChatCompletionChunk(
+        id=f"chatcmpl-{uuid.uuid4()}",
+        choices=(
+            ChunkChoice(
+                index=0,
+                finish_reason="length",
+                delta=ChoiceDelta(role="assistant", content=""),
+            ),
+        ),
+        created=int(time.time()),
+        model=model,
+        object="chat.completion.chunk",
+    )
+
+
+def _iter_once(chunk: ChatCompletionChunk) -> Iterator[ChatCompletionChunk]:
+    yield chunk
+
+
+async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletionChunk]:
+    yield chunk
+
+
+def build_output_token_limit_response(
+    e: openai.BadRequestError, data: Mapping[str, object], is_async: bool
+) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]:
+    """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets.
+
+    The provider billed the prompt it processed but sends no usage object with the 400, so the prompt is estimated
+    the way every other usage-less path estimates it: reporting zero would spend input tokens against no budget.
+    """
+    model: Final[str] = str(data.get("model", ""))
+    messages: Final = data.get("messages")
+    prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0
+    if not data.get("stream"):
+        return e.response.headers, _output_token_limit_completion(model, prompt_tokens)
+    chunk: Final = _output_token_limit_chunk(model)
+    return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
+
+
 class BaseOpenAILLM:
     """
     Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings

litellm/llms/openai/openai.py

@@ -46,7 +46,9 @@ from .chat.o_series_transformation import OpenAIOSeriesConfig
 from .common_utils import (
     BaseOpenAILLM,
     OpenAIError,
+    build_output_token_limit_response,
     drop_params_from_unprocessable_entity_error,
+    is_output_token_limit_error,
 )
 
 openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
@@ -436,6 +438,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             time_delta: Final = round(end_time - start_time, 2)
             e.message += f" - timeout value={timeout}, time taken={time_delta} seconds"
             raise e
+        except openai.BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=True)
         except Exception as e:
             raise e
 
@@ -469,6 +475,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             return headers, response
         except OpenAIError:
             raise
+        except openai.BadRequestError as e:
+            if not is_output_token_limit_error(e):
+                raise
+            return build_output_token_limit_response(e=e, data=data, is_async=False)
         except Exception as e:
             if raw_response is not None:
                 raise Exception(

tests/test_litellm/llms/openai/test_openai_common_utils.py

@@ -2,6 +2,8 @@ import os
 import sys
 from unittest.mock import MagicMock, call, patch
 
+import httpx
+import openai
 import pytest
 
 sys.path.insert(
@@ -9,6 +11,7 @@ sys.path.insert(
 )  # Adds the parent directory to the system path
 
 import litellm
+from litellm.litellm_core_utils.token_counter import token_counter
 from litellm.llms.openai.common_utils import BaseOpenAILLM
 
 # Test parameters for different API functions
@@ -247,3 +250,145 @@ def test_a_client_litellm_built_its_own_http_client_for_is_still_closed(monkeypa
     closer.reap()
 
     assert wrapper.is_closed() is True
+
+
+OUTPUT_LIMIT_400_MESSAGE = (
+    "Could not finish the message because max_tokens or model output limit was reached. "
+    "Please try again with higher max_tokens."
+)
+GENUINE_400_MESSAGE = "Invalid value for 'max_tokens': integer above maximum value. Expected <= 128000, got 999999999."
+LONG_PROMPT = "please summarise the following notes for me: " + ("token " * 200)
+
+CALL_KWARGS_BY_PROVIDER = {
+    "openai": {"model": "gpt-5.6-sol", "api_key": "sk-not-a-real-key"},
+    "azure": {
+        "model": "azure/gpt-5.6-sol",
+        "api_key": "not-a-real-key",
+        "api_base": "https://not-a-real-resource.openai.azure.com",
+        "api_version": "2024-10-21",
+    },
+}
+
+
+def _transport(message: str) -> httpx.MockTransport:
+    def _handler(_request: httpx.Request) -> httpx.Response:
+        return httpx.Response(400, json={"error": {"message": message, "type": "invalid_request_error"}})
+
+    return httpx.MockTransport(_handler)
+
+
+def _sync_client_raising(provider: str, message: str):
+    http_client = httpx.Client(transport=_transport(message))
+    if provider == "azure":
+        return openai.AzureOpenAI(
+            api_key="not-a-real-key",
+            azure_endpoint="https://not-a-real-resource.openai.azure.com",
+            api_version="2024-10-21",
+            http_client=http_client,
+        )
+    return openai.OpenAI(api_key="sk-not-a-real-key", http_client=http_client)
+
+
+def _async_client_raising(provider: str, message: str):
+    http_client = httpx.AsyncClient(transport=_transport(message))
+    if provider == "azure":
+        return openai.AsyncAzureOpenAI(
+            api_key="not-a-real-key",
+            azure_endpoint="https://not-a-real-resource.openai.azure.com",
+            api_version="2024-10-21",
+            http_client=http_client,
+        )
+    return openai.AsyncOpenAI(api_key="sk-not-a-real-key", http_client=http_client)
+
+
+def _completion_kwargs(provider: str, client, **overrides) -> dict:
+    return {
+        **CALL_KWARGS_BY_PROVIDER[provider],
+        "messages": [{"role": "user", "content": "hi"}],
+        "max_tokens": 1,
+        "client": client,
+        **overrides,
+    }
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+def test_sync_output_limit_400_maps_to_length_truncated_response(provider):
+    response = litellm.completion(
+        **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE))
+    )
+
+    assert response.choices[0].finish_reason == "length"
+    assert response.choices[0].message.content == ""
+    assert response.usage.completion_tokens == 0
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+def test_mapped_response_still_bills_the_prompt_the_provider_processed(provider):
+    messages = [{"role": "user", "content": LONG_PROMPT}]
+    expected_prompt_tokens = token_counter(model="gpt-5.6-sol", messages=messages)
+    assert expected_prompt_tokens > 100, "the fixture prompt must be big enough for a zeroed count to stand out"
+
+    response = litellm.completion(
+        **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), messages=messages)
+    )
+
+    assert response.usage.prompt_tokens == expected_prompt_tokens
+    assert response.usage.completion_tokens == 0
+    assert litellm.completion_cost(completion_response=response) > 0
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.asyncio
+async def test_async_output_limit_400_maps_to_length_truncated_response(provider):
+    response = await litellm.acompletion(
+        **_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE))
+    )
+
+    assert response.choices[0].finish_reason == "length"
+    assert response.choices[0].message.content == ""
+    assert response.usage.completion_tokens == 0
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+def test_sync_streaming_output_limit_400_maps_to_length_truncated_stream(provider):
+    stream = litellm.completion(
+        **_completion_kwargs(provider, _sync_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), stream=True)
+    )
+    chunks = list(stream)
+
+    assert [c.choices[0].finish_reason for c in chunks].count("length") == 1
+    assert all(not c.choices[0].delta.content for c in chunks)
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.asyncio
+async def test_async_streaming_output_limit_400_maps_to_length_truncated_stream(provider):
+    stream = await litellm.acompletion(
+        **_completion_kwargs(provider, _async_client_raising(provider, OUTPUT_LIMIT_400_MESSAGE), stream=True)
+    )
+    chunks = [chunk async for chunk in stream]
+
+    assert [c.choices[0].finish_reason for c in chunks].count("length") == 1
+    assert all(not c.choices[0].delta.content for c in chunks)
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.parametrize("stream", [False, True])
+def test_sync_genuine_bad_request_still_raises(provider, stream):
+    with pytest.raises(litellm.BadRequestError):
+        result = litellm.completion(
+            **_completion_kwargs(provider, _sync_client_raising(provider, GENUINE_400_MESSAGE), stream=stream)
+        )
+        list(result)
+
+
+@pytest.mark.parametrize("provider", ["openai", "azure"])
+@pytest.mark.parametrize("stream", [False, True])
+@pytest.mark.asyncio
+async def test_async_genuine_bad_request_still_raises(provider, stream):
+    with pytest.raises(litellm.BadRequestError):
+        result = await litellm.acompletion(
+            **_completion_kwargs(provider, _async_client_raising(provider, GENUINE_400_MESSAGE), stream=stream)
+        )
+        async for _ in result:
+            pass