From 5a75f09d6dd04a080b97d62e996de92c6a57ef19 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:00:14 -0700 Subject: [PATCH] fix(vertex_ai): surface the Gemma container's own error inside a 200 :predict response (#43075) * fix(vertex_ai): surface the Gemma container's own error inside a 200 :predict response * refactor(vertex_ai): move the gemma container error parser next to its adapter --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../vertex_gemma_models/transformation.py | 19 ++- litellm/types/llms/vertex_ai_gemma.py | 10 ++ .../test_vertex_gemma_transformation.py | 159 ++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 litellm/types/llms/vertex_ai_gemma.py diff --git a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py index 2ae8b4cd188..a774dba6cf2 100644 --- a/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py +++ b/litellm/llms/vertex_ai/vertex_gemma_models/transformation.py @@ -12,6 +12,7 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Any, Final, cast import httpx +from pydantic import ValidationError from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -21,6 +22,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.vertex_ai_gemma import VertexGemmaContainerError from litellm.types.utils import ModelResponse if TYPE_CHECKING: @@ -29,6 +31,13 @@ if TYPE_CHECKING: from litellm.llms.base_llm.base_model_iterator import MockResponseIterator +def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None: + try: + return VertexGemmaContainerError.model_validate(predictions) + except ValidationError: + return None + + class VertexGemmaConfig(OpenAIGPTConfig): """ Configuration and transformation class for Vertex AI Gemma models @@ -123,7 +132,9 @@ class VertexGemmaConfig(OpenAIGPTConfig): Unwrap the Vertex Gemma predictions format to OpenAI format. Vertex Gemma wraps the OpenAI-compatible response in a 'predictions' field. - This method extracts it so the parent class can process it normally. + This method extracts it so the parent class can process it normally. A serving + container can also answer with its own OpenAI-shaped error object inside that + field, still under HTTP 200, which is raised with its own status and message. """ if "predictions" not in response_json: raise BaseLLMException( @@ -131,7 +142,11 @@ class VertexGemmaConfig(OpenAIGPTConfig): message="Invalid response format: missing 'predictions' field", ) - return response_json["predictions"] + predictions: Final = response_json["predictions"] + container_error: Final = parse_vertex_gemma_container_error(predictions) + if container_error is None: + return predictions + raise BaseLLMException(status_code=container_error.code, message=container_error.message) @staticmethod def _sync_post( diff --git a/litellm/types/llms/vertex_ai_gemma.py b/litellm/types/llms/vertex_ai_gemma.py new file mode 100644 index 00000000000..f64f4d1d4fc --- /dev/null +++ b/litellm/types/llms/vertex_ai_gemma.py @@ -0,0 +1,10 @@ +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class VertexGemmaContainerError(BaseModel): + model_config = ConfigDict(frozen=True) + object: Literal["error"] + message: str + code: Annotated[int, Field(ge=400, le=599)] diff --git a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py index 5f74f0f602f..e9ae5234094 100644 --- a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py @@ -273,6 +273,165 @@ class TestVertexGemmaCompletion: # Verify the error message contains the original error assert "missing 'predictions' field" in str(exc_info.value) + @pytest.mark.asyncio + @pytest.mark.parametrize("stream", [False, True]) + async def test_acompletion_surfaces_container_error_object_as_its_own_status_and_message(self, stream): + """ + A serving container can reject the request with its own OpenAI-shaped error object, + which Vertex still wraps in an HTTP 200 :predict response. The container's status and + message must reach the caller instead of a 500 "no 'choices'". + """ + from litellm.exceptions import BadRequestError + + container_message = '"auto" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set' + vertex_response = { + "deployedModelId": "123", + "predictions": { + "code": 400, + "message": container_message, + "object": "error", + "param": None, + "type": "BadRequestError", + }, + } + + with ( + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client, + patch( + "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token", + return_value=("fake-access-token", "test-project"), + ), + ): + mock_client = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = vertex_response + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + with pytest.raises(BadRequestError) as exc_info: + await litellm.acompletion( + model="vertex_ai/gemma/gemma-2-2b-it", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {}}}], + stream=stream, + api_base="https://test.prediction.vertexai.goog/v1/projects/test/locations/us-central1/endpoints/123:predict", + vertex_project="test-project", + vertex_location="us-central1", + ) + + assert exc_info.value.status_code == 400 + assert container_message in str(exc_info.value) + assert "no 'choices'" not in str(exc_info.value) + + @pytest.mark.asyncio + async def test_acompletion_keeps_container_error_status_beyond_400(self): + """The container's status is forwarded as is, not collapsed to 400.""" + from litellm.exceptions import RateLimitError + + vertex_response = { + "deployedModelId": "123", + "predictions": {"code": 429, "message": "engine overloaded", "object": "error", "type": "RateLimitError"}, + } + + with ( + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client, + patch( + "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token", + return_value=("fake-access-token", "test-project"), + ), + ): + mock_client = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = vertex_response + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + with pytest.raises(RateLimitError) as exc_info: + await litellm.acompletion( + model="vertex_ai/gemma/gemma-2-2b-it", + messages=[{"role": "user", "content": "Test"}], + api_base="https://test.prediction.vertexai.goog/v1/projects/test/locations/us-central1/endpoints/123:predict", + vertex_project="test-project", + vertex_location="us-central1", + ) + + assert exc_info.value.status_code == 429 + assert "engine overloaded" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_acompletion_error_object_without_http_status_keeps_generic_handling(self): + """An error-shaped body whose code is not an HTTP error status is not trusted as one.""" + from litellm.exceptions import APIError + + vertex_response = { + "deployedModelId": "123", + "predictions": {"code": 0, "message": "unknown failure", "object": "error"}, + } + + with ( + patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client, + patch( + "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token", + return_value=("fake-access-token", "test-project"), + ), + ): + mock_client = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = vertex_response + mock_client.post = AsyncMock(return_value=mock_response) + mock_get_client.return_value = mock_client + + with pytest.raises(APIError) as exc_info: + await litellm.acompletion( + model="vertex_ai/gemma/gemma-2-2b-it", + messages=[{"role": "user", "content": "Test"}], + api_base="https://test.prediction.vertexai.goog/v1/projects/test/locations/us-central1/endpoints/123:predict", + vertex_project="test-project", + vertex_location="us-central1", + ) + + assert exc_info.value.status_code == 500 + + def test_sync_completion_surfaces_container_error_object_as_its_own_status_and_message(self): + """The synchronous path unwraps the same container error object.""" + from litellm.exceptions import BadRequestError + + container_message = '"auto" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set' + vertex_response = { + "deployedModelId": "123", + "predictions": {"code": 400, "message": container_message, "object": "error", "type": "BadRequestError"}, + } + + with ( + patch("litellm.llms.vertex_ai.vertex_gemma_models.transformation._get_httpx_client") as mock_get_client, + patch( + "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token", + return_value=("fake-access-token", "PROJECT_ID"), + ), + ): + mock_client = Mock() + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = vertex_response + mock_client.post = Mock(return_value=mock_response) + mock_get_client.return_value = mock_client + + with pytest.raises(BadRequestError) as exc_info: + litellm.completion( + model="vertex_ai/gemma/gemma-2-2b-it", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {}}}], + api_base="https://test.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict", + vertex_project="PROJECT_ID", + vertex_location="us-central1", + ) + + assert exc_info.value.status_code == 400 + assert container_message in str(exc_info.value) + @pytest.mark.asyncio async def test_acompletion_fake_streaming(self): """