fix(vertex_ai): surface the Gemma container's own error inside a 200 :predict response (#43075)

* fix(vertex_ai): surface the Gemma container's own error inside a 200 :predict response

* refactor(vertex_ai): move the gemma container error parser next to its adapter

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>

litellm/llms/vertex_ai/vertex_gemma_models/transformation.py

@@ -12,6 +12,7 @@ from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, Final, cast
 
 import httpx
+from pydantic import ValidationError
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -21,6 +22,7 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.types.llms.openai import AllMessageValues
+from litellm.types.llms.vertex_ai_gemma import VertexGemmaContainerError
 from litellm.types.utils import ModelResponse
 
 if TYPE_CHECKING:
@@ -29,6 +31,13 @@ if TYPE_CHECKING:
     from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
 
 
+def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
+    try:
+        return VertexGemmaContainerError.model_validate(predictions)
+    except ValidationError:
+        return None
+
+
 class VertexGemmaConfig(OpenAIGPTConfig):
     """
     Configuration and transformation class for Vertex AI Gemma models
@@ -123,7 +132,9 @@ class VertexGemmaConfig(OpenAIGPTConfig):
         Unwrap the Vertex Gemma predictions format to OpenAI format.
 
         Vertex Gemma wraps the OpenAI-compatible response in a 'predictions' field.
-        This method extracts it so the parent class can process it normally.
+        This method extracts it so the parent class can process it normally. A serving
+        container can also answer with its own OpenAI-shaped error object inside that
+        field, still under HTTP 200, which is raised with its own status and message.
         """
         if "predictions" not in response_json:
             raise BaseLLMException(
@@ -131,7 +142,11 @@ class VertexGemmaConfig(OpenAIGPTConfig):
                 message="Invalid response format: missing 'predictions' field",
             )
 
-        return response_json["predictions"]
+        predictions: Final = response_json["predictions"]
+        container_error: Final = parse_vertex_gemma_container_error(predictions)
+        if container_error is None:
+            return predictions
+        raise BaseLLMException(status_code=container_error.code, message=container_error.message)
 
     @staticmethod
     def _sync_post(

litellm/types/llms/vertex_ai_gemma.py

@@ -0,0 +1,10 @@
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class VertexGemmaContainerError(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    object: Literal["error"]
+    message: str
+    code: Annotated[int, Field(ge=400, le=599)]

tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py

@@ -273,6 +273,165 @@ class TestVertexGemmaCompletion:
             # Verify the error message contains the original error
             assert "missing 'predictions' field" in str(exc_info.value)
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("stream", [False, True])
+    async def test_acompletion_surfaces_container_error_object_as_its_own_status_and_message(self, stream):
+        """
+        A serving container can reject the request with its own OpenAI-shaped error object,
+        which Vertex still wraps in an HTTP 200 :predict response. The container's status and
+        message must reach the caller instead of a 500 "no 'choices'".
+        """
+        from litellm.exceptions import BadRequestError
+
+        container_message = '"auto" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set'
+        vertex_response = {
+            "deployedModelId": "123",
+            "predictions": {
+                "code": 400,
+                "message": container_message,
+                "object": "error",
+                "param": None,
+                "type": "BadRequestError",
+            },
+        }
+
+        with (
+            patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client,
+            patch(
+                "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token",
+                return_value=("fake-access-token", "test-project"),
+            ),
+        ):
+            mock_client = Mock()
+            mock_response = Mock()
+            mock_response.status_code = 200
+            mock_response.json.return_value = vertex_response
+            mock_client.post = AsyncMock(return_value=mock_response)
+            mock_get_client.return_value = mock_client
+
+            with pytest.raises(BadRequestError) as exc_info:
+                await litellm.acompletion(
+                    model="vertex_ai/gemma/gemma-2-2b-it",
+                    messages=[{"role": "user", "content": "What is the weather in Paris?"}],
+                    tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {}}}],
+                    stream=stream,
+                    api_base="https://test.prediction.vertexai.goog/v1/projects/test/locations/us-central1/endpoints/123:predict",
+                    vertex_project="test-project",
+                    vertex_location="us-central1",
+                )
+
+        assert exc_info.value.status_code == 400
+        assert container_message in str(exc_info.value)
+        assert "no 'choices'" not in str(exc_info.value)
+
+    @pytest.mark.asyncio
+    async def test_acompletion_keeps_container_error_status_beyond_400(self):
+        """The container's status is forwarded as is, not collapsed to 400."""
+        from litellm.exceptions import RateLimitError
+
+        vertex_response = {
+            "deployedModelId": "123",
+            "predictions": {"code": 429, "message": "engine overloaded", "object": "error", "type": "RateLimitError"},
+        }
+
+        with (
+            patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client,
+            patch(
+                "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token",
+                return_value=("fake-access-token", "test-project"),
+            ),
+        ):
+            mock_client = Mock()
+            mock_response = Mock()
+            mock_response.status_code = 200
+            mock_response.json.return_value = vertex_response
+            mock_client.post = AsyncMock(return_value=mock_response)
+            mock_get_client.return_value = mock_client
+
+            with pytest.raises(RateLimitError) as exc_info:
+                await litellm.acompletion(
+                    model="vertex_ai/gemma/gemma-2-2b-it",
+                    messages=[{"role": "user", "content": "Test"}],
+                    api_base="https://test.prediction.vertexai.goog/v1/projects/test/locations/us-central1/endpoints/123:predict",
+                    vertex_project="test-project",
+                    vertex_location="us-central1",
+                )
+
+        assert exc_info.value.status_code == 429
+        assert "engine overloaded" in str(exc_info.value)
+
+    @pytest.mark.asyncio
+    async def test_acompletion_error_object_without_http_status_keeps_generic_handling(self):
+        """An error-shaped body whose code is not an HTTP error status is not trusted as one."""
+        from litellm.exceptions import APIError
+
+        vertex_response = {
+            "deployedModelId": "123",
+            "predictions": {"code": 0, "message": "unknown failure", "object": "error"},
+        }
+
+        with (
+            patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client,
+            patch(
+                "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token",
+                return_value=("fake-access-token", "test-project"),
+            ),
+        ):
+            mock_client = Mock()
+            mock_response = Mock()
+            mock_response.status_code = 200
+            mock_response.json.return_value = vertex_response
+            mock_client.post = AsyncMock(return_value=mock_response)
+            mock_get_client.return_value = mock_client
+
+            with pytest.raises(APIError) as exc_info:
+                await litellm.acompletion(
+                    model="vertex_ai/gemma/gemma-2-2b-it",
+                    messages=[{"role": "user", "content": "Test"}],
+                    api_base="https://test.prediction.vertexai.goog/v1/projects/test/locations/us-central1/endpoints/123:predict",
+                    vertex_project="test-project",
+                    vertex_location="us-central1",
+                )
+
+        assert exc_info.value.status_code == 500
+
+    def test_sync_completion_surfaces_container_error_object_as_its_own_status_and_message(self):
+        """The synchronous path unwraps the same container error object."""
+        from litellm.exceptions import BadRequestError
+
+        container_message = '"auto" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set'
+        vertex_response = {
+            "deployedModelId": "123",
+            "predictions": {"code": 400, "message": container_message, "object": "error", "type": "BadRequestError"},
+        }
+
+        with (
+            patch("litellm.llms.vertex_ai.vertex_gemma_models.transformation._get_httpx_client") as mock_get_client,
+            patch(
+                "litellm.llms.vertex_ai.vertex_gemma_models.main.VertexAIGemmaModels._ensure_access_token",
+                return_value=("fake-access-token", "PROJECT_ID"),
+            ),
+        ):
+            mock_client = Mock()
+            mock_response = Mock()
+            mock_response.status_code = 200
+            mock_response.json.return_value = vertex_response
+            mock_client.post = Mock(return_value=mock_response)
+            mock_get_client.return_value = mock_client
+
+            with pytest.raises(BadRequestError) as exc_info:
+                litellm.completion(
+                    model="vertex_ai/gemma/gemma-2-2b-it",
+                    messages=[{"role": "user", "content": "What is the weather in Paris?"}],
+                    tools=[{"type": "function", "function": {"name": "get_weather", "parameters": {}}}],
+                    api_base="https://test.prediction.vertexai.goog/v1/projects/PROJECT_ID/locations/us-central1/endpoints/ENDPOINT_ID:predict",
+                    vertex_project="PROJECT_ID",
+                    vertex_location="us-central1",
+                )
+
+        assert exc_info.value.status_code == 400
+        assert container_message in str(exc_info.value)
+
     @pytest.mark.asyncio
     async def test_acompletion_fake_streaming(self):
         """