Merge origin/litellm_internal_staging into litellm_e2e_azure_openai_chat

Union the import block with the customer chat/messages coverage that landed
in #34164; the Azure classes and the new provider classes coexist unchanged

tests/e2e/llm_translation/test_chat_completions_regression_e2e.py

@@ -37,6 +37,7 @@ from __future__ import annotations
 import os
 
 import pytest
+from pydantic import BaseModel
 
 from e2e_config import (
     AZURE_CHAT_DEPLOYMENTS,
@@ -46,15 +47,174 @@ from e2e_config import (
     require_env,
     unique_marker,
 )
-from e2e_http import unwrap
+from e2e_http import StreamingResponse, unwrap
 from lifecycle import ResourceManager
-from models import ChatBody, ChatMessage, ChatStreamChunk, LiteLLMParamsBody
+from models import (
+    ChatBody,
+    ChatMessage,
+    ChatResponse,
+    ChatStreamChunk,
+    ChatTool,
+    ChatToolFunction,
+    ImageContentPart,
+    ImageUrl,
+    LiteLLMParamsBody,
+    TextContentPart,
+    ThinkingParam,
+)
 from passthrough_client import PassthroughClient
 
 pytestmark = pytest.mark.e2e
 
 COHERE_BACKEND = "cohere/command-r-08-2024"
 GEMINI_BACKEND = "gemini/gemini-2.5-flash"
+OPENAI_BACKEND = "openai/gpt-5.6"
+BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+
+
+class _StreamToolCallFunction(BaseModel):
+    name: str | None = None
+    arguments: str | None = None
+
+
+class _StreamToolCall(BaseModel):
+    function: _StreamToolCallFunction = _StreamToolCallFunction()
+
+
+class _StreamDelta(BaseModel):
+    content: str | None = None
+    tool_calls: list[_StreamToolCall] | None = None
+
+
+class _StreamChoice(BaseModel):
+    delta: _StreamDelta = _StreamDelta()
+
+
+class _StreamChunk(BaseModel):
+    choices: list[_StreamChoice] = []
+
+
+def _streamed_tool_call(events: list[str]) -> tuple[str, str]:
+    """Reassemble the tool call streamed across chunks: the name arrives once and the
+    arguments arrive as fragments, so concatenating both and parsing the arguments as
+    JSON catches a stream that never completes the call or splits its argument JSON."""
+    chunks = [_StreamChunk.model_validate_json(event) for event in events]
+    calls = [call for chunk in chunks for choice in chunk.choices for call in (choice.delta.tool_calls or [])]
+    name = "".join(call.function.name or "" for call in calls)
+    arguments = "".join(call.function.arguments or "" for call in calls)
+    return name, arguments
+
+
+CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
+OPENAI_VISION_BACKEND = "openai/gpt-4o"
+
+# OpenAI caches a shared prompt prefix once it exceeds ~1024 tokens; this is well
+# past that, so a repeat call reports cached prompt tokens.
+CACHE_PREFIX = (
+    "You are a meticulous assistant. Follow these standing instructions exactly. "
+    * 300
+)
+
+
+def _vision_messages() -> list[ChatMessage]:
+    return [
+        ChatMessage(
+            role="user",
+            content=[
+                TextContentPart(text="What animal is in this image? Answer in one word."),
+                ImageContentPart(image_url=ImageUrl(url=CAT_IMAGE_URL)),
+            ],
+        )
+    ]
+
+
+def _assert_describes_cat(response: ChatResponse) -> None:
+    assert response.choices, f"vision returned no choices: {response}"
+    message = response.choices[0].message
+    content = (message.content if message else None) or ""
+    assert "cat" in content.lower() or "feline" in content.lower(), (
+        f"vision response did not describe the image: {content[:200]}"
+    )
+
+
+def _streamed_text(events: list[str]) -> str:
+    """Concatenate the delta content across streamed chunks. Parsing every event as
+    JSON also fails loudly on a truncated or garbled chunk (the vertex/gemini image
+    streaming regression class), so an incomplete stream cannot pass as content."""
+    chunks = [_StreamChunk.model_validate_json(event) for event in events]
+    return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
+
+
+def _assert_streamed_completion(result: StreamingResponse) -> None:
+    """A streamed /chat/completions must deliver real content, not a clean-but-empty
+    stream (the #28991 class on the streaming path)."""
+    assert result.ok and result.is_streaming, f"stream was not established: {result}"
+    assert result.stream_error is None, f"stream carried an error event: {result.stream_error}"
+    assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}"
+    assert _streamed_text(result.stream_events).strip(), (
+        f"stream completed with no content deltas: {result.stream_events[:3]}"
+    )
+
+
+def _bedrock_params() -> LiteLLMParamsBody:
+    return LiteLLMParamsBody(
+        model=BEDROCK_CONVERSE_BACKEND,
+        aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+        aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+        aws_region_name="os.environ/AWS_REGION",
+    )
+
+
+class _WeatherArgs(BaseModel):
+    location: str
+
+
+_WEATHER_TOOL = ChatTool(
+    function=ChatToolFunction(
+        name="get_weather",
+        description="Get the current weather for a location",
+        parameters={
+            "type": "object",
+            "properties": {"location": {"type": "string"}},
+            "required": ["location"],
+        },
+    )
+)
+
+
+def _assert_weather_tool_call(response: ChatResponse) -> None:
+    """The model, forced to call the tool, must return a get_weather call whose
+    arguments parse as JSON and carry a location. A regression that drops tool_calls
+    or emits malformed argument JSON fails here rather than passing on a 200."""
+    assert response.choices, f"chat returned no choices: {response}"
+    message = response.choices[0].message
+    calls = message.tool_calls if message else None
+    assert calls, f"model returned no tool call for a tool-forced prompt: {response}"
+    weather = next((call for call in calls if call.function.name == "get_weather"), None)
+    assert weather is not None, f"expected a get_weather call, got {[c.function.name for c in calls]}"
+    assert weather.function.arguments, f"get_weather call carried no arguments: {weather}"
+    args = _WeatherArgs.model_validate_json(weather.function.arguments)
+    assert args.location.strip(), f"get_weather arguments missing location: {weather.function.arguments}"
+
+
+class _Person(BaseModel):
+    name: str
+    age: int
+
+
+_PERSON_SCHEMA: dict[str, object] = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "person",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {"name": {"type": "string"}, "age": {"type": "integer"}},
+            "required": ["name", "age"],
+            "additionalProperties": False,
+        },
+    },
+}
 
 CHAT_MODELS: tuple[tuple[str, str], ...] = (
     ("gpt-5.5", "openai"),
@@ -470,3 +630,391 @@ class TestHostedVllmChat:
         assert response.choices, f"hosted_vllm chat returned no choices: {response}"
         content = response.choices[0].message.content if response.choices[0].message else None
         assert content and content.strip(), f"hosted_vllm empty content: {response}"
+
+
+class TestOpenAIChatCompletions:
+    """OpenAI /chat/completions, the SDK path the customer runs against the proxy.
+
+    The streamed call must deliver real content deltas (a clean-but-empty stream is
+    the regression), and a non-streamed call must be costed so per-request spend and
+    the response-cost header stay accurate.
+    """
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.basic.stream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_streams_real_content(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-chat-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key,
+            ChatBody(
+                model=model,
+                messages=[
+                    ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}")
+                ],
+                max_tokens=64,
+                stream=True,
+            ),
+        )
+        _assert_streamed_completion(result)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.basic.nonstream.cost_logged",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_logs_cost(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-cost-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")],
+                    max_tokens=16,
+                ),
+            )
+        )
+        assert response.choices, f"openai chat returned no choices: {response}"
+
+        rows = client.proxy.poll_logs_for_key(
+            key, min_rows=1, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs)
+        )
+        priced = [r for r in rows if (r.spend or 0) > 0]
+        assert priced, f"openai chat was not costed on key ...{key[-6:]}: {rows}"
+        assert priced[0].status == "success", f"openai chat spend status={priced[0].status!r}"
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.tool_use.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_returns_tool_call(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-tool-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[
+                        ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.")
+                    ],
+                    tools=[_WEATHER_TOOL],
+                    tool_choice="required",
+                    max_tokens=128,
+                ),
+            )
+        )
+        _assert_weather_tool_call(response)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.structured_output.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_structured_output_conforms_to_schema(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-schema-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content="Extract the person. John Doe is 42 years old.")],
+                    response_format=_PERSON_SCHEMA,
+                    max_tokens=128,
+                ),
+            )
+        )
+        assert response.choices, f"structured output returned no choices: {response}"
+        content = response.choices[0].message.content if response.choices[0].message else None
+        assert content, f"structured output returned empty content: {response}"
+        person = _Person.model_validate_json(content)
+        assert person.name.strip() and person.age == 42, (
+            f"schema-constrained extraction was wrong: {person}"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.thinking.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_reasoning_reports_reasoning_tokens(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-reasoning-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[
+                        ChatMessage(
+                            role="user",
+                            content="A train travels 60 miles in 1.5 hours. What is its average speed in mph?",
+                        )
+                    ],
+                    reasoning_effort="low",
+                    max_tokens=2048,
+                ),
+            )
+        )
+        assert response.choices, f"reasoning call returned no choices: {response}"
+        message = response.choices[0].message
+        assert message and message.content and message.content.strip(), f"reasoning call had no answer: {response}"
+        details = response.usage.completion_tokens_details if response.usage else None
+        assert details and details.reasoning_tokens and details.reasoning_tokens > 0, (
+            f"a reasoning model must report reasoning tokens, got usage={response.usage}"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.vision.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_vision_describes_image(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-vision-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_VISION_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
+        _assert_describes_cat(response)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.prompt_cache_5m.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_prompt_cache_hits_on_repeat(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-cache-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        body = ChatBody(
+            model=model,
+            messages=[
+                ChatMessage(role="system", content=CACHE_PREFIX),
+                ChatMessage(role="user", content="Reply with the single word pong."),
+            ],
+            max_tokens=16,
+        )
+        unwrap(client.proxy.chat(key, body))
+        second = unwrap(client.proxy.chat(key, body))
+
+        details = second.usage.prompt_tokens_details if second.usage else None
+        assert details and details.cached_tokens and details.cached_tokens > 0, (
+            f"a repeated large-prefix prompt must report cached prompt tokens, got usage={second.usage}"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.tool_use.stream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_streams_tool_call(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-tool-stream-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key,
+            ChatBody(
+                model=model,
+                messages=[
+                    ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.")
+                ],
+                tools=[_WEATHER_TOOL],
+                tool_choice="required",
+                max_tokens=128,
+                stream=True,
+            ),
+        )
+        assert result.ok and result.is_streaming, f"tool stream was not established: {result}"
+        assert result.stream_error is None, f"tool stream carried an error event: {result.stream_error}"
+        name, arguments = _streamed_tool_call(result.stream_events)
+        assert name == "get_weather", f"streamed tool call named {name!r}: {result.stream_events[:5]}"
+        args = _WeatherArgs.model_validate_json(arguments)
+        assert args.location.strip(), f"streamed tool call arguments missing location: {arguments!r}"
+
+
+class TestBedrockConverseChatCompletions:
+    """Bedrock Converse via /chat/completions, the customer's AWS stack. A non-OpenAI
+    provider must return real content on both the non-streamed and streamed paths.
+    """
+
+    def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"{prefix}-{unique_marker()}"
+        model_id = client.proxy.create_model(model, _bedrock_params())
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        return model
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.basic.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_returns_content(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-chat")
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")],
+                    max_tokens=32,
+                ),
+            )
+        )
+        assert response.choices, f"bedrock converse chat returned no choices: {response}"
+        content = response.choices[0].message.content if response.choices[0].message else None
+        assert content and content.strip(), f"bedrock converse returned empty content: {response}"
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.basic.stream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_streams_real_content(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-stream")
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key,
+            ChatBody(
+                model=model,
+                messages=[
+                    ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}")
+                ],
+                max_tokens=64,
+                stream=True,
+            ),
+        )
+        _assert_streamed_completion(result)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.tool_use.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_returns_tool_call(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-tool")
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[
+                        ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.")
+                    ],
+                    tools=[_WEATHER_TOOL],
+                    tool_choice="required",
+                    max_tokens=128,
+                ),
+            )
+        )
+        _assert_weather_tool_call(response)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.thinking.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_returns_reasoning(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-thinking")
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content="What is 17 times 23? Think it through step by step.")],
+                    thinking=ThinkingParam(type="enabled", budget_tokens=1024),
+                    max_tokens=2048,
+                ),
+            )
+        )
+        assert response.choices, f"bedrock thinking returned no choices: {response}"
+        message = response.choices[0].message
+        assert message and message.content and message.content.strip(), (
+            f"bedrock thinking returned no answer content: {response}"
+        )
+        assert message.reasoning_content and message.reasoning_content.strip(), (
+            "thinking was enabled but no reasoning_content came back on the Bedrock Converse path"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.vision.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_vision_describes_image(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-vision")
+        key = resources.key()
+
+        response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
+        _assert_describes_cat(response)

tests/e2e/llm_translation/test_image_generation_e2e.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import pytest
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call
 from endpoints_client import EndpointsClient, ImagesResult
 from lifecycle import ResourceManager
@@ -18,6 +18,15 @@ from models import LiteLLMParamsBody
 pytestmark = pytest.mark.e2e
 
 
+def _assert_image_returned(body: str) -> None:
+    parsed = ImagesResult.model_validate_json(body)
+    assert parsed.data, f"/images/generations returned no data: {body[:300]}"
+    first = parsed.data[0]
+    assert first.b64_json or first.url, (
+        f"generated image has neither b64_json nor url: {body[:300]}"
+    )
+
+
 class TestImageGeneration:
     @pytest.mark.covers("llm.images_generations.openai.basic.nonstream.works")
     def test_image_generation_returns_image(
@@ -35,9 +44,26 @@ class TestImageGeneration:
 
         result = endpoints_client.images(key, model, "Draw a cute cat")
         require_successful_call(result)
-        parsed = ImagesResult.model_validate_json(result.body)
-        assert parsed.data, f"/images/generations returned no data: {result.body[:300]}"
-        first = parsed.data[0]
-        assert first.b64_json or first.url, (
-            f"generated image has neither b64_json nor url: {result.body[:300]}"
+        _assert_image_returned(result.body)
+
+    @pytest.mark.covers("llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"])
+    def test_bedrock_image_generation_returns_image(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-bedrock-image-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="bedrock/amazon.titan-image-generator-v2:0",
+                aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+                aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+                aws_region_name="os.environ/AWS_REGION",
+            ),
         )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.images(key, model, "Draw a cute cat")
+        require_successful_call(result)
+        _assert_image_returned(result.body)

tests/e2e/llm_translation/test_messages_e2e.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 import pytest
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call, unwrap
 from endpoints_client import EndpointsClient, MessagesResult
 from lifecycle import ResourceManager
@@ -20,11 +20,14 @@ from models import (
     ChatMessage,
     JsonSchemaProperty,
     LiteLLMParamsBody,
+    SpendLogRow,
     ToolInputSchema,
 )
 
 pytestmark = pytest.mark.e2e
 
+ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5"
+
 WEATHER_TOOL = AnthropicCustomTool(
     name="get_weather",
     description="Get the current weather for a city.",
@@ -35,6 +38,11 @@ WEATHER_TOOL = AnthropicCustomTool(
 )
 
 
+def _approx_equal(actual: float, expected: float) -> bool:
+    """Within 1% or 1e-9 absolute - spend math, not exact float identity."""
+    return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2)
+
+
 class TestAnthropicMessages:
     def _register(
         self, endpoints_client: EndpointsClient, resources: ResourceManager
@@ -43,7 +51,7 @@ class TestAnthropicMessages:
         model_id = endpoints_client.create_model(
             model,
             LiteLLMParamsBody(
-                model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY"
+                model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"
             ),
         )
         resources.defer(lambda: endpoints_client.delete_model(model_id))
@@ -61,6 +69,58 @@ class TestAnthropicMessages:
         assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}"
         assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}"
 
+    @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.cost_logged")
+    def test_messages_logs_cost_matching_the_response_header(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("ANTHROPIC_API_KEY")
+        model = f"e2e-messages-cost-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.messages(key, model, f"reply with one word {unique_marker()}")
+        require_successful_call(result)
+        parsed = MessagesResult.model_validate_json(result.body)
+        assert parsed.role == "assistant" and parsed.text.strip(), (
+            f"/v1/messages returned no assistant text: {result.body[:300]}"
+        )
+
+        # The customer reads per-request cost off the response header (LIT-4076), so
+        # it must be present and positive on /v1/messages, not only /chat/completions.
+        header_cost = result.response_cost
+        assert header_cost is not None and header_cost > 0, (
+            "x-litellm-response-cost header missing or non-positive on /v1/messages; "
+            f"headers={result.headers}"
+        )
+
+        # Correlate the spend row by the unique scoped key, not the Anthropic response
+        # id: on /v1/messages the spend-log request_id is the proxy's own call id, which
+        # need not equal the message body id, so an id-based poll can miss a correctly
+        # logged row and time out. The key is fresh per test, so its only priced row is
+        # this call.
+        def _priced(rows: list[SpendLogRow]) -> bool:
+            return any(r.spend is not None and r.spend > 0 for r in rows)
+
+        rows = endpoints_client.proxy.poll_logs_for_key(key, predicate=_priced)
+        priced = [r for r in rows if r.spend is not None and r.spend > 0]
+        assert priced, (
+            f"no priced /spend/logs row landed for key {key} within the poll window; got {rows}"
+        )
+        row = priced[0]
+        assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, (
+            f"messages spend row missing token counts, so the cost is not real usage: {row}"
+        )
+        assert row.spend is not None and _approx_equal(row.spend, header_cost), (
+            f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}; "
+            "the customer bills against the header, so the two must match"
+        )
+
     @pytest.mark.covers("llm.messages.anthropic.basic.stream.works")
     def test_messages_streams_completion(
         self, endpoints_client: EndpointsClient, resources: ResourceManager

tests/e2e/llm_translation/test_passthrough_e2e.py

@@ -15,7 +15,8 @@ import pytest
 
 from e2e_config import unique_marker
 from e2e_http import StreamingResponse, require_successful_call
-from models import SpendLogRow
+from lifecycle import ResourceManager
+from models import KeyGenerateBody, SpendLogRow
 from passthrough_client import (
     AnthropicTool,
     GeminiFunctionDeclaration,
@@ -157,3 +158,25 @@ def test_anthropic_passthrough_tool_call_logs_cost(
 
     row = _fetch_cost_breakdown(client, result)
     assert row.custom_llm_provider == "anthropic"
+
+
+class TestPassthroughModelAllowlist:
+    """A passthrough route must honor the calling key's model allow-list.
+
+    The customer fronts native provider calls through the proxy with custom auth,
+    so a key scoped to one model must not reach a different model just because the
+    request goes through the passthrough route rather than /chat/completions.
+    """
+
+    @pytest.mark.covers("other.auth.passthrough.model_allowlist_enforced")
+    def test_passthrough_denies_model_outside_key_allowlist(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        key = client.proxy.generate_key(KeyGenerateBody(models=["gemini-2.5-flash"]))
+        resources.defer(lambda: client.proxy.delete_key(key))
+
+        result = client.anthropic_message(key, "claude-haiku-4-5", f"say hi {unique_marker()}")
+        assert result.status_code == 403, (
+            "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
+            f"got {result.status_code}: {result.body[:300]}"
+        )

tests/e2e/llm_translation/test_rerank_e2e.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import pytest
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call
 from endpoints_client import EndpointsClient, RerankResult
 from lifecycle import ResourceManager
@@ -23,6 +23,16 @@ DOCUMENTS = [
     "Washington, D.C. is the capital of the United States.",
     "Capital punishment has existed in the United States since before it was a country.",
 ]
+QUERY = "What is the capital of the United States?"
+
+
+def _assert_top_n_scored(body: str) -> None:
+    parsed = RerankResult.model_validate_json(body)
+    assert parsed.results, f"/rerank returned no results: {body[:300]}"
+    assert len(parsed.results) <= 3, f"top_n=3 not honored: {body[:300]}"
+    assert parsed.results[0].relevance_score is not None, (
+        f"top rerank result has no relevance_score: {body[:300]}"
+    )
 
 
 class TestRerank:
@@ -38,13 +48,28 @@ class TestRerank:
         resources.defer(lambda: endpoints_client.delete_model(model_id))
         key = resources.key()
 
-        result = endpoints_client.rerank(
-            key, model, "What is the capital of the United States?", DOCUMENTS, top_n=3
-        )
+        result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3)
         require_successful_call(result)
-        parsed = RerankResult.model_validate_json(result.body)
-        assert parsed.results, f"/rerank returned no results: {result.body[:300]}"
-        assert len(parsed.results) <= 3, f"top_n=3 not honored: {result.body[:300]}"
-        assert parsed.results[0].relevance_score is not None, (
-            f"top rerank result has no relevance_score: {result.body[:300]}"
+        _assert_top_n_scored(result.body)
+
+    @pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"])
+    def test_bedrock_rerank_scores_top_n(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-bedrock-rerank-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="bedrock/amazon.rerank-v1:0",
+                aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+                aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+                aws_region_name="os.environ/AWS_REGION",
+            ),
         )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3)
+        require_successful_call(result)
+        _assert_top_n_scored(result.body)

tests/e2e/llm_translation/test_responses_e2e.py

@@ -13,7 +13,7 @@ from typing import cast
 import pytest
 from pydantic import BaseModel, ValidationError
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call
 from endpoints_client import (
     EndpointsClient,
@@ -29,6 +29,26 @@ from models import LiteLLMParamsBody
 
 pytestmark = pytest.mark.e2e
 
+BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+
+WEATHER_TOOL = ResponsesFunctionTool(
+    name="get_weather",
+    description="Get the weather for a location",
+    parameters=FunctionParameters(
+        properties={"location": FunctionParameterProperty(type="string")},
+        required=["location"],
+    ),
+)
+
+
+def _bedrock_params() -> LiteLLMParamsBody:
+    return LiteLLMParamsBody(
+        model=BEDROCK_CONVERSE_BACKEND,
+        aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+        aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+        aws_region_name="os.environ/AWS_REGION",
+    )
+
 
 class WeatherArguments(BaseModel):
     location: str
@@ -190,6 +210,84 @@ class TestResponses:
         parsed = ResponsesResult.model_validate_json(result.body)
         assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}"
 
+    @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works")
+    def test_responses_anthropic_returns_function_call(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = f"e2e-responses-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.responses_with_tools(
+            key,
+            model,
+            "What is the weather in San Francisco? Use the get_weather tool.",
+            [
+                ResponsesFunctionTool(
+                    name="get_weather",
+                    description="Get the weather for a location",
+                    parameters=FunctionParameters(
+                        properties={"location": FunctionParameterProperty(type="string")},
+                        required=["location"],
+                    ),
+                )
+            ],
+        )
+        require_successful_call(result)
+        parsed = ResponsesResult.model_validate_json(result.body)
+        function_call = next(
+            (call for call in parsed.function_calls if call.name == "get_weather"),
+            None,
+        )
+        assert function_call is not None, f"no get_weather function call: {result.body[:500]}"
+        assert function_call.arguments is not None
+        raw_arguments = cast(object, json.loads(function_call.arguments))
+        arguments = WeatherArguments.model_validate(raw_arguments)
+        assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
+
+    @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works")
+    def test_responses_bedrock_returns_completion(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-responses-{unique_marker()}"
+        model_id = endpoints_client.create_model(model, _bedrock_params())
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.responses(key, model, "reply with one word")
+        require_successful_call(result)
+        parsed = ResponsesResult.model_validate_json(result.body)
+        assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}"
+
+    @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
+    def test_responses_bedrock_returns_function_call(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-responses-{unique_marker()}"
+        model_id = endpoints_client.create_model(model, _bedrock_params())
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.responses_with_tools(
+            key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL]
+        )
+        require_successful_call(result)
+        parsed = ResponsesResult.model_validate_json(result.body)
+        function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None)
+        assert function_call is not None, f"no get_weather function call over bedrock: {result.body[:500]}"
+        assert function_call.arguments is not None
+        raw_arguments = cast(object, json.loads(function_call.arguments))
+        arguments = WeatherArguments.model_validate(raw_arguments)
+        assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
+
 
 def _parse_stream_event(
     event: str,

tests/e2e/management/management_client.py

@@ -14,6 +14,10 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un
 from models import (
     ChatBody,
     ChatMessage,
+    CustomerDeleteBody,
+    CustomerInfoParams,
+    CustomerNewBody,
+    CustomerResponse,
     KeyBlockBody,
     KeyDeleteBody,
     KeyGenerateBody,
@@ -270,6 +274,35 @@ class ManagementClient:
             )
         ).user_id
 
+    def create_customer(self, user_id: str) -> str:
+        _ = unwrap(
+            self.proxy.transport.post(
+                "/customer/new",
+                headers=self.proxy.transport.master,
+                json=CustomerNewBody(user_id=user_id),
+                response_type=CustomerResponse,
+            )
+        )
+        return user_id
+
+    def customer_info(self, end_user_id: str) -> CustomerResponse:
+        return unwrap(
+            self.proxy.transport.get(
+                "/customer/info",
+                headers=self.proxy.transport.master,
+                params=CustomerInfoParams(end_user_id=end_user_id),
+                response_type=CustomerResponse,
+            )
+        )
+
+    def delete_customer(self, user_id: str) -> None:
+        _ = self.proxy.transport.post(
+            "/customer/delete",
+            headers=self.proxy.transport.master,
+            json=CustomerDeleteBody(user_ids=[user_id]),
+            response_type=NoBody,
+        )
+
     def update_user(self, body: UserUpdateBody) -> None:
         _ = unwrap(
             self.proxy.transport.post(

tests/e2e/management/test_management_e2e.py

@@ -610,3 +610,18 @@ class TestManagementRoutePermissions:
             f"/team/info returned {team_probe.status_code}: {team_probe.body[:300]}"
         )
         assert client.user_count(user_id) == 0, f"user {user_id} was created despite the 403 route denial"
+
+
+class TestCustomer:
+    @pytest.mark.covers("mgmt.end_user.new.happy_path")
+    def test_customer_create_persists_to_info(
+        self, client: ManagementClient, resources: ResourceManager
+    ) -> None:
+        customer = f"e2e-customer-{unique_marker()}"
+        client.create_customer(customer)
+        resources.defer(lambda: client.delete_customer(customer))
+
+        info = client.customer_info(customer)
+        assert info.user_id == customer, (
+            f"/customer/info did not report the created end-user; got {info.user_id!r}"
+        )

tests/e2e/models.py

@@ -115,6 +115,18 @@ class KeyInfoResponse(BaseModel):
 # ---------- customers ----------
 
 
+class CustomerNewBody(BaseModel):
+    user_id: str
+
+
+class CustomerResponse(BaseModel):
+    user_id: str | None = None
+
+
+class CustomerInfoParams(BaseModel):
+    end_user_id: str
+
+
 class CustomerDeleteBody(BaseModel):
     user_ids: list[str]
 
@@ -126,9 +138,26 @@ class ChatMetadata(BaseModel):
     tags: list[str] | None = None
 
 
+class ImageUrl(BaseModel):
+    url: str
+
+
+class TextContentPart(BaseModel):
+    type: str = "text"
+    text: str
+
+
+class ImageContentPart(BaseModel):
+    type: str = "image_url"
+    image_url: ImageUrl
+
+
+ContentPart = TextContentPart | ImageContentPart
+
+
 class ChatMessage(BaseModel):
     role: str
-    content: str
+    content: str | list[ContentPart]
 
 
 class CacheControl(BaseModel):
@@ -181,6 +210,7 @@ class ChatBody(BaseModel):
     tools: list[ChatTool] | None = None
     tool_choice: str | None = None
     guardrails: list[str] | None = None
+    response_format: dict[str, object] | None = None
 
 
 class RouterSettingsOverride(BaseModel):
@@ -204,9 +234,19 @@ class ReliabilityChatBody(ChatBody):
     router_settings_override: RouterSettingsOverride | None = None
 
 
+class ToolCallFunction(BaseModel):
+    name: str | None = None
+    arguments: str | None = None
+
+
+class ToolCall(BaseModel):
+    function: ToolCallFunction = ToolCallFunction()
+
+
 class OutMessage(BaseModel):
     content: str | None = None
     reasoning_content: str | None = None
+    tool_calls: list[ToolCall] | None = None
 
 
 class ChatChoice(BaseModel):