diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index c4b427807b5..13b48e1fd0f 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -37,6 +37,7 @@ from __future__ import annotations import os import pytest +from pydantic import BaseModel from e2e_config import ( AZURE_CHAT_DEPLOYMENTS, @@ -46,15 +47,174 @@ from e2e_config import ( require_env, unique_marker, ) -from e2e_http import unwrap +from e2e_http import StreamingResponse, unwrap from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamChunk, LiteLLMParamsBody +from models import ( + ChatBody, + ChatMessage, + ChatResponse, + ChatStreamChunk, + ChatTool, + ChatToolFunction, + ImageContentPart, + ImageUrl, + LiteLLMParamsBody, + TextContentPart, + ThinkingParam, +) from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" GEMINI_BACKEND = "gemini/gemini-2.5-flash" +OPENAI_BACKEND = "openai/gpt-5.6" +BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +class _StreamToolCallFunction(BaseModel): + name: str | None = None + arguments: str | None = None + + +class _StreamToolCall(BaseModel): + function: _StreamToolCallFunction = _StreamToolCallFunction() + + +class _StreamDelta(BaseModel): + content: str | None = None + tool_calls: list[_StreamToolCall] | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta = _StreamDelta() + + +class _StreamChunk(BaseModel): + choices: list[_StreamChoice] = [] + + +def _streamed_tool_call(events: list[str]) -> tuple[str, str]: + """Reassemble the tool call streamed across chunks: the name arrives once and the + arguments arrive as fragments, so concatenating both and parsing the arguments as + JSON catches a stream that never completes the call or splits its argument JSON.""" + chunks = [_StreamChunk.model_validate_json(event) for event in events] + calls = [call for chunk in chunks for choice in chunk.choices for call in (choice.delta.tool_calls or [])] + name = "".join(call.function.name or "" for call in calls) + arguments = "".join(call.function.arguments or "" for call in calls) + return name, arguments + + +CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" +OPENAI_VISION_BACKEND = "openai/gpt-4o" + +# OpenAI caches a shared prompt prefix once it exceeds ~1024 tokens; this is well +# past that, so a repeat call reports cached prompt tokens. +CACHE_PREFIX = ( + "You are a meticulous assistant. Follow these standing instructions exactly. " + * 300 +) + + +def _vision_messages() -> list[ChatMessage]: + return [ + ChatMessage( + role="user", + content=[ + TextContentPart(text="What animal is in this image? Answer in one word."), + ImageContentPart(image_url=ImageUrl(url=CAT_IMAGE_URL)), + ], + ) + ] + + +def _assert_describes_cat(response: ChatResponse) -> None: + assert response.choices, f"vision returned no choices: {response}" + message = response.choices[0].message + content = (message.content if message else None) or "" + assert "cat" in content.lower() or "feline" in content.lower(), ( + f"vision response did not describe the image: {content[:200]}" + ) + + +def _streamed_text(events: list[str]) -> str: + """Concatenate the delta content across streamed chunks. Parsing every event as + JSON also fails loudly on a truncated or garbled chunk (the vertex/gemini image + streaming regression class), so an incomplete stream cannot pass as content.""" + chunks = [_StreamChunk.model_validate_json(event) for event in events] + return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + + +def _assert_streamed_completion(result: StreamingResponse) -> None: + """A streamed /chat/completions must deliver real content, not a clean-but-empty + stream (the #28991 class on the streaming path).""" + assert result.ok and result.is_streaming, f"stream was not established: {result}" + assert result.stream_error is None, f"stream carried an error event: {result.stream_error}" + assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}" + assert _streamed_text(result.stream_events).strip(), ( + f"stream completed with no content deltas: {result.stream_events[:3]}" + ) + + +def _bedrock_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=BEDROCK_CONVERSE_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ) + + +class _WeatherArgs(BaseModel): + location: str + + +_WEATHER_TOOL = ChatTool( + function=ChatToolFunction( + name="get_weather", + description="Get the current weather for a location", + parameters={ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + ) +) + + +def _assert_weather_tool_call(response: ChatResponse) -> None: + """The model, forced to call the tool, must return a get_weather call whose + arguments parse as JSON and carry a location. A regression that drops tool_calls + or emits malformed argument JSON fails here rather than passing on a 200.""" + assert response.choices, f"chat returned no choices: {response}" + message = response.choices[0].message + calls = message.tool_calls if message else None + assert calls, f"model returned no tool call for a tool-forced prompt: {response}" + weather = next((call for call in calls if call.function.name == "get_weather"), None) + assert weather is not None, f"expected a get_weather call, got {[c.function.name for c in calls]}" + assert weather.function.arguments, f"get_weather call carried no arguments: {weather}" + args = _WeatherArgs.model_validate_json(weather.function.arguments) + assert args.location.strip(), f"get_weather arguments missing location: {weather.function.arguments}" + + +class _Person(BaseModel): + name: str + age: int + + +_PERSON_SCHEMA: dict[str, object] = { + "type": "json_schema", + "json_schema": { + "name": "person", + "strict": True, + "schema": { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, + "required": ["name", "age"], + "additionalProperties": False, + }, + }, +} CHAT_MODELS: tuple[tuple[str, str], ...] = ( ("gpt-5.5", "openai"), @@ -470,3 +630,391 @@ class TestHostedVllmChat: assert response.choices, f"hosted_vllm chat returned no choices: {response}" content = response.choices[0].message.content if response.choices[0].message else None assert content and content.strip(), f"hosted_vllm empty content: {response}" + + +class TestOpenAIChatCompletions: + """OpenAI /chat/completions, the SDK path the customer runs against the proxy. + + The streamed call must deliver real content deltas (a clean-but-empty stream is + the regression), and a non-streamed call must be costed so per-request spend and + the response-cost header stay accurate. + """ + + @pytest.mark.covers( + "llm.chat_completions.openai.basic.stream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_streams_real_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}") + ], + max_tokens=64, + stream=True, + ), + ) + _assert_streamed_completion(result) + + @pytest.mark.covers( + "llm.chat_completions.openai.basic.nonstream.cost_logged", + exercised_on=["chat_completions"], + ) + def test_openai_chat_logs_cost( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-cost-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")], + max_tokens=16, + ), + ) + ) + assert response.choices, f"openai chat returned no choices: {response}" + + rows = client.proxy.poll_logs_for_key( + key, min_rows=1, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs) + ) + priced = [r for r in rows if (r.spend or 0) > 0] + assert priced, f"openai chat was not costed on key ...{key[-6:]}: {rows}" + assert priced[0].status == "success", f"openai chat spend status={priced[0].status!r}" + + @pytest.mark.covers( + "llm.chat_completions.openai.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-tool-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.") + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + @pytest.mark.covers( + "llm.chat_completions.openai.structured_output.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_structured_output_conforms_to_schema( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-schema-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="Extract the person. John Doe is 42 years old.")], + response_format=_PERSON_SCHEMA, + max_tokens=128, + ), + ) + ) + assert response.choices, f"structured output returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content, f"structured output returned empty content: {response}" + person = _Person.model_validate_json(content) + assert person.name.strip() and person.age == 42, ( + f"schema-constrained extraction was wrong: {person}" + ) + + @pytest.mark.covers( + "llm.chat_completions.openai.thinking.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_reasoning_reports_reasoning_tokens( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-reasoning-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="A train travels 60 miles in 1.5 hours. What is its average speed in mph?", + ) + ], + reasoning_effort="low", + max_tokens=2048, + ), + ) + ) + assert response.choices, f"reasoning call returned no choices: {response}" + message = response.choices[0].message + assert message and message.content and message.content.strip(), f"reasoning call had no answer: {response}" + details = response.usage.completion_tokens_details if response.usage else None + assert details and details.reasoning_tokens and details.reasoning_tokens > 0, ( + f"a reasoning model must report reasoning tokens, got usage={response.usage}" + ) + + @pytest.mark.covers( + "llm.chat_completions.openai.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-vision-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_VISION_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) + + @pytest.mark.covers( + "llm.chat_completions.openai.prompt_cache_5m.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_prompt_cache_hits_on_repeat( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-cache-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + body = ChatBody( + model=model, + messages=[ + ChatMessage(role="system", content=CACHE_PREFIX), + ChatMessage(role="user", content="Reply with the single word pong."), + ], + max_tokens=16, + ) + unwrap(client.proxy.chat(key, body)) + second = unwrap(client.proxy.chat(key, body)) + + details = second.usage.prompt_tokens_details if second.usage else None + assert details and details.cached_tokens and details.cached_tokens > 0, ( + f"a repeated large-prefix prompt must report cached prompt tokens, got usage={second.usage}" + ) + + @pytest.mark.covers( + "llm.chat_completions.openai.tool_use.stream.works", + exercised_on=["chat_completions"], + ) + def test_openai_chat_streams_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + require_env("OPENAI_API_KEY") + model = f"e2e-openai-tool-stream-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.") + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + stream=True, + ), + ) + assert result.ok and result.is_streaming, f"tool stream was not established: {result}" + assert result.stream_error is None, f"tool stream carried an error event: {result.stream_error}" + name, arguments = _streamed_tool_call(result.stream_events) + assert name == "get_weather", f"streamed tool call named {name!r}: {result.stream_events[:5]}" + args = _WeatherArgs.model_validate_json(arguments) + assert args.location.strip(), f"streamed tool call arguments missing location: {arguments!r}" + + +class TestBedrockConverseChatCompletions: + """Bedrock Converse via /chat/completions, the customer's AWS stack. A non-OpenAI + provider must return real content on both the non-streamed and streamed paths. + """ + + def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"{prefix}-{unique_marker()}" + model_id = client.proxy.create_model(model, _bedrock_params()) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-chat") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")], + max_tokens=32, + ), + ) + ) + assert response.choices, f"bedrock converse chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"bedrock converse returned empty content: {response}" + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.basic.stream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_streams_real_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-stream") + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}") + ], + max_tokens=64, + stream=True, + ), + ) + _assert_streamed_completion(result) + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-tool") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.") + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.thinking.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_returns_reasoning( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-thinking") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content="What is 17 times 23? Think it through step by step.")], + thinking=ThinkingParam(type="enabled", budget_tokens=1024), + max_tokens=2048, + ), + ) + ) + assert response.choices, f"bedrock thinking returned no choices: {response}" + message = response.choices[0].message + assert message and message.content and message.content.strip(), ( + f"bedrock thinking returned no answer content: {response}" + ) + assert message.reasoning_content and message.reasoning_content.strip(), ( + "thinking was enabled but no reasoning_content came back on the Bedrock Converse path" + ) + + @pytest.mark.covers( + "llm.chat_completions.bedrock_converse.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_bedrock_converse_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-bedrock-vision") + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index d4080afb7dc..1ba78a7e083 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, ImagesResult from lifecycle import ResourceManager @@ -18,6 +18,15 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e +def _assert_image_returned(body: str) -> None: + parsed = ImagesResult.model_validate_json(body) + assert parsed.data, f"/images/generations returned no data: {body[:300]}" + first = parsed.data[0] + assert first.b64_json or first.url, ( + f"generated image has neither b64_json nor url: {body[:300]}" + ) + + class TestImageGeneration: @pytest.mark.covers("llm.images_generations.openai.basic.nonstream.works") def test_image_generation_returns_image( @@ -35,9 +44,26 @@ class TestImageGeneration: result = endpoints_client.images(key, model, "Draw a cute cat") require_successful_call(result) - parsed = ImagesResult.model_validate_json(result.body) - assert parsed.data, f"/images/generations returned no data: {result.body[:300]}" - first = parsed.data[0] - assert first.b64_json or first.url, ( - f"generated image has neither b64_json nor url: {result.body[:300]}" + _assert_image_returned(result.body) + + @pytest.mark.covers("llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"]) + def test_bedrock_image_generation_returns_image( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-bedrock-image-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="bedrock/amazon.titan-image-generator-v2:0", + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ), ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.images(key, model, "Draw a cute cat") + require_successful_call(result) + _assert_image_returned(result.body) diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index a14bf8e82b3..44376218c6b 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -10,7 +10,7 @@ from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call, unwrap from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager @@ -20,11 +20,14 @@ from models import ( ChatMessage, JsonSchemaProperty, LiteLLMParamsBody, + SpendLogRow, ToolInputSchema, ) pytestmark = pytest.mark.e2e +ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5" + WEATHER_TOOL = AnthropicCustomTool( name="get_weather", description="Get the current weather for a city.", @@ -35,6 +38,11 @@ WEATHER_TOOL = AnthropicCustomTool( ) +def _approx_equal(actual: float, expected: float) -> bool: + """Within 1% or 1e-9 absolute - spend math, not exact float identity.""" + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + class TestAnthropicMessages: def _register( self, endpoints_client: EndpointsClient, resources: ResourceManager @@ -43,7 +51,7 @@ class TestAnthropicMessages: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" + model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" ), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) @@ -61,6 +69,58 @@ class TestAnthropicMessages: assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}" assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}" + @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.cost_logged") + def test_messages_logs_cost_matching_the_response_header( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("ANTHROPIC_API_KEY") + model = f"e2e-messages-cost-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.messages(key, model, f"reply with one word {unique_marker()}") + require_successful_call(result) + parsed = MessagesResult.model_validate_json(result.body) + assert parsed.role == "assistant" and parsed.text.strip(), ( + f"/v1/messages returned no assistant text: {result.body[:300]}" + ) + + # The customer reads per-request cost off the response header (LIT-4076), so + # it must be present and positive on /v1/messages, not only /chat/completions. + header_cost = result.response_cost + assert header_cost is not None and header_cost > 0, ( + "x-litellm-response-cost header missing or non-positive on /v1/messages; " + f"headers={result.headers}" + ) + + # Correlate the spend row by the unique scoped key, not the Anthropic response + # id: on /v1/messages the spend-log request_id is the proxy's own call id, which + # need not equal the message body id, so an id-based poll can miss a correctly + # logged row and time out. The key is fresh per test, so its only priced row is + # this call. + def _priced(rows: list[SpendLogRow]) -> bool: + return any(r.spend is not None and r.spend > 0 for r in rows) + + rows = endpoints_client.proxy.poll_logs_for_key(key, predicate=_priced) + priced = [r for r in rows if r.spend is not None and r.spend > 0] + assert priced, ( + f"no priced /spend/logs row landed for key {key} within the poll window; got {rows}" + ) + row = priced[0] + assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, ( + f"messages spend row missing token counts, so the cost is not real usage: {row}" + ) + assert row.spend is not None and _approx_equal(row.spend, header_cost), ( + f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}; " + "the customer bills against the header, so the two must match" + ) + @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") def test_messages_streams_completion( self, endpoints_client: EndpointsClient, resources: ResourceManager diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index c8806faf3ea..ed5c657d23e 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -15,7 +15,8 @@ import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call -from models import SpendLogRow +from lifecycle import ResourceManager +from models import KeyGenerateBody, SpendLogRow from passthrough_client import ( AnthropicTool, GeminiFunctionDeclaration, @@ -157,3 +158,25 @@ def test_anthropic_passthrough_tool_call_logs_cost( row = _fetch_cost_breakdown(client, result) assert row.custom_llm_provider == "anthropic" + + +class TestPassthroughModelAllowlist: + """A passthrough route must honor the calling key's model allow-list. + + The customer fronts native provider calls through the proxy with custom auth, + so a key scoped to one model must not reach a different model just because the + request goes through the passthrough route rather than /chat/completions. + """ + + @pytest.mark.covers("other.auth.passthrough.model_allowlist_enforced") + def test_passthrough_denies_model_outside_key_allowlist( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + key = client.proxy.generate_key(KeyGenerateBody(models=["gemini-2.5-flash"])) + resources.defer(lambda: client.proxy.delete_key(key)) + + result = client.anthropic_message(key, "claude-haiku-4-5", f"say hi {unique_marker()}") + assert result.status_code == 403, ( + "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, " + f"got {result.status_code}: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index 31801b306a8..0857ff65a52 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -9,7 +9,7 @@ from __future__ import annotations import pytest -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, RerankResult from lifecycle import ResourceManager @@ -23,6 +23,16 @@ DOCUMENTS = [ "Washington, D.C. is the capital of the United States.", "Capital punishment has existed in the United States since before it was a country.", ] +QUERY = "What is the capital of the United States?" + + +def _assert_top_n_scored(body: str) -> None: + parsed = RerankResult.model_validate_json(body) + assert parsed.results, f"/rerank returned no results: {body[:300]}" + assert len(parsed.results) <= 3, f"top_n=3 not honored: {body[:300]}" + assert parsed.results[0].relevance_score is not None, ( + f"top rerank result has no relevance_score: {body[:300]}" + ) class TestRerank: @@ -38,13 +48,28 @@ class TestRerank: resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() - result = endpoints_client.rerank( - key, model, "What is the capital of the United States?", DOCUMENTS, top_n=3 - ) + result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3) require_successful_call(result) - parsed = RerankResult.model_validate_json(result.body) - assert parsed.results, f"/rerank returned no results: {result.body[:300]}" - assert len(parsed.results) <= 3, f"top_n=3 not honored: {result.body[:300]}" - assert parsed.results[0].relevance_score is not None, ( - f"top rerank result has no relevance_score: {result.body[:300]}" + _assert_top_n_scored(result.body) + + @pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"]) + def test_bedrock_rerank_scores_top_n( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-bedrock-rerank-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="bedrock/amazon.rerank-v1:0", + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ), ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3) + require_successful_call(result) + _assert_top_n_scored(result.body) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index bd98f11c045..d24d2b53b71 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -13,7 +13,7 @@ from typing import cast import pytest from pydantic import BaseModel, ValidationError -from e2e_config import unique_marker +from e2e_config import require_env, unique_marker from e2e_http import require_successful_call from endpoints_client import ( EndpointsClient, @@ -29,6 +29,26 @@ from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e +BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + +WEATHER_TOOL = ResponsesFunctionTool( + name="get_weather", + description="Get the weather for a location", + parameters=FunctionParameters( + properties={"location": FunctionParameterProperty(type="string")}, + required=["location"], + ), +) + + +def _bedrock_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=BEDROCK_CONVERSE_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ) + class WeatherArguments(BaseModel): location: str @@ -190,6 +210,84 @@ class TestResponses: parsed = ResponsesResult.model_validate_json(result.body) assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" + @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") + def test_responses_anthropic_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, + model, + "What is the weather in San Francisco? Use the get_weather tool.", + [ + ResponsesFunctionTool( + name="get_weather", + description="Get the weather for a location", + parameters=FunctionParameters( + properties={"location": FunctionParameterProperty(type="string")}, + required=["location"], + ), + ) + ], + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next( + (call for call in parsed.function_calls if call.name == "get_weather"), + None, + ) + assert function_call is not None, f"no get_weather function call: {result.body[:500]}" + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + + @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") + def test_responses_bedrock_returns_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model(model, _bedrock_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses(key, model, "reply with one word") + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}" + + @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") + def test_responses_bedrock_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") + model = f"e2e-responses-{unique_marker()}" + model_id = endpoints_client.create_model(model, _bedrock_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL] + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None) + assert function_call is not None, f"no get_weather function call over bedrock: {result.body[:500]}" + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + def _parse_stream_event( event: str, diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index a9dedac8e61..cdc31aeea79 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -14,6 +14,10 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un from models import ( ChatBody, ChatMessage, + CustomerDeleteBody, + CustomerInfoParams, + CustomerNewBody, + CustomerResponse, KeyBlockBody, KeyDeleteBody, KeyGenerateBody, @@ -270,6 +274,35 @@ class ManagementClient: ) ).user_id + def create_customer(self, user_id: str) -> str: + _ = unwrap( + self.proxy.transport.post( + "/customer/new", + headers=self.proxy.transport.master, + json=CustomerNewBody(user_id=user_id), + response_type=CustomerResponse, + ) + ) + return user_id + + def customer_info(self, end_user_id: str) -> CustomerResponse: + return unwrap( + self.proxy.transport.get( + "/customer/info", + headers=self.proxy.transport.master, + params=CustomerInfoParams(end_user_id=end_user_id), + response_type=CustomerResponse, + ) + ) + + def delete_customer(self, user_id: str) -> None: + _ = self.proxy.transport.post( + "/customer/delete", + headers=self.proxy.transport.master, + json=CustomerDeleteBody(user_ids=[user_id]), + response_type=NoBody, + ) + def update_user(self, body: UserUpdateBody) -> None: _ = unwrap( self.proxy.transport.post( diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index 18bc384a879..9b398963ac9 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -610,3 +610,18 @@ class TestManagementRoutePermissions: f"/team/info returned {team_probe.status_code}: {team_probe.body[:300]}" ) assert client.user_count(user_id) == 0, f"user {user_id} was created despite the 403 route denial" + + +class TestCustomer: + @pytest.mark.covers("mgmt.end_user.new.happy_path") + def test_customer_create_persists_to_info( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + customer = f"e2e-customer-{unique_marker()}" + client.create_customer(customer) + resources.defer(lambda: client.delete_customer(customer)) + + info = client.customer_info(customer) + assert info.user_id == customer, ( + f"/customer/info did not report the created end-user; got {info.user_id!r}" + ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 62b8ffc82d8..ad0bfaf2941 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -115,6 +115,18 @@ class KeyInfoResponse(BaseModel): # ---------- customers ---------- +class CustomerNewBody(BaseModel): + user_id: str + + +class CustomerResponse(BaseModel): + user_id: str | None = None + + +class CustomerInfoParams(BaseModel): + end_user_id: str + + class CustomerDeleteBody(BaseModel): user_ids: list[str] @@ -126,9 +138,26 @@ class ChatMetadata(BaseModel): tags: list[str] | None = None +class ImageUrl(BaseModel): + url: str + + +class TextContentPart(BaseModel): + type: str = "text" + text: str + + +class ImageContentPart(BaseModel): + type: str = "image_url" + image_url: ImageUrl + + +ContentPart = TextContentPart | ImageContentPart + + class ChatMessage(BaseModel): role: str - content: str + content: str | list[ContentPart] class CacheControl(BaseModel): @@ -181,6 +210,7 @@ class ChatBody(BaseModel): tools: list[ChatTool] | None = None tool_choice: str | None = None guardrails: list[str] | None = None + response_format: dict[str, object] | None = None class RouterSettingsOverride(BaseModel): @@ -204,9 +234,19 @@ class ReliabilityChatBody(ChatBody): router_settings_override: RouterSettingsOverride | None = None +class ToolCallFunction(BaseModel): + name: str | None = None + arguments: str | None = None + + +class ToolCall(BaseModel): + function: ToolCallFunction = ToolCallFunction() + + class OutMessage(BaseModel): content: str | None = None reasoning_content: str | None = None + tool_calls: list[ToolCall] | None = None class ChatChoice(BaseModel):