From 76bf0cd5797a47b21c6d96ebd8ce645a197b519c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:43:01 +0000 Subject: [PATCH 1/4] test(e2e): replace custom endpoints_client with provider SDK clients The llm_translation suite drove /v1/responses, /v1/messages, /embeddings, /v1/images/generations, /v1/moderations and /v1/audio/* through a bespoke endpoints_client wrapper that no customer runs. Tests now call the proxy the way customers do: the OpenAI SDK for the OpenAI-compatible surface and the Anthropic SDK for /v1/messages, wired through a session-scoped sdk fixture (sdk_clients.py) that points both SDKs at the proxy with a virtual key. Endpoints no official SDK covers keep the shared typed transport: rerank moves onto ProxyClient (RerankBody/RerankResponse in models.py) and the passthrough header test parses with the shared AnthropicMessagesResponse model. Files that only used endpoints_client for model registration now use the proxy fixture directly. endpoints_client.py is deleted; anthropic joins the e2e-dev dependency group so the lint env resolves the SDK imports. Resolves LIT-4577 --- pyproject.toml | 1 + tests/e2e/CLAUDE.md | 2 + tests/e2e/CONTRIBUTING.md | 2 + tests/e2e/llm_translation/conftest.py | 10 +- tests/e2e/llm_translation/endpoints_client.py | 385 ------------------ tests/e2e/llm_translation/sdk_clients.py | 53 +++ .../llm_translation/test_audio_speech_e2e.py | 99 ++--- .../test_audio_transcriptions_e2e.py | 25 +- .../e2e/llm_translation/test_cache_control.py | 5 +- .../test_credential_messages_e2e.py | 31 +- .../test_custom_pricing_e2e.py | 33 +- .../test_embeddings_endpoint_e2e.py | 92 ++--- .../test_image_generation_e2e.py | 73 ++-- .../test_messages_azure_foundry_e2e.py | 195 ++++----- .../e2e/llm_translation/test_messages_e2e.py | 176 ++++---- ...st_messages_mid_conversation_system_e2e.py | 124 +++--- ...onversation_system_native_providers_e2e.py | 136 ++++--- .../llm_translation/test_moderations_e2e.py | 61 +-- .../e2e/llm_translation/test_ocr_rust_e2e.py | 10 +- .../test_passthrough_headers_e2e.py | 8 +- tests/e2e/llm_translation/test_rerank_e2e.py | 48 +-- .../e2e/llm_translation/test_responses_e2e.py | 355 +++++++--------- .../test_responses_metadata_e2e.py | 69 ++-- tests/e2e/models.py | 19 + tests/e2e/proxy_client.py | 12 + uv.lock | 4 +- 26 files changed, 828 insertions(+), 1200 deletions(-) delete mode 100644 tests/e2e/llm_translation/endpoints_client.py create mode 100644 tests/e2e/llm_translation/sdk_clients.py diff --git a/pyproject.toml b/pyproject.toml index 62bd37c3db6..f152a6a73cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -198,6 +198,7 @@ e2e-dev = [ "playwright==1.61.0", "websockets>=15.0.1,<16.0", "locust==2.45.0", + "anthropic==0.84.0", ] proxy-dev = [ "prisma==0.11.0", diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 0e39664e358..fa73cbacd96 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -58,6 +58,8 @@ That snippet only conveys intent. What you actually write uses the real harness: Every HTTP call goes through the shared transport, never through `requests.*` in a test. `e2e_http.py` is the only module permitted to call `requests.*`, and that is enforced in CI by `tests/code_coverage_tests/check_e2e_no_raw_requests.py`. A test that imports requests will fail the check +One deliberate exception: LLM-endpoint calls in `llm_translation/` go through the real provider SDKs (OpenAI, Anthropic) via the suite's `sdk` fixture (`llm_translation/sdk_clients.py`), because that is what customers actually run against the proxy (LIT-4577). The SDKs raise their own typed exceptions on failure, which is exactly the customer-observable contract; management routes (model/key CRUD, spend read-back) and endpoints no official SDK covers (e.g. `/v1/rerank`, `/v1/ocr`, custom passthrough paths) stay on the shared transport. Raw HTTP client imports remain banned either way + The shape is layered so tests stay declarative `transport.py` exposes a `Transport` Protocol with `post`, `get`, `delete`, `send`, `stream`, `probe`, plus `bearer(key)` and the `master` header. `HttpTransport` fulfils it, and `SplitTransport` routes each call by path to the data plane or the control plane so a split control-plane/data-plane deployment works without any change in the test diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index fc43769aca8..fa0174f686c 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -122,6 +122,8 @@ That snippet only conveys intent. What you actually write uses the real harness: Every HTTP call goes through the shared transport, never through `requests.*` in a test. `e2e_http.py` is the only module permitted to call `requests.*`, and that is enforced in CI by `tests/code_coverage_tests/check_e2e_no_raw_requests.py`. A test that imports requests will fail the check +One deliberate exception: LLM-endpoint calls in `llm_translation/` go through the real provider SDKs (OpenAI, Anthropic) via the suite's `sdk` fixture (`llm_translation/sdk_clients.py`), because that is what customers actually run against the proxy (LIT-4577). Management routes and endpoints no official SDK covers stay on the shared transport, and raw HTTP client imports remain banned either way + The shape is layered so tests stay declarative `transport.py` exposes a `Transport` Protocol with `post`, `get`, `delete`, `send`, `stream`, `probe`, plus `bearer(key)` and the `master` header. `HttpTransport` fulfils it, and `SplitTransport` routes each call by path to the data plane or the control plane so a split control-plane/data-plane deployment works without any change in the test diff --git a/tests/e2e/llm_translation/conftest.py b/tests/e2e/llm_translation/conftest.py index f35ecf0760d..9fd45799773 100644 --- a/tests/e2e/llm_translation/conftest.py +++ b/tests/e2e/llm_translation/conftest.py @@ -2,14 +2,16 @@ The shared lifecycle (resources/scoped_key), proxy liveness gate, and e2e marker live in the parent tests/e2e/conftest.py. PassthroughClient holds the shared -ProxyClient, so the `resources` fixture cleans up keys this suite creates. +ProxyClient, so the `resources` fixture cleans up keys this suite creates. The +`sdk` fixture hands tests real provider SDK clients (OpenAI, Anthropic) pointed +at the proxy, the way customers actually call it. """ import pytest -from endpoints_client import EndpointsClient, build_endpoints_client from passthrough_client import PassthroughClient, build_client from proxy_client import ProxyClient +from sdk_clients import SdkClients, build_sdk_clients def pytest_configure(config: pytest.Config) -> None: @@ -25,5 +27,5 @@ def client(proxy: ProxyClient) -> PassthroughClient: @pytest.fixture(scope="session") -def endpoints_client(proxy: ProxyClient) -> EndpointsClient: - return build_endpoints_client(proxy) +def sdk() -> SdkClients: + return build_sdk_clients() diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py deleted file mode 100644 index ace621d03b3..00000000000 --- a/tests/e2e/llm_translation/endpoints_client.py +++ /dev/null @@ -1,385 +0,0 @@ -"""Client for the non-chat inference endpoints (responses, messages, rerank, -embeddings, audio speech, image generation). - -Each test registers the deployment it needs through /model/new (deleted on -teardown), so nothing is hardcoded into the gateway config, then drives the -endpoint with `send` and parses the provider-native body with a suite-local model -so the assertion is on real content, not just a 200. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -from pydantic import BaseModel - -from proxy_client import ProxyClient -from e2e_http import BinaryStream, Result, StreamingResponse -from models import CacheControl, ChatMessage, LiteLLMParamsBody, RichMessage, TextBlock - -__all__ = [ - "CacheControl", - "RichMessage", - "TextBlock", -] - - -class FunctionParameterProperty(BaseModel): - type: str - description: str | None = None - - -class FunctionParameters(BaseModel): - type: Literal["object"] = "object" - properties: dict[str, FunctionParameterProperty] - required: list[str] = [] - - -class ResponsesFunctionTool(BaseModel): - type: Literal["function"] = "function" - name: str - description: str | None = None - parameters: FunctionParameters - - -class ResponsesInputTextPart(BaseModel): - type: Literal["input_text"] = "input_text" - text: str - - -class ResponsesInputImagePart(BaseModel): - type: Literal["input_image"] = "input_image" - image_url: str - - -ResponsesInputContentPart = ResponsesInputTextPart | ResponsesInputImagePart - - -class ResponsesInputMessage(BaseModel): - role: Literal["user", "assistant", "system"] = "user" - content: list[ResponsesInputContentPart] - - -ResponsesInput = str | list[ResponsesInputMessage] - - -class ResponsesRequest(BaseModel): - model: str - input: ResponsesInput - instructions: str | None = None - stream: bool = False - tools: list[ResponsesFunctionTool] | None = None - - -class MessagesRequest(BaseModel): - model: str - max_tokens: int - messages: list[ChatMessage] - - -class RichMessagesRequest(BaseModel): - model: str - max_tokens: int = 64 - system: list[TextBlock] - messages: list[RichMessage] - - -class EmbeddingsRequest(BaseModel): - model: str - input: str - - -class RerankRequest(BaseModel): - model: str - query: str - documents: list[str] - top_n: int - - -class SpeechRequest(BaseModel): - model: str - input: str - voice: str - - -class ImageRequest(BaseModel): - model: str - prompt: str - n: int = 1 - size: str = "1024x1024" - - -class TranscriptionForm(BaseModel): - model: str - response_format: str = "json" - - -class ModerationRequest(BaseModel): - model: str - input: str - - -class ResponsesOutputContent(BaseModel): - type: str | None = None - text: str | None = None - - -class ResponsesOutputItem(BaseModel): - type: str | None = None - content: list[ResponsesOutputContent] = [] - name: str | None = None - arguments: str | None = None - call_id: str | None = None - - -class ResponsesResult(BaseModel): - id: str | None = None - status: str | None = None - model: str | None = None - output: list[ResponsesOutputItem] = [] - - @property - def text(self) -> str: - return "".join( - content.text or "" for item in self.output for content in item.content - ) - - @property - def function_calls(self) -> tuple[ResponsesOutputItem, ...]: - return tuple( - item - for item in self.output - if item.type == "function_call" - and item.name is not None - and item.arguments is not None - ) - - -class ResponsesStreamEvent(BaseModel): - event_id: str | None = None - - -class ResponsesStreamEventType(BaseModel): - type: str - - -class ResponsesOutputTextDeltaEvent(ResponsesStreamEvent): - type: Literal["response.output_text.delta"] - delta: str - - -class AnthropicContentBlock(BaseModel): - type: str | None = None - text: str | None = None - - -class MessagesUsage(BaseModel): - input_tokens: int = 0 - output_tokens: int = 0 - cache_creation_input_tokens: int = 0 - cache_read_input_tokens: int = 0 - - -class MessagesResult(BaseModel): - id: str | None = None - role: str | None = None - model: str | None = None - content: list[AnthropicContentBlock] = [] - usage: MessagesUsage = MessagesUsage() - - @property - def text(self) -> str: - return "".join(block.text or "" for block in self.content) - - -class EmbeddingItem(BaseModel): - embedding: list[float] = [] - - -class EmbeddingsResult(BaseModel): - data: list[EmbeddingItem] = [] - - @property - def first_vector(self) -> tuple[float, ...]: - return tuple(self.data[0].embedding) if self.data else () - - -class RerankItem(BaseModel): - index: int | None = None - relevance_score: float | None = None - - -class RerankResult(BaseModel): - results: list[RerankItem] = [] - - -class ImageItem(BaseModel): - url: str | None = None - b64_json: str | None = None - - -class ImagesResult(BaseModel): - data: list[ImageItem] = [] - - -class TranscriptionResult(BaseModel): - text: str = "" - - -class ModerationResultItem(BaseModel): - flagged: bool - categories: dict[str, bool] = {} - - @property - def flagged_categories(self) -> tuple[str, ...]: - return tuple(name for name, hit in self.categories.items() if hit) - - -class ModerationResult(BaseModel): - results: list[ModerationResultItem] = [] - - @property - def first(self) -> ModerationResultItem | None: - return self.results[0] if self.results else None - - -@dataclass(frozen=True, slots=True) -class EndpointsClient: - proxy: ProxyClient - - def create_model(self, model_name: str, litellm_params: LiteLLMParamsBody) -> str: - return self.proxy.create_model(model_name, litellm_params) - - def delete_model(self, model_id: str) -> None: - self.proxy.delete_model(model_id) - - def _send( - self, path: str, key: str, body: BaseModel, *, stream: bool = False - ) -> StreamingResponse: - return self.proxy.transport.send( - path, - headers=self.proxy.transport.bearer(key), - json=body, - stream=stream, - ) - - def responses( - self, key: str, model: str, text: str, *, stream: bool = False - ) -> StreamingResponse: - return self._send( - "/v1/responses", - key, - ResponsesRequest( - model=model, - input=text, - instructions="You are a helpful assistant", - stream=stream, - ), - stream=stream, - ) - - def responses_vision( - self, key: str, model: str, text: str, image_url: str - ) -> StreamingResponse: - return self._send( - "/v1/responses", - key, - ResponsesRequest( - model=model, - input=[ - ResponsesInputMessage( - content=[ - ResponsesInputTextPart(text=text), - ResponsesInputImagePart(image_url=image_url), - ] - ) - ], - instructions="You are a helpful assistant", - ), - ) - - def responses_with_tools( - self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool] - ) -> StreamingResponse: - return self._send( - "/v1/responses", - key, - ResponsesRequest( - model=model, - input=text, - instructions="You are a helpful assistant", - tools=tools, - ), - ) - - def messages( - self, key: str, model: str, text: str, *, max_tokens: int = 64 - ) -> StreamingResponse: - return self._send( - "/v1/messages", - key, - MessagesRequest( - model=model, - max_tokens=max_tokens, - messages=[ChatMessage(role="user", content=text)], - ), - ) - - def embeddings(self, key: str, model: str, text: str) -> StreamingResponse: - return self._send("/embeddings", key, EmbeddingsRequest(model=model, input=text)) - - def rerank( - self, key: str, model: str, query: str, documents: list[str], top_n: int - ) -> StreamingResponse: - return self._send( - "/v1/rerank", - key, - RerankRequest(model=model, query=query, documents=documents, top_n=top_n), - ) - - def audio_speech( - self, key: str, model: str, text: str, *, voice: str = "alloy" - ) -> StreamingResponse: - return self._send( - "/v1/audio/speech", key, SpeechRequest(model=model, input=text, voice=voice) - ) - - def audio_speech_stream( - self, key: str, model: str, text: str, *, voice: str = "alloy" - ) -> BinaryStream: - return self.proxy.transport.stream_binary( - "/v1/audio/speech", - headers=self.proxy.transport.bearer(key), - json=SpeechRequest(model=model, input=text, voice=voice), - ) - - def transcribe( - self, key: str, model: str, *, filename: str, content: bytes - ) -> Result[TranscriptionResult]: - return self.proxy.transport.upload( - "/v1/audio/transcriptions", - headers=self.proxy.transport.bearer(key), - form=TranscriptionForm(model=model), - filename=filename, - content=content, - file_content_type="audio/wav", - response_type=TranscriptionResult, - ) - - def moderations(self, key: str, model: str, text: str) -> Result[ModerationResult]: - return self.proxy.transport.post( - "/v1/moderations", - headers=self.proxy.transport.bearer(key), - json=ModerationRequest(model=model, input=text), - response_type=ModerationResult, - ) - - def images(self, key: str, model: str, prompt: str) -> StreamingResponse: - return self._send( - "/v1/images/generations", key, ImageRequest(model=model, prompt=prompt) - ) - - -def build_endpoints_client(proxy: ProxyClient) -> EndpointsClient: - return EndpointsClient(proxy=proxy) diff --git a/tests/e2e/llm_translation/sdk_clients.py b/tests/e2e/llm_translation/sdk_clients.py new file mode 100644 index 00000000000..8a6b187f979 --- /dev/null +++ b/tests/e2e/llm_translation/sdk_clients.py @@ -0,0 +1,53 @@ +"""Real provider SDK clients pointed at the proxy, connected the way customers +connect (LIT-4577). + +The OpenAI SDK drives the OpenAI-compatible surface (/responses, /embeddings, +/images/generations, /moderations, /audio/*) and the Anthropic SDK drives +/v1/messages, each authenticated with a litellm virtual key. Errors surface as +the SDK's own exceptions, exactly what an end user sees. Retries are disabled +so a proxy fault fails the test instead of being papered over, and the timeout +matches the shared transport's request budget. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +from anthropic import Anthropic +from openai import OpenAI + +from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT + + +def response_header(headers: Mapping[str, str], name: str) -> str | None: + """Typed read of an SDK response header: httpx.Headers.get returns Any and + httpx itself is a banned import in suite code, so tests read headers through + the Mapping[str, str] interface Headers fulfils.""" + return headers[name] if name in headers else None + + +@dataclass(frozen=True, slots=True) +class SdkClients: + base_url: str + request_timeout: float + + def openai(self, key: str) -> OpenAI: + return OpenAI( + base_url=self.base_url, + api_key=key, + timeout=self.request_timeout, + max_retries=0, + ) + + def anthropic(self, key: str) -> Anthropic: + return Anthropic( + base_url=self.base_url, + api_key=key, + timeout=self.request_timeout, + max_retries=0, + ) + + +def build_sdk_clients() -> SdkClients: + return SdkClients(base_url=PROXY_BASE_URL, request_timeout=REQUEST_TIMEOUT) diff --git a/tests/e2e/llm_translation/test_audio_speech_e2e.py b/tests/e2e/llm_translation/test_audio_speech_e2e.py index b95cef8db4d..859292ecc7b 100644 --- a/tests/e2e/llm_translation/test_audio_speech_e2e.py +++ b/tests/e2e/llm_translation/test_audio_speech_e2e.py @@ -1,9 +1,10 @@ """Live e2e: POST /v1/audio/speech returns audio, non-streamed and streamed. -The non-streamed call asserts an audio (not JSON) body. The streamed call consumes -the response the way a player would and asserts customer-observable streaming: -chunked transfer encoding (a buffered body would carry a content-length) with -non-zero audio bytes. +Both calls go through the real OpenAI SDK (LIT-4577). The non-streamed call +asserts an audio (not JSON) body. The streamed call consumes the response the +way a player would and asserts customer-observable streaming: chunked transfer +encoding (a buffered body would carry a content-length) with non-zero audio +bytes. """ from __future__ import annotations @@ -11,68 +12,70 @@ from __future__ import annotations import pytest from e2e_config import unique_marker -from e2e_http import require_successful_call -from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients, response_header pytestmark = pytest.mark.e2e +def _register(proxy: ProxyClient, resources: ResourceManager, prefix: str) -> str: + model = f"{prefix}-{unique_marker()}" + model_id = proxy.create_model( + model, + LiteLLMParamsBody(model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY"), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return model + + class TestAudioSpeech: @pytest.mark.covers("llm.audio_speech.openai.basic.nonstream.works") def test_audio_speech_returns_audio( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-speech-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY" - ), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, "e2e-speech") + client = sdk.openai(resources.key()) - result = endpoints_client.audio_speech(key, model, "Hello!") - require_successful_call(result) - assert "audio" in (result.content_type or ""), ( - f"/audio/speech content-type is not audio: {result.content_type!r}" + response = client.audio.speech.with_raw_response.create( + model=model, voice="alloy", input="Hello!" ) - assert result.body, "/audio/speech returned an empty body" + content_type = response_header(response.headers, "content-type") + assert "audio" in (content_type or ""), ( + f"/audio/speech content-type is not audio: {content_type!r}" + ) + assert response.content, "/audio/speech returned an empty body" @pytest.mark.covers("llm.audio_speech.openai.basic.stream.works") def test_audio_speech_streams_audio_chunks( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-speech-stream-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY" - ), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, "e2e-speech-stream") + client = sdk.openai(resources.key()) - result = endpoints_client.audio_speech_stream( - key, - model, - "Streaming speech should arrive in several audio chunks so a client can " - "begin playback well before the whole clip has finished generating.", + with client.audio.speech.with_streaming_response.create( + model=model, + voice="alloy", + input=( + "Streaming speech should arrive in several audio chunks so a client can " + "begin playback well before the whole clip has finished generating." + ), + ) as response: + content_type = response_header(response.headers, "content-type") + transfer_encoding = response_header(response.headers, "transfer-encoding") + content_length = response_header(response.headers, "content-length") + total_bytes = sum(len(chunk) for chunk in response.iter_bytes(chunk_size=8192)) + + assert "audio" in (content_type or ""), ( + f"/audio/speech content-type is not audio: {content_type!r}" ) - assert result.ok, ( - f"/audio/speech stream failed (status {result.status_code}); body={result.error_body}" + assert "chunked" in (transfer_encoding or ""), ( + f"/audio/speech did not stream: transfer-encoding={transfer_encoding!r}, " + f"content-length={content_length!r} (a buffered body is not a stream)" ) - assert "audio" in (result.content_type or ""), ( - f"/audio/speech content-type is not audio: {result.content_type!r}" - ) - assert result.chunked, ( - f"/audio/speech did not stream: transfer-encoding={result.transfer_encoding!r}, " - f"content-length={result.content_length!r} (a buffered body is not a stream)" - ) - assert result.content_length is None, ( - f"/audio/speech advertised content-length={result.content_length!r} on a " + assert content_length is None, ( + f"/audio/speech advertised content-length={content_length!r} on a " f"streamed response (a buffered body is not a stream)" ) - assert result.total_bytes > 0, "/audio/speech stream returned no audio bytes" + assert total_bytes > 0, "/audio/speech stream returned no audio bytes" diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py index af6123dc46a..617192f88c7 100644 --- a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -1,8 +1,9 @@ """Live e2e: POST /v1/audio/transcriptions turns speech into text. Registers an OpenAI speech-to-text deployment at runtime and uploads a spoken -weather question (the realtime suite's 24kHz WAV fixture) as multipart, asserting -the returned transcript is non-empty and mentions the word it was asked about. +weather question (the realtime suite's 24kHz WAV fixture) through the real +OpenAI SDK (LIT-4577), asserting the returned transcript is non-empty and +mentions the word it was asked about. """ from __future__ import annotations @@ -12,10 +13,10 @@ from pathlib import Path import pytest from e2e_config import unique_marker -from e2e_http import unwrap -from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e @@ -27,24 +28,22 @@ WEATHER_WAV = ( class TestAudioTranscriptions: @pytest.mark.covers("llm.audio_transcriptions.openai.basic.nonstream.works") def test_audio_transcriptions_returns_text( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: model = f"e2e-transcribe-{unique_marker()}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody( model="openai/gpt-4o-mini-transcribe", api_key="os.environ/OPENAI_API_KEY" ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + resources.defer(lambda: proxy.delete_model(model_id)) + client = sdk.openai(resources.key()) - result = unwrap( - endpoints_client.transcribe( - key, model, filename=WEATHER_WAV.name, content=WEATHER_WAV.read_bytes() - ) + transcription = client.audio.transcriptions.create( + model=model, file=(WEATHER_WAV.name, WEATHER_WAV.read_bytes(), "audio/wav") ) - text = result.text.strip() + text = transcription.text.strip() assert text, "/audio/transcriptions returned an empty transcript" assert "weather" in text.lower(), ( f"transcript of a spoken weather question does not mention weather: {text!r}" diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index a2c17b0fb66..888cbb422eb 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -15,7 +15,7 @@ service_tier lives in test_provider_features_e2e.py. The provider-native cache_control request shape is not expressible with the shared ``ChatBody`` (whose content is a plain string), so the cacheable body is -built from the typed content blocks shared in ``endpoints_client.py``. +built from the typed content blocks shared in ``models.py``. """ from __future__ import annotations @@ -27,9 +27,8 @@ from pydantic import BaseModel from e2e_config import unique_marker from e2e_http import Result, unwrap -from endpoints_client import CacheControl, RichMessage, TextBlock from lifecycle import ResourceManager -from models import ChatResponse, LiteLLMParamsBody, Usage +from models import CacheControl, ChatResponse, LiteLLMParamsBody, RichMessage, TextBlock, Usage from passthrough_client import PassthroughClient import os diff --git a/tests/e2e/llm_translation/test_credential_messages_e2e.py b/tests/e2e/llm_translation/test_credential_messages_e2e.py index 49ea748430e..d4bf4566cc0 100644 --- a/tests/e2e/llm_translation/test_credential_messages_e2e.py +++ b/tests/e2e/llm_translation/test_credential_messages_e2e.py @@ -7,43 +7,48 @@ import os import pytest from e2e_config import unique_marker -from e2e_http import require_successful_call -from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager from models import CredentialCreateBody, LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e class TestCredentialBackedMessages: @pytest.mark.covers("mgmt.credential.new.serves_request") - def test_credential_backed_messages(self, endpoints_client: EndpointsClient, resources: ResourceManager) -> None: + def test_credential_backed_messages( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: marker = unique_marker() credential_name = f"e2e-cred-{marker}" model = f"e2e-cred-messages-{marker}" anthropic_api_key = os.getenv("ANTHROPIC_API_KEY") assert anthropic_api_key, "ANTHROPIC_API_KEY must be set for this live e2e test" - endpoints_client.proxy.create_credential( + proxy.create_credential( CredentialCreateBody( credential_name=credential_name, credential_values={"api_key": anthropic_api_key}, ) ) - resources.defer(lambda: endpoints_client.proxy.delete_credential(credential_name)) + resources.defer(lambda: proxy.delete_credential(credential_name)) - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody( model="anthropic/claude-haiku-4-5", litellm_credential_name=credential_name, ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + resources.defer(lambda: proxy.delete_model(model_id)) - key = resources.key() - result = endpoints_client.messages(key, model, "reply with one word") - require_successful_call(result) - parsed = MessagesResult.model_validate_json(result.body) - assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}" - assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}" + client = sdk.anthropic(resources.key()) + message = client.messages.create( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": "reply with one word"}], + ) + assert message.role == "assistant", f"unexpected role: {message.role!r}" + text = "".join(block.text for block in message.content if block.type == "text") + assert text.strip(), f"/v1/messages returned no text: {message.content!r}" diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index b4ff631a56b..1cebf90fa21 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -25,7 +25,6 @@ from pydantic import BaseModel, RootModel from e2e_config import unique_marker from proxy_client import ProxyClient from e2e_http import Success, unwrap -from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import ( ChatBody, @@ -71,7 +70,7 @@ def _approx_equal(actual: float, expected: float) -> bool: def _provision( - endpoints_client: EndpointsClient, + proxy: ProxyClient, resources: ResourceManager, prefix: str, *, @@ -84,7 +83,7 @@ def _provision( marker keeps the name unique so concurrent runs on the shared proxy never collide.""" model_name = f"{prefix}-{unique_marker()}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model_name, LiteLLMParamsBody( model=BACKEND_MODEL, @@ -93,15 +92,15 @@ def _provision( output_cost_per_token=output_cost_per_token, ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + resources.defer(lambda: proxy.delete_model(model_id)) return model_name def _provision_custom_priced( - endpoints_client: EndpointsClient, resources: ResourceManager + proxy: ProxyClient, resources: ResourceManager ) -> str: return _provision( - endpoints_client, + proxy, resources, "custom-priced-flash", input_cost_per_token=CUSTOM_INPUT_RATE, @@ -151,14 +150,14 @@ def _poll_breakdown_row(proxy: ProxyClient, key: str, response_id: str | None) - class TestCustomPricing: def test_custom_pricing_is_billed_at_configured_rate( self, - endpoints_client: EndpointsClient, + proxy: ProxyClient, resources: ResourceManager, scoped_key: str, ) -> None: - model = _provision_custom_priced(endpoints_client, resources) + model = _provision_custom_priced(proxy, resources) chat = unwrap( - endpoints_client.proxy.chat( + proxy.chat( scoped_key, ChatBody( model=model, @@ -172,7 +171,7 @@ class TestCustomPricing: ) ) - row = _poll_breakdown_row(endpoints_client.proxy, scoped_key, chat.id) + row = _poll_breakdown_row(proxy, scoped_key, chat.id) assert row.metadata and row.metadata.cost_breakdown # guaranteed by the poll breakdown = row.metadata.cost_breakdown @@ -195,10 +194,10 @@ class TestCustomPricing: ) def test_model_info_reports_custom_pricing( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager ) -> None: - model = _provision_custom_priced(endpoints_client, resources) - entry = _model_info_entry(endpoints_client.proxy.model_info(), model) + model = _provision_custom_priced(proxy, resources) + entry = _model_info_entry(proxy.model_info(), model) assert entry.litellm_params.input_cost_per_token == CUSTOM_INPUT_RATE, ( f"/model/info litellm_params input rate " @@ -210,20 +209,20 @@ class TestCustomPricing: ) def test_custom_pricing_is_isolated_from_sibling_deployment( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager ) -> None: # Register the override first so its rate is in the backend cost map before # the sibling resolves; a leak (LIT-3897) would then poison the sibling. - custom = _provision_custom_priced(endpoints_client, resources) + custom = _provision_custom_priced(proxy, resources) sibling = _provision( - endpoints_client, + proxy, resources, "base-flash", input_cost_per_token=None, output_cost_per_token=None, ) - entries = {entry.model_name: entry for entry in endpoints_client.proxy.model_info()} + entries = {entry.model_name: entry for entry in proxy.model_info()} custom_entry = entries.get(custom) sibling_entry = entries.get(sibling) assert custom_entry is not None, f"{custom} absent from /model/info" diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 157caedd561..dac69591341 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -1,9 +1,10 @@ """Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex. -Each test registers the deployment it needs at runtime (deleted on teardown) and -asserts a non-empty, non-zero vector came back. The LIT-3167 guard in -tests/e2e/embeddings/ covers the Gemini embedding path; embeddings cost tracking is -covered by tests/e2e/quota_management/spend_tracking/. +Each test registers the deployment it needs at runtime (deleted on teardown), +drives the endpoint with the real OpenAI SDK (LIT-4577), and asserts a +non-empty, non-zero vector came back. The LIT-3167 guard in +tests/e2e/embeddings/ covers the Gemini embedding path; embeddings cost tracking +is covered by tests/e2e/quota_management/spend_tracking/. """ from __future__ import annotations @@ -11,79 +12,74 @@ from __future__ import annotations import pytest from e2e_config import unique_marker -from e2e_http import require_successful_call -from endpoints_client import EmbeddingsResult, EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e +def _assert_embedding_vector( + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + prefix: str, + params: LiteLLMParamsBody, +) -> None: + model = f"{prefix}-{unique_marker()}" + model_id = proxy.create_model(model, params) + resources.defer(lambda: proxy.delete_model(model_id)) + client = sdk.openai(resources.key()) + + embeddings = client.embeddings.create(model=model, input="Say this is a test!") + assert embeddings.data, f"/embeddings returned no data: {embeddings!r}" + vector = embeddings.data[0].embedding + assert vector, f"/embeddings returned no vector: {embeddings!r}" + assert any(component != 0.0 for component in vector), "embedding vector is all zeros" + + class TestEmbeddingsEndpoint: @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") def test_embeddings_returns_vector( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-embeddings-{unique_marker()}" - model_id = endpoints_client.create_model( - model, + _assert_embedding_vector( + proxy, + resources, + sdk, + "e2e-embeddings", LiteLLMParamsBody( model="openai/text-embedding-3-small", api_key="os.environ/OPENAI_API_KEY" ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() - - result = endpoints_client.embeddings(key, model, "Say this is a test!") - require_successful_call(result) - parsed = EmbeddingsResult.model_validate_json(result.body) - assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" - assert any(component != 0.0 for component in parsed.first_vector), ( - f"embedding vector is all zeros: {result.body[:300]}" - ) @pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works") def test_bedrock_embeddings_returns_vector( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-embeddings-bedrock-{unique_marker()}" - model_id = endpoints_client.create_model( - model, + _assert_embedding_vector( + proxy, + resources, + sdk, + "e2e-embeddings-bedrock", LiteLLMParamsBody( model="bedrock/amazon.titan-embed-text-v2:0", aws_region_name="us-west-2" ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() - - result = endpoints_client.embeddings(key, model, "Say this is a test!") - require_successful_call(result) - parsed = EmbeddingsResult.model_validate_json(result.body) - assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" - assert any(component != 0.0 for component in parsed.first_vector), ( - f"embedding vector is all zeros: {result.body[:300]}" - ) @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") def test_vertex_embeddings_returns_vector( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-embeddings-vertex-{unique_marker()}" - model_id = endpoints_client.create_model( - model, + _assert_embedding_vector( + proxy, + resources, + sdk, + "e2e-embeddings-vertex", LiteLLMParamsBody( model="vertex_ai/gemini-embedding-2", vertex_project="os.environ/VERTEXAI_PROJECT", vertex_location="us-central1", ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() - - result = endpoints_client.embeddings(key, model, "Say this is a test!") - require_successful_call(result) - parsed = EmbeddingsResult.model_validate_json(result.body) - assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}" - assert any(component != 0.0 for component in parsed.first_vector), ( - f"embedding vector is all zeros: {result.body[:300]}" - ) diff --git a/tests/e2e/llm_translation/test_image_generation_e2e.py b/tests/e2e/llm_translation/test_image_generation_e2e.py index 1ba78a7e083..c5204162501 100644 --- a/tests/e2e/llm_translation/test_image_generation_e2e.py +++ b/tests/e2e/llm_translation/test_image_generation_e2e.py @@ -1,8 +1,7 @@ """Live e2e: POST /v1/images/generations returns an image. -Registers an OpenAI image deployment at runtime and asserts the response carries a -generated image (url or base64). Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py. +Registers an image deployment at runtime, drives it through the real OpenAI SDK +(LIT-4577), and asserts the response carries a generated image (url or base64). """ from __future__ import annotations @@ -10,50 +9,58 @@ from __future__ import annotations import pytest from e2e_config import require_env, unique_marker -from e2e_http import require_successful_call -from endpoints_client import EndpointsClient, ImagesResult from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e -def _assert_image_returned(body: str) -> None: - parsed = ImagesResult.model_validate_json(body) - assert parsed.data, f"/images/generations returned no data: {body[:300]}" - first = parsed.data[0] - assert first.b64_json or first.url, ( - f"generated image has neither b64_json nor url: {body[:300]}" - ) +def _assert_image_returned( + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + prefix: str, + params: LiteLLMParamsBody, +) -> None: + model = f"{prefix}-{unique_marker()}" + model_id = proxy.create_model(model, params) + resources.defer(lambda: proxy.delete_model(model_id)) + client = sdk.openai(resources.key()) + + images = client.images.generate(model=model, prompt="Draw a cute cat", n=1, size="1024x1024") + data = images.data or [] + assert data, f"/images/generations returned no data: {images!r}" + first = data[0] + assert first.b64_json or first.url, "generated image has neither b64_json nor url" class TestImageGeneration: @pytest.mark.covers("llm.images_generations.openai.basic.nonstream.works") def test_image_generation_returns_image( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-image-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model="openai/gpt-image-1-mini", api_key="os.environ/OPENAI_API_KEY" - ), + _assert_image_returned( + proxy, + resources, + sdk, + "e2e-image", + LiteLLMParamsBody(model="openai/gpt-image-1-mini", api_key="os.environ/OPENAI_API_KEY"), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() - result = endpoints_client.images(key, model, "Draw a cute cat") - require_successful_call(result) - _assert_image_returned(result.body) - - @pytest.mark.covers("llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"]) + @pytest.mark.covers( + "llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"] + ) def test_bedrock_image_generation_returns_image( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") - model = f"e2e-bedrock-image-{unique_marker()}" - model_id = endpoints_client.create_model( - model, + _assert_image_returned( + proxy, + resources, + sdk, + "e2e-bedrock-image", LiteLLMParamsBody( model="bedrock/amazon.titan-image-generator-v2:0", aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", @@ -61,9 +68,3 @@ class TestImageGeneration: aws_region_name="os.environ/AWS_REGION", ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() - - result = endpoints_client.images(key, model, "Draw a cute cat") - require_successful_call(result) - _assert_image_returned(result.body) diff --git a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py index d8d44820e80..90568d8c951 100644 --- a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py +++ b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py @@ -1,9 +1,9 @@ """Live e2e: POST /v1/messages routed to Azure AI Foundry Anthropic deployments. Registers `azure_ai/` deployments at runtime and drives the Messages -endpoint through the gateway across the behaviors an Anthropic client relies on: -a basic completion, a streamed completion, and tool use (non-streaming and -streaming). Auth is the Azure API key (`x-api-key`); the deployment reads +endpoint through the gateway with the real Anthropic SDK (LIT-4577) across the +behaviors an Anthropic client relies on: a basic completion, a streamed +completion, and tool use (non-streaming and streaming). The deployment reads `AZURE_AI_API_BASE` / `AZURE_AI_API_KEY` from the proxy env, so no secret is sent in the request. """ @@ -11,60 +11,50 @@ sent in the request. from __future__ import annotations import pytest +from anthropic.types import RawMessageStreamEvent, ToolParam from e2e_config import EXPECT_RUST, unique_marker -from e2e_http import StreamingResponse, require_successful_call, unwrap -from endpoints_client import EndpointsClient from lifecycle import ResourceManager -from models import ( - AnthropicCustomTool, - AnthropicMessagesBody, - ChatMessage, - JsonSchemaProperty, - LiteLLMParamsBody, - ToolInputSchema, -) +from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e AZURE_FOUNDRY_MODEL = "azure_ai/claude-haiku-4-5" -WEATHER_TOOL = AnthropicCustomTool( - name="get_weather", - description="Get the current weather for a city.", - input_schema=ToolInputSchema( - properties={"city": JsonSchemaProperty(type="string")}, - required=["city"], - ), -) +WEATHER_TOOL: ToolParam = { + "name": "get_weather", + "description": "Get the current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, +} -def _assert_streamed_ok(result: StreamingResponse) -> None: - require_successful_call(result) - assert result.is_streaming, f"response was not streamed: {result.headers}" - assert not result.stream_error, f"stream errored: {result.stream_error}" - assert result.stream_events, "stream produced no SSE events" - assert any("content_block_delta" in event for event in result.stream_events), ( - "stream carried no content deltas" +def _assert_rust_served(headers: dict[str, str]) -> None: + if not EXPECT_RUST: + return + assert headers.get("x-litellm-rust") == "true", ( + "E2E_EXPECT_RUST is set, so this gateway must serve /v1/messages through the " + "Rust path, but the response carried no x-litellm-rust marker. The request " + "still succeeded, which is exactly the failure mode: a gateway whose native " + f"extension is unavailable falls back to Python silently. headers={headers}" ) - assert any("message_stop" in event for event in result.stream_events), ( - "stream never reached message_stop" - ) - if EXPECT_RUST: - assert result.headers.get("x-litellm-rust") == "true", ( - "E2E_EXPECT_RUST is set, so this gateway must serve /v1/messages through the " - "Rust path, but the response carried no x-litellm-rust marker. The request " - "still succeeded, which is exactly the failure mode: a gateway whose native " - f"extension is unavailable falls back to Python silently. headers={result.headers}" - ) + + +def _assert_streamed_ok(event_types: list[str]) -> None: + assert event_types, "stream produced no SSE events" + assert "content_block_delta" in event_types, "stream carried no content deltas" + assert "message_stop" in event_types, "stream never reached message_stop" class TestAzureFoundryMessages: - def _register( - self, endpoints_client: EndpointsClient, resources: ResourceManager - ) -> tuple[str, str]: + def _register(self, proxy: ProxyClient, resources: ResourceManager) -> str: model = f"e2e-azure-foundry-messages-{unique_marker()}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody( model=AZURE_FOUNDRY_MODEL, @@ -72,91 +62,78 @@ class TestAzureFoundryMessages: api_key="os.environ/AZURE_AI_API_KEY", ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - return model, resources.key(models=[model]) + resources.defer(lambda: proxy.delete_model(model_id)) + return model @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") def test_basic_nonstream( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) - response = unwrap( - endpoints_client.proxy.messages( - key, - AnthropicMessagesBody( - model=model, - max_tokens=64, - messages=[ChatMessage(role="user", content="Reply with one word.")], - ), - ) + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key(models=[model])) + + message = client.messages.create( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": "Reply with one word."}], ) - assert response.content, f"no content blocks in response: {response}" - text = "".join(block.text or "" for block in response.content if block.type == "text") - assert text.strip(), f"/v1/messages returned no text: {response}" + assert message.content, f"no content blocks in response: {message!r}" + text = "".join(block.text for block in message.content if block.type == "text") + assert text.strip(), f"/v1/messages returned no text: {message.content!r}" @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") def test_basic_stream( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) - result = endpoints_client.proxy.messages_stream( - key, - AnthropicMessagesBody( - model=model, - max_tokens=64, - stream=True, - messages=[ChatMessage(role="user", content="Count from one to three.")], - ), + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key(models=[model])) + + raw = client.messages.with_raw_response.create( + model=model, + max_tokens=64, + stream=True, + messages=[{"role": "user", "content": "Count from one to three."}], ) - _assert_streamed_ok(result) + _assert_rust_served({name.lower(): value for name, value in raw.headers.items()}) + _assert_streamed_ok([event.type for event in raw.parse()]) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") def test_tool_use_nonstream( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) - response = unwrap( - endpoints_client.proxy.messages( - key, - AnthropicMessagesBody( - model=model, - max_tokens=256, - tools=[WEATHER_TOOL], - messages=[ - ChatMessage(role="user", content="What is the weather in Paris? Use the tool.") - ], - ), - ) + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key(models=[model])) + + message = client.messages.create( + model=model, + max_tokens=256, + tools=[WEATHER_TOOL], + messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], ) - assert response.content, f"no content blocks in response: {response}" - assert any(block.type == "tool_use" for block in response.content), ( - f"model did not call the tool: {response}" + assert message.content, f"no content blocks in response: {message!r}" + assert any(block.type == "tool_use" for block in message.content), ( + f"model did not call the tool: {message.content!r}" ) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") def test_tool_use_stream( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) - result = endpoints_client.proxy.messages_stream( - key, - AnthropicMessagesBody( - model=model, - max_tokens=256, - stream=True, - tools=[WEATHER_TOOL], - messages=[ - ChatMessage(role="user", content="What is the weather in Paris? Use the tool.") - ], - ), - ) - require_successful_call(result) - assert result.is_streaming, f"response was not streamed: {result.headers}" - assert not result.stream_error, f"stream errored: {result.stream_error}" - assert result.stream_events, "stream produced no SSE events" - assert any("tool_use" in event for event in result.stream_events), ( - "stream carried no tool_use block" - ) - assert any("message_stop" in event for event in result.stream_events), ( - "stream never reached message_stop" + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key(models=[model])) + + stream = client.messages.create( + model=model, + max_tokens=256, + stream=True, + tools=[WEATHER_TOOL], + messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], ) + events: list[RawMessageStreamEvent] = list(stream) + event_types = [event.type for event in events] + assert event_types, "stream produced no SSE events" + assert any( + event.type == "content_block_start" and event.content_block.type == "tool_use" + for event in events + ), "stream carried no tool_use block" + assert "message_stop" in event_types, "stream never reached message_stop" diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index 44376218c6b..0c5734bc299 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -1,41 +1,35 @@ """Live e2e: POST /v1/messages (Anthropic Messages API) returns a real completion. -Registers an Anthropic deployment at runtime, drives the Messages endpoint through -the gateway, and asserts an assistant message with text came back, both -non-streaming and streamed. Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py. +Registers an Anthropic deployment at runtime and drives the Messages endpoint +through the gateway with the real Anthropic SDK, the client customers actually +use (LIT-4577), asserting an assistant message with text came back, both +non-streaming and streamed. """ from __future__ import annotations import pytest +from anthropic.types import Message, ToolParam from e2e_config import require_env, unique_marker -from e2e_http import require_successful_call, unwrap -from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager -from models import ( - AnthropicCustomTool, - AnthropicMessagesBody, - ChatMessage, - JsonSchemaProperty, - LiteLLMParamsBody, - SpendLogRow, - ToolInputSchema, -) +from models import LiteLLMParamsBody, SpendLogRow +from proxy_client import ProxyClient +from sdk_clients import SdkClients, response_header pytestmark = pytest.mark.e2e ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5" -WEATHER_TOOL = AnthropicCustomTool( - name="get_weather", - description="Get the current weather for a city.", - input_schema=ToolInputSchema( - properties={"city": JsonSchemaProperty(type="string")}, - required=["city"], - ), -) +WEATHER_TOOL: ToolParam = { + "name": "get_weather", + "description": "Get the current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, +} def _approx_equal(actual: float, expected: float) -> bool: @@ -43,60 +37,66 @@ def _approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) +def _text(message: Message) -> str: + return "".join(block.text for block in message.content if block.type == "text") + + class TestAnthropicMessages: def _register( - self, endpoints_client: EndpointsClient, resources: ResourceManager - ) -> tuple[str, str]: - model = f"e2e-messages-{unique_marker()}" - model_id = endpoints_client.create_model( + self, proxy: ProxyClient, resources: ResourceManager, prefix: str = "e2e-messages" + ) -> str: + model = f"{prefix}-{unique_marker()}" + model_id = proxy.create_model( model, - LiteLLMParamsBody( - model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" - ), + LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - return model, resources.key() + resources.defer(lambda: proxy.delete_model(model_id)) + return model @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") def test_messages_returns_completion( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key()) - result = endpoints_client.messages(key, model, "reply with one word") - require_successful_call(result) - parsed = MessagesResult.model_validate_json(result.body) - assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}" - assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}" + message = client.messages.create( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": "reply with one word"}], + ) + assert message.role == "assistant", f"unexpected role: {message.role!r}" + assert _text(message).strip(), f"/v1/messages returned no text: {message.content!r}" @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.cost_logged") def test_messages_logs_cost_matching_the_response_header( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: require_env("ANTHROPIC_API_KEY") - model = f"e2e-messages-cost-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY" - ), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + model = self._register(proxy, resources, prefix="e2e-messages-cost") key = resources.key() + client = sdk.anthropic(key) - result = endpoints_client.messages(key, model, f"reply with one word {unique_marker()}") - require_successful_call(result) - parsed = MessagesResult.model_validate_json(result.body) - assert parsed.role == "assistant" and parsed.text.strip(), ( - f"/v1/messages returned no assistant text: {result.body[:300]}" + raw = client.messages.with_raw_response.create( + model=model, + max_tokens=64, + messages=[{"role": "user", "content": f"reply with one word {unique_marker()}"}], + ) + message = raw.parse() + assert message.role == "assistant" and _text(message).strip(), ( + f"/v1/messages returned no assistant text: {message.content!r}" ) # The customer reads per-request cost off the response header (LIT-4076), so # it must be present and positive on /v1/messages, not only /chat/completions. - header_cost = result.response_cost - assert header_cost is not None and header_cost > 0, ( - "x-litellm-response-cost header missing or non-positive on /v1/messages; " - f"headers={result.headers}" + raw_header_cost = response_header(raw.headers, "x-litellm-response-cost") + assert raw_header_cost is not None, ( + "x-litellm-response-cost header missing on /v1/messages; " + f"headers={dict(raw.headers)}" + ) + header_cost = float(raw_header_cost) + assert header_cost > 0, ( + f"x-litellm-response-cost header non-positive on /v1/messages: {header_cost}" ) # Correlate the spend row by the unique scoped key, not the Anthropic response @@ -107,7 +107,7 @@ class TestAnthropicMessages: def _priced(rows: list[SpendLogRow]) -> bool: return any(r.spend is not None and r.spend > 0 for r in rows) - rows = endpoints_client.proxy.poll_logs_for_key(key, predicate=_priced) + rows = proxy.poll_logs_for_key(key, predicate=_priced) priced = [r for r in rows if r.spend is not None and r.spend > 0] assert priced, ( f"no priced /spend/logs row landed for key {key} within the poll window; got {rows}" @@ -123,50 +123,36 @@ class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") def test_messages_streams_completion( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key()) - result = endpoints_client.proxy.messages_stream( - key, - AnthropicMessagesBody( - model=model, - max_tokens=64, - stream=True, - messages=[ChatMessage(role="user", content="Count from one to three.")], - ), - ) - require_successful_call(result) - assert result.is_streaming, f"response was not streamed: {result.headers}" - assert not result.stream_error, f"stream errored: {result.stream_error}" - assert result.stream_events, "stream produced no SSE events" - assert any("content_block_delta" in event for event in result.stream_events), ( - "stream carried no content deltas" - ) - assert any("message_stop" in event for event in result.stream_events), ( - "stream never reached message_stop" + stream = client.messages.create( + model=model, + max_tokens=64, + stream=True, + messages=[{"role": "user", "content": "Count from one to three."}], ) + event_types = [event.type for event in stream] + assert event_types, "stream produced no SSE events" + assert "content_block_delta" in event_types, "stream carried no content deltas" + assert "message_stop" in event_types, "stream never reached message_stop" @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") def test_messages_tool_use( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model, key = self._register(endpoints_client, resources) + model = self._register(proxy, resources) + client = sdk.anthropic(resources.key()) - response = unwrap( - endpoints_client.proxy.messages( - key, - AnthropicMessagesBody( - model=model, - max_tokens=256, - tools=[WEATHER_TOOL], - messages=[ - ChatMessage(role="user", content="What is the weather in Paris? Use the tool.") - ], - ), - ) + message = client.messages.create( + model=model, + max_tokens=256, + tools=[WEATHER_TOOL], + messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], ) - assert response.content, f"no content blocks in response: {response}" - assert any(block.type == "tool_use" for block in response.content), ( - f"model did not call the tool: {response}" + assert message.content, f"no content blocks in response: {message!r}" + assert any(block.type == "tool_use" for block in message.content), ( + f"model did not call the tool: {message.content!r}" ) diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index 4b3191e60bb..b79f3027692 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -17,27 +17,28 @@ entry whose prefix spans ``system`` plus message turns is invalidated when the reminder is hoisted (the ``system`` field mutates and a turn disappears from ``messages``), while an entry ending at the system block itself would survive the hoist and mask the regression. + +Calls go through the real Anthropic SDK (LIT-4577). The SDK's ``MessageParam`` +type only admits user/assistant roles, so the system reminder turn is cast to +it; the SDK serializes the dict verbatim, which is exactly the wire shape under +test. """ from __future__ import annotations import time +from typing import cast import pytest +from anthropic import Anthropic +from anthropic.types import Message, MessageParam, TextBlockParam from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import Result, unwrap -from endpoints_client import ( - CacheControl, - EndpointsClient, - MessagesResult, - RichMessage, - RichMessagesRequest, - TextBlock, -) from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e @@ -48,50 +49,51 @@ CACHE_PRIMING_DEADLINE_SECONDS = 60.0 CACHE_PRIMING_INTERVAL_SECONDS = 3.0 -def _cacheable_system_block(marker: str) -> TextBlock: +def _cacheable_system_block(marker: str) -> TextBlockParam: """A system prompt comfortably above Sonnet's 1024-token minimum cacheable size, unique per run so no other run's cache entry can satisfy the read.""" text = " ".join( f"Reference paragraph {index} for run {marker}." for index in range(300) ) - return TextBlock(text=text, cache_control=CacheControl()) + return {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} -def _user_turn(text: str, *, cached: bool = False) -> RichMessage: - block = TextBlock(text=text, cache_control=CacheControl() if cached else None) - return RichMessage(role="user", content=[block]) +def _user_turn(text: str, *, cached: bool = False) -> MessageParam: + block: TextBlockParam = ( + {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} + if cached + else {"type": "text", "text": text} + ) + return {"role": "user", "content": [block]} -def _system_reminder_turn() -> RichMessage: - return RichMessage( - role="system", - content=[ - TextBlock( - text="Answer with exactly one word." - ) - ], +def _system_reminder_turn() -> MessageParam: + return cast( + "MessageParam", + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Answer with exactly one word.", + } + ], + }, ) -def _post_messages( - client: EndpointsClient, key: str, body: RichMessagesRequest -) -> Result[MessagesResult]: - return client.proxy.transport.post( - "/v1/messages", - headers=client.proxy.transport.bearer(key), - json=body, - response_type=MessagesResult, - ) +def _text(message: Message) -> str: + return "".join(block.text for block in message.content if block.type == "text") def _register_invoke_deployment( - client: EndpointsClient, resources: ResourceManager, bedrock_model: str + proxy: ProxyClient, resources: ResourceManager, bedrock_model: str ) -> str: model = f"e2e-midsys-{unique_marker()}" - model_id = client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody(model=bedrock_model, aws_region_name=AWS_REGION) ) - resources.defer(lambda: client.delete_model(model_id)) + resources.defer(lambda: proxy.delete_model(model_id)) return model @@ -114,7 +116,7 @@ class PrimedCache(BaseModel): def _prime_prompt_cache( - client: EndpointsClient, key: str, model: str, system_block: TextBlock + client: Anthropic, model: str, system_block: TextBlockParam ) -> PrimedCache: """Send first-turn calls (fresh cache-marked user turn each attempt, identical system prefix) until one both reads the system prefix back from @@ -125,17 +127,19 @@ def _prime_prompt_cache( deadline = time.monotonic() + CACHE_PRIMING_DEADLINE_SECONDS while True: user_text = _first_turn_user_text(unique_marker()) - body = RichMessagesRequest( + usage = client.messages.create( model=model, + max_tokens=64, system=[system_block], messages=[_user_turn(user_text, cached=True)], - ) - usage = unwrap(_post_messages(client, key, body)).usage - if usage.cache_read_input_tokens > 0 and usage.cache_creation_input_tokens > 0: + ).usage + read_tokens = usage.cache_read_input_tokens or 0 + creation_tokens = usage.cache_creation_input_tokens or 0 + if read_tokens > 0 and creation_tokens > 0: return PrimedCache( first_user_text=user_text, - prefix_read_tokens=usage.cache_read_input_tokens, - first_turn_creation_tokens=usage.cache_creation_input_tokens, + prefix_read_tokens=read_tokens, + first_turn_creation_tokens=creation_tokens, ) if time.monotonic() >= deadline: pytest.fail( @@ -151,32 +155,30 @@ class TestBedrockInvokeMidConversationSystem: exercised_on=[], ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register_invoke_deployment( - endpoints_client, resources, FLAGGED_INVOKE_MODEL - ) - key = resources.key(models=[model]) + model = _register_invoke_deployment(proxy, resources, FLAGGED_INVOKE_MODEL) + client = sdk.anthropic(resources.key(models=[model])) system_block = _cacheable_system_block(unique_marker()) - primed = _prime_prompt_cache(endpoints_client, key, model, system_block) + primed = _prime_prompt_cache(client, model, system_block) - reminder_turn_body = RichMessagesRequest( + second = client.messages.create( model=model, + max_tokens=64, system=[system_block], messages=[ _user_turn(primed.first_user_text, cached=True), _system_reminder_turn(), - RichMessage(role="assistant", content=[TextBlock(text="OK.")]), + {"role": "assistant", "content": [{"type": "text", "text": "OK."}]}, _user_turn("Reply with one word again.", cached=True), ], ) - second = unwrap(_post_messages(endpoints_client, key, reminder_turn_body)) - assert second.text.strip(), ( + assert _text(second).strip(), ( f"{model}: reminder turn returned no completion text" ) - assert second.usage.cache_read_input_tokens >= primed.full_prefix_tokens, ( + assert (second.usage.cache_read_input_tokens or 0) >= primed.full_prefix_tokens, ( f"{model}: turn with a mid-conversation system reminder read " f"{second.usage.cache_read_input_tokens} cached tokens, expected at " f"least the {primed.full_prefix_tokens} cached on turn one " @@ -191,29 +193,27 @@ class TestBedrockInvokeMidConversationSystem: exercised_on=[], ) def test_unflagged_model_hoists_system_reminder_and_succeeds( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register_invoke_deployment( - endpoints_client, resources, UNFLAGGED_INVOKE_MODEL - ) - key = resources.key(models=[model]) + model = _register_invoke_deployment(proxy, resources, UNFLAGGED_INVOKE_MODEL) + client = sdk.anthropic(resources.key(models=[model])) - body = RichMessagesRequest( + completion = client.messages.create( model=model, - system=[TextBlock(text="You are terse.")], + max_tokens=64, + system=[{"type": "text", "text": "You are terse."}], messages=[ _user_turn(f"Say hi. Run {unique_marker()}."), _system_reminder_turn(), - RichMessage(role="assistant", content=[TextBlock(text="Hi.")]), + {"role": "assistant", "content": [{"type": "text", "text": "Hi."}]}, _user_turn("Say bye."), ], ) - completion = unwrap(_post_messages(endpoints_client, key, body)) assert completion.role == "assistant", ( f"{model}: unexpected role {completion.role!r}" ) - assert completion.text.strip(), ( + assert _text(completion).strip(), ( f"{model}: conversation with a mid-conversation system reminder " f"returned no text; the reminder was forwarded in place to a model " f"that rejects role 'system' inside messages instead of being hoisted" diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index 97d24e0564b..9a0af34fb5d 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -24,27 +24,28 @@ entry whose prefix spans ``system`` plus message turns is invalidated when the reminder is hoisted (the ``system`` field mutates and a turn disappears from ``messages``), while an entry ending at the system block itself would survive the hoist and mask the regression. + +Calls go through the real Anthropic SDK (LIT-4577). The SDK's ``MessageParam`` +type only admits user/assistant roles, so the system reminder turn is cast to +it; the SDK serializes the dict verbatim, which is exactly the wire shape under +test. """ from __future__ import annotations import time +from typing import cast import pytest +from anthropic import Anthropic +from anthropic.types import Message, MessageParam, TextBlockParam from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import Result, unwrap -from endpoints_client import ( - CacheControl, - EndpointsClient, - MessagesResult, - RichMessage, - RichMessagesRequest, - TextBlock, -) from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e @@ -68,40 +69,47 @@ def _vertex_params(model: str) -> LiteLLMParamsBody: ) -def _cacheable_system_block(marker: str) -> TextBlock: +def _cacheable_system_block(marker: str) -> TextBlockParam: """A system prompt comfortably above the 1024-token minimum cacheable size, unique per run so no other run's cache entry can satisfy the read.""" text = " ".join(f"Reference paragraph {index} for run {marker}." for index in range(300)) - return TextBlock(text=text, cache_control=CacheControl()) + return {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} -def _user_turn(text: str, *, cached: bool = False) -> RichMessage: - block = TextBlock(text=text, cache_control=CacheControl() if cached else None) - return RichMessage(role="user", content=[block]) +def _user_turn(text: str, *, cached: bool = False) -> MessageParam: + block: TextBlockParam = ( + {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} + if cached + else {"type": "text", "text": text} + ) + return {"role": "user", "content": [block]} -def _system_reminder_turn() -> RichMessage: - return RichMessage( - role="system", - content=[TextBlock(text="Answer with exactly one word.")], +def _system_reminder_turn() -> MessageParam: + return cast( + "MessageParam", + { + "role": "system", + "content": [ + { + "type": "text", + "text": "Answer with exactly one word.", + } + ], + }, ) -def _post_messages(client: EndpointsClient, key: str, body: RichMessagesRequest) -> Result[MessagesResult]: - return client.proxy.transport.post( - "/v1/messages", - headers=client.proxy.transport.bearer(key), - json=body, - response_type=MessagesResult, - ) +def _text(message: Message) -> str: + return "".join(block.text for block in message.content if block.type == "text") def _register_deployment( - client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody + proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody ) -> str: model = f"e2e-midsys-{unique_marker()}" - model_id = client.create_model(model, params) - resources.defer(lambda: client.delete_model(model_id)) + model_id = proxy.create_model(model, params) + resources.defer(lambda: proxy.delete_model(model_id)) return model @@ -124,7 +132,7 @@ class PrimedCache(BaseModel): def _prime_prompt_cache( - client: EndpointsClient, key: str, model: str, system_block: TextBlock + client: Anthropic, model: str, system_block: TextBlockParam ) -> PrimedCache: """Send first-turn calls (fresh cache-marked user turn each attempt, identical system prefix) until one both reads the system prefix back from @@ -135,17 +143,19 @@ def _prime_prompt_cache( deadline = time.monotonic() + CACHE_PRIMING_DEADLINE_SECONDS while True: user_text = _first_turn_user_text(unique_marker()) - body = RichMessagesRequest( + usage = client.messages.create( model=model, + max_tokens=64, system=[system_block], messages=[_user_turn(user_text, cached=True)], - ) - usage = unwrap(_post_messages(client, key, body)).usage - if usage.cache_read_input_tokens > 0 and usage.cache_creation_input_tokens > 0: + ).usage + read_tokens = usage.cache_read_input_tokens or 0 + creation_tokens = usage.cache_creation_input_tokens or 0 + if read_tokens > 0 and creation_tokens > 0: return PrimedCache( first_user_text=user_text, - prefix_read_tokens=usage.cache_read_input_tokens, - first_turn_creation_tokens=usage.cache_creation_input_tokens, + prefix_read_tokens=read_tokens, + first_turn_creation_tokens=creation_tokens, ) if time.monotonic() >= deadline: pytest.fail( @@ -156,28 +166,31 @@ def _prime_prompt_cache( def _assert_flagged_model_keeps_cache( - client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + params: LiteLLMParamsBody, ) -> None: - model = _register_deployment(client, resources, params) - key = resources.key(models=[model]) + model = _register_deployment(proxy, resources, params) + client = sdk.anthropic(resources.key(models=[model])) system_block = _cacheable_system_block(unique_marker()) - primed = _prime_prompt_cache(client, key, model, system_block) + primed = _prime_prompt_cache(client, model, system_block) - reminder_turn_body = RichMessagesRequest( + second = client.messages.create( model=model, + max_tokens=64, system=[system_block], messages=[ _user_turn(primed.first_user_text, cached=True), _system_reminder_turn(), - RichMessage(role="assistant", content=[TextBlock(text="OK.")]), + {"role": "assistant", "content": [{"type": "text", "text": "OK."}]}, _user_turn("Reply with one word again.", cached=True), ], ) - second = unwrap(_post_messages(client, key, reminder_turn_body)) - assert second.text.strip(), f"{model}: reminder turn returned no completion text" - assert second.usage.cache_read_input_tokens >= primed.full_prefix_tokens, ( + assert _text(second).strip(), f"{model}: reminder turn returned no completion text" + assert (second.usage.cache_read_input_tokens or 0) >= primed.full_prefix_tokens, ( f"{model}: turn with a mid-conversation system reminder read " f"{second.usage.cache_read_input_tokens} cached tokens, expected at " f"least the {primed.full_prefix_tokens} cached on turn one " @@ -189,25 +202,28 @@ def _assert_flagged_model_keeps_cache( def _assert_unflagged_model_hoists_and_succeeds( - client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + params: LiteLLMParamsBody, ) -> None: - model = _register_deployment(client, resources, params) - key = resources.key(models=[model]) + model = _register_deployment(proxy, resources, params) + client = sdk.anthropic(resources.key(models=[model])) - body = RichMessagesRequest( + completion = client.messages.create( model=model, - system=[TextBlock(text="You are terse.")], + max_tokens=64, + system=[{"type": "text", "text": "You are terse."}], messages=[ _user_turn(f"Say hi. Run {unique_marker()}."), _system_reminder_turn(), - RichMessage(role="assistant", content=[TextBlock(text="Hi.")]), + {"role": "assistant", "content": [{"type": "text", "text": "Hi."}]}, _user_turn("Say bye."), ], ) - completion = unwrap(_post_messages(client, key, body)) assert completion.role == "assistant", f"{model}: unexpected role {completion.role!r}" - assert completion.text.strip(), ( + assert _text(completion).strip(), ( f"{model}: conversation with a mid-conversation system reminder returned " f"no text; the reminder was forwarded in place to a model that rejects " f"role 'system' inside messages instead of being hoisted" @@ -223,19 +239,19 @@ class TestAzureFoundryMidConversationSystem: exercised_on=[], ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - _assert_flagged_model_keeps_cache(endpoints_client, resources, _azure_params(self.FLAGGED_MODEL)) + _assert_flagged_model_keeps_cache(proxy, resources, sdk, _azure_params(self.FLAGGED_MODEL)) @pytest.mark.covers( "llm.messages.azure_foundry.mid_conversation_system.nonstream.works", exercised_on=[], ) def test_unflagged_model_hoists_system_reminder_and_succeeds( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: _assert_unflagged_model_hoists_and_succeeds( - endpoints_client, resources, _azure_params(self.UNFLAGGED_MODEL) + proxy, resources, sdk, _azure_params(self.UNFLAGGED_MODEL) ) @@ -248,17 +264,17 @@ class TestVertexMidConversationSystem: exercised_on=[], ) def test_flagged_model_keeps_prompt_cache_across_system_reminder( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - _assert_flagged_model_keeps_cache(endpoints_client, resources, _vertex_params(self.FLAGGED_MODEL)) + _assert_flagged_model_keeps_cache(proxy, resources, sdk, _vertex_params(self.FLAGGED_MODEL)) @pytest.mark.covers( "llm.messages.vertex.mid_conversation_system.nonstream.works", exercised_on=[], ) def test_unflagged_model_hoists_system_reminder_and_succeeds( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: _assert_unflagged_model_hoists_and_succeeds( - endpoints_client, resources, _vertex_params(self.UNFLAGGED_MODEL) + proxy, resources, sdk, _vertex_params(self.UNFLAGGED_MODEL) ) diff --git a/tests/e2e/llm_translation/test_moderations_e2e.py b/tests/e2e/llm_translation/test_moderations_e2e.py index 69cf4414a48..bc26d79dcab 100644 --- a/tests/e2e/llm_translation/test_moderations_e2e.py +++ b/tests/e2e/llm_translation/test_moderations_e2e.py @@ -1,19 +1,22 @@ """Live e2e: POST /v1/moderations classifies content against the provider policy. -Registers OpenAI's omni moderation model at runtime and asserts the product -promise on both sides of the decision: clearly violent text comes back flagged -with at least one policy category tripped, and benign text comes back not flagged. +Registers OpenAI's omni moderation model at runtime, drives it through the real +OpenAI SDK (LIT-4577), and asserts the product promise on both sides of the +decision: clearly violent text comes back flagged with at least one policy +category tripped, and benign text comes back not flagged. """ from __future__ import annotations import pytest +from openai.types import Moderation +from pydantic import TypeAdapter from e2e_config import unique_marker -from e2e_http import unwrap -from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e @@ -21,45 +24,49 @@ VIOLENT_TEXT = "I am going to find you and kill you, and I will hurt everyone yo BENIGN_TEXT = "I enjoyed the sunny afternoon and a relaxing walk in the park today." -def _register_moderation_model( - endpoints_client: EndpointsClient, resources: ResourceManager -) -> str: +def _register_moderation_model(proxy: ProxyClient, resources: ResourceManager) -> str: model = f"e2e-moderation-{unique_marker()}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody( model="openai/omni-moderation-latest", api_key="os.environ/OPENAI_API_KEY" ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + resources.defer(lambda: proxy.delete_model(model_id)) return model +_CATEGORY_FLAGS = TypeAdapter(dict[str, bool | None]) + + +def _flagged_categories(item: Moderation) -> tuple[str, ...]: + flags = _CATEGORY_FLAGS.validate_python(item.categories.model_dump()) + return tuple(name for name, hit in flags.items() if hit) + + class TestModerations: @pytest.mark.covers("llm.moderations.openai.basic.nonstream.works") def test_moderations_flags_violent_content( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register_moderation_model(endpoints_client, resources) - key = resources.key() + model = _register_moderation_model(proxy, resources) + client = sdk.openai(resources.key()) - result = unwrap(endpoints_client.moderations(key, model, VIOLENT_TEXT)) - item = result.first - assert item is not None, f"/moderations returned no results: {result}" - assert item.flagged, f"violent text was not flagged: {item}" - assert item.flagged_categories, ( - f"flagged result reported no true category: {item}" - ) + moderation = client.moderations.create(model=model, input=VIOLENT_TEXT) + assert moderation.results, f"/moderations returned no results: {moderation!r}" + item = moderation.results[0] + assert item.flagged, f"violent text was not flagged: {item!r}" + assert _flagged_categories(item), f"flagged result reported no true category: {item!r}" def test_moderations_passes_benign_content( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register_moderation_model(endpoints_client, resources) - key = resources.key() + model = _register_moderation_model(proxy, resources) + client = sdk.openai(resources.key()) - result = unwrap(endpoints_client.moderations(key, model, BENIGN_TEXT)) - item = result.first - assert item is not None, f"/moderations returned no results: {result}" + moderation = client.moderations.create(model=model, input=BENIGN_TEXT) + assert moderation.results, f"/moderations returned no results: {moderation!r}" + item = moderation.results[0] assert not item.flagged, ( - f"benign text was flagged as {item.flagged_categories}: {item}" + f"benign text was flagged as {_flagged_categories(item)}: {item!r}" ) diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index cdbf1883314..986ac675e98 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -22,9 +22,9 @@ import pytest from e2e_config import unique_marker from e2e_http import unwrap -from endpoints_client import EndpointsClient from lifecycle import ResourceManager from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse +from proxy_client import ProxyClient pytestmark = pytest.mark.e2e @@ -143,14 +143,14 @@ def _assert_ocr_document(response: OcrResponse) -> None: class TestRustOcrGateway: @pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS) def test_rust_ocr_response( - self, endpoints_client: EndpointsClient, resources: ResourceManager, case: _OcrCase + self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase ) -> None: model = f"rust-ocr-{case.suffix}-{unique_marker()}" - model_id = endpoints_client.create_model(model, case.provider.litellm_params()) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + model_id = proxy.create_model(model, case.provider.litellm_params()) + resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() - response = unwrap(endpoints_client.proxy.ocr(key, OcrBody(model=model, document=case.document))) + response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document))) _assert_ocr_document(response) diff --git a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py index 045988334d5..00ca810db15 100644 --- a/tests/e2e/llm_translation/test_passthrough_headers_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_headers_e2e.py @@ -18,9 +18,8 @@ from pydantic import BaseModel, Field from e2e_config import unique_marker from e2e_http import AuthHeaders, NoBody, require_successful_call, unwrap -from endpoints_client import MessagesResult from lifecycle import ResourceManager -from models import ChatMessage, KeyGenerateBody +from models import AnthropicMessagesResponse, ChatMessage, KeyGenerateBody from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e @@ -128,8 +127,9 @@ class TestPassthroughHeaders: json=_messages_body(), ) require_successful_call(result) - completion = MessagesResult.model_validate_json(result.body) - assert completion.text.strip(), ( + completion = AnthropicMessagesResponse.model_validate_json(result.body) + text = "".join(block.text or "" for block in (completion.content or [])) + assert text.strip(), ( f"static x-api-key must reach Anthropic for the call to succeed at all; got {result.body[:300]}" ) diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index 0857ff65a52..c0e4d7def42 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -1,8 +1,9 @@ """Live e2e: POST /v1/rerank ranks documents by relevance. Registers a Cohere rerank deployment at runtime and asserts the endpoint returns -scored results within the requested top_n. Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py. +scored results within the requested top_n. No official OpenAI/Anthropic SDK +covers /v1/rerank, so the call rides the shared typed transport via +ProxyClient.rerank. """ from __future__ import annotations @@ -10,10 +11,10 @@ from __future__ import annotations import pytest from e2e_config import require_env, unique_marker -from e2e_http import require_successful_call -from endpoints_client import EndpointsClient, RerankResult +from e2e_http import unwrap from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import LiteLLMParamsBody, RerankBody, RerankResponse +from proxy_client import ProxyClient pytestmark = pytest.mark.e2e @@ -26,39 +27,39 @@ DOCUMENTS = [ QUERY = "What is the capital of the United States?" -def _assert_top_n_scored(body: str) -> None: - parsed = RerankResult.model_validate_json(body) - assert parsed.results, f"/rerank returned no results: {body[:300]}" - assert len(parsed.results) <= 3, f"top_n=3 not honored: {body[:300]}" - assert parsed.results[0].relevance_score is not None, ( - f"top rerank result has no relevance_score: {body[:300]}" +def _assert_top_n_scored(response: RerankResponse) -> None: + assert response.results, f"/rerank returned no results: {response!r}" + assert len(response.results) <= 3, f"top_n=3 not honored: {response!r}" + assert response.results[0].relevance_score is not None, ( + f"top rerank result has no relevance_score: {response!r}" ) class TestRerank: @pytest.mark.covers("llm.rerank.cohere.basic.nonstream.works") def test_rerank_scores_top_n( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager ) -> None: model = f"e2e-rerank-{unique_marker()}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody(model="cohere/rerank-v3.5", api_key="os.environ/COHERE_API_KEY"), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() - result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3) - require_successful_call(result) - _assert_top_n_scored(result.body) + response = unwrap( + proxy.rerank(key, RerankBody(model=model, query=QUERY, documents=DOCUMENTS, top_n=3)) + ) + _assert_top_n_scored(response) @pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"]) def test_bedrock_rerank_scores_top_n( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager ) -> None: require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") model = f"e2e-bedrock-rerank-{unique_marker()}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody( model="bedrock/amazon.rerank-v1:0", @@ -67,9 +68,10 @@ class TestRerank: aws_region_name="os.environ/AWS_REGION", ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) + resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() - result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3) - require_successful_call(result) - _assert_top_n_scored(result.body) + response = unwrap( + proxy.rerank(key, RerankBody(model=model, query=QUERY, documents=DOCUMENTS, top_n=3)) + ) + _assert_top_n_scored(response) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index d24d2b53b71..93f844505d0 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -1,8 +1,8 @@ """Live e2e: POST /v1/responses returns a real completion. -Registers an OpenAI deployment at runtime, drives the Responses API through the -gateway, and asserts output text came back. Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py. +Registers an OpenAI deployment at runtime and drives the Responses API through +the gateway with the real OpenAI SDK, the client customers actually use +(LIT-4577), asserting output text came back. """ from __future__ import annotations @@ -11,34 +11,45 @@ import json from typing import cast import pytest -from pydantic import BaseModel, ValidationError +from openai.types.responses import ( + FunctionToolParam, + Response, + ResponseFunctionToolCall, + ResponseInputParam, +) +from pydantic import BaseModel from e2e_config import require_env, unique_marker -from e2e_http import require_successful_call -from endpoints_client import ( - EndpointsClient, - FunctionParameterProperty, - FunctionParameters, - ResponsesFunctionTool, - ResponsesOutputTextDeltaEvent, - ResponsesResult, - ResponsesStreamEventType, -) from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +INSTRUCTIONS = "You are a helpful assistant" +CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" -WEATHER_TOOL = ResponsesFunctionTool( - name="get_weather", - description="Get the weather for a location", - parameters=FunctionParameters( - properties={"location": FunctionParameterProperty(type="string")}, - required=["location"], - ), -) +WEATHER_TOOL: FunctionToolParam = { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + "strict": False, +} + + +def _openai_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY") + + +def _anthropic_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY") def _bedrock_params() -> LiteLLMParamsBody: @@ -50,6 +61,25 @@ def _bedrock_params() -> LiteLLMParamsBody: ) +def _register(proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody) -> str: + model = f"e2e-responses-{unique_marker()}" + model_id = proxy.create_model(model, params) + resources.defer(lambda: proxy.delete_model(model_id)) + return model + + +def _function_calls(response: Response) -> tuple[ResponseFunctionToolCall, ...]: + return tuple(item for item in response.output if isinstance(item, ResponseFunctionToolCall)) + + +def _assert_weather_call(response: Response) -> None: + function_call = next((call for call in _function_calls(response) if call.name == "get_weather"), None) + assert function_call is not None, f"no get_weather function call: {response.output!r}" + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + + class WeatherArguments(BaseModel): location: str @@ -57,242 +87,157 @@ class WeatherArguments(BaseModel): class TestResponses: @pytest.mark.covers("llm.responses.openai.basic.nonstream.works") def test_responses_returns_completion( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _openai_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses(key, model, "reply with one word") - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS + ) + assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.openai.basic.stream.works") def test_responses_streaming_returns_completion( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _openai_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses(key, model, "reply with one word", stream=True) - require_successful_call(result) - delta_events = tuple( - parsed - for event in result.stream_events - if (parsed := _parse_stream_event(event)) is not None + stream = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, stream=True + ) + events = list(stream) + assert events, "responses stream returned no events" + deltas = [event.delta for event in events if event.type == "response.output_text.delta"] + assert any(delta for delta in deltas), "responses stream returned no text deltas" + assert events[-1].type == "response.completed", ( + f"responses stream did not terminate with response.completed: {events[-1].type}" ) - - assert any(event.delta for event in delta_events), "responses stream returned no text deltas" - assert result.stream_events, "responses stream returned no events" - assert ( - ResponsesStreamEventType.model_validate_json(result.stream_events[-1]).type - == "response.completed" - ), "responses stream did not terminate with response.completed" @pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged") def test_responses_logs_cost( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), + model = _register(proxy, resources, _openai_params()) + client = sdk.openai(resources.key()) + + raw = client.responses.with_raw_response.create( + model=model, input=f"reply with one word {unique_marker()}", instructions=INSTRUCTIONS + ) + response = raw.parse() + assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" + assert raw.headers.get("x-litellm-call-id") and response.id, ( + f"missing response identifiers: id={response.id!r}, headers={dict(raw.headers)}" ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() - result = endpoints_client.responses(key, model, f"reply with one word {unique_marker()}") - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" - assert result.call_id and parsed.id, f"missing response identifiers: {result.body[:300]}" - - rows = endpoints_client.proxy.poll_logs_for_request_id( - parsed.id, + rows = proxy.poll_logs_for_request_id( + response.id, predicate=lambda logged_rows: any((row.spend or 0) > 0 for row in logged_rows), ) row = next((logged_row for logged_row in rows if (logged_row.spend or 0) > 0), None) - assert row is not None, f"no costed spend row for response id {parsed.id}" + assert row is not None, f"no costed spend row for response id {response.id}" assert "gpt-4o-mini" in (row.model or ""), f"unexpected spend row model: {row.model}" @pytest.mark.covers("llm.responses.openai.tool_use.nonstream.works") def test_responses_returns_function_call( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _openai_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses_with_tools( - key, - model, - "What is the weather in San Francisco? Use the get_weather tool.", - [ - ResponsesFunctionTool( - name="get_weather", - description="Get the weather for a location", - parameters=FunctionParameters( - properties={"location": FunctionParameterProperty(type="string")}, - required=["location"], - ), - ) - ], + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], ) - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - function_call = next( - (call for call in parsed.function_calls if call.name == "get_weather"), - None, - ) - assert function_call is not None, f"no get_weather function call: {result.body[:500]}" - assert function_call.arguments is not None - raw_arguments = cast(object, json.loads(function_call.arguments)) - arguments = WeatherArguments.model_validate(raw_arguments) - assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + _assert_weather_call(response) @pytest.mark.covers("llm.responses.openai.vision.nonstream.works") def test_responses_vision_describes_image( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, + model = _register( + proxy, + resources, LiteLLMParamsBody(model="openai/gpt-4o", api_key="os.environ/OPENAI_API_KEY"), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + client = sdk.openai(resources.key()) - result = endpoints_client.responses_vision( - key, - model, - "What animal is shown in this image? Answer in one word", - "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg", + vision_input: ResponseInputParam = [ + { + "role": "user", + "content": [ + { + "type": "input_text", + "text": "What animal is shown in this image? Answer in one word", + }, + {"type": "input_image", "image_url": CAT_IMAGE_URL, "detail": "auto"}, + ], + } + ] + response = client.responses.create(model=model, input=vision_input, instructions=INSTRUCTIONS) + text = response.output_text.strip().lower() + assert text, f"/responses vision returned no output text: {response.output!r}" + assert any(keyword in text for keyword in ("cat", "feline")), ( + f"vision response did not describe the image: {text[:300]}" ) - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - text = parsed.text.strip().lower() - assert text, f"/responses vision returned no output text: {result.body[:300]}" - assert any( - keyword in text - for keyword in ("cat", "feline") - ), f"vision response did not describe the image: {parsed.text[:300]}" @pytest.mark.covers("llm.responses.anthropic.basic.nonstream.works") def test_responses_anthropic_returns_completion( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" - ), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _anthropic_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses(key, model, "reply with one word") - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}" + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS + ) + assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") def test_responses_anthropic_returns_function_call( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model( - model, - LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" - ), - ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _anthropic_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses_with_tools( - key, - model, - "What is the weather in San Francisco? Use the get_weather tool.", - [ - ResponsesFunctionTool( - name="get_weather", - description="Get the weather for a location", - parameters=FunctionParameters( - properties={"location": FunctionParameterProperty(type="string")}, - required=["location"], - ), - ) - ], + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], ) - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - function_call = next( - (call for call in parsed.function_calls if call.name == "get_weather"), - None, - ) - assert function_call is not None, f"no get_weather function call: {result.body[:500]}" - assert function_call.arguments is not None - raw_arguments = cast(object, json.loads(function_call.arguments)) - arguments = WeatherArguments.model_validate(raw_arguments) - assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + _assert_weather_call(response) @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") def test_responses_bedrock_returns_completion( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model(model, _bedrock_params()) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _bedrock_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses(key, model, "reply with one word") - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}" + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS + ) + assert response.output_text.strip(), ( + f"/responses over bedrock returned no output text: {response.output!r}" + ) @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") def test_responses_bedrock_returns_function_call( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION") - model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model(model, _bedrock_params()) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + model = _register(proxy, resources, _bedrock_params()) + client = sdk.openai(resources.key()) - result = endpoints_client.responses_with_tools( - key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL] + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], ) - require_successful_call(result) - parsed = ResponsesResult.model_validate_json(result.body) - function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None) - assert function_call is not None, f"no get_weather function call over bedrock: {result.body[:500]}" - assert function_call.arguments is not None - raw_arguments = cast(object, json.loads(function_call.arguments)) - arguments = WeatherArguments.model_validate(raw_arguments) - assert arguments.location, f"function call arguments missing location: {function_call.arguments}" - - -def _parse_stream_event( - event: str, -) -> ResponsesOutputTextDeltaEvent | None: - try: - return ResponsesOutputTextDeltaEvent.model_validate_json(event) - except ValidationError: - return None + _assert_weather_call(response) diff --git a/tests/e2e/llm_translation/test_responses_metadata_e2e.py b/tests/e2e/llm_translation/test_responses_metadata_e2e.py index 6cf24348095..87727bde4d8 100644 --- a/tests/e2e/llm_translation/test_responses_metadata_e2e.py +++ b/tests/e2e/llm_translation/test_responses_metadata_e2e.py @@ -1,8 +1,8 @@ """Live e2e: /v1/responses with store + metadata (LIT-1201 customer path). -Customers attach metadata and store=true, then continue with previous_response_id. -Both turns must succeed, and any Redis keys written for the session must carry a -positive TTL (not unbounded). +Customers attach metadata and store=true through the OpenAI SDK, then continue +with previous_response_id. Both turns must succeed, and any Redis keys written +for the session must carry a positive TTL (not unbounded). """ from __future__ import annotations @@ -15,21 +15,14 @@ import pytest from pydantic import BaseModel, ConfigDict from e2e_config import require_env, unique_marker -from e2e_http import require_successful_call -from endpoints_client import EndpointsClient, ResponsesResult from lifecycle import ResourceManager from models import LiteLLMParamsBody +from proxy_client import ProxyClient +from sdk_clients import SdkClients pytestmark = pytest.mark.e2e - -class ResponsesMetadataBody(BaseModel): - model: str - input: str - store: bool = True - metadata: dict[str, str] - previous_response_id: str | None = None - instructions: str | None = "You are a helpful assistant." +INSTRUCTIONS = "You are a helpful assistant." class RedisKeyInfo(BaseModel): @@ -67,50 +60,42 @@ class TestResponsesMetadata: exercised_on=["responses"], ) def test_store_metadata_continues_and_redis_keys_have_ttl( - self, endpoints_client: EndpointsClient, resources: ResourceManager + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: # Anthropic avoids OpenAI/Gemini quota flakes; Responses translation still # exercises store + metadata + previous_response_id on the proxy. marker = unique_marker() model = f"e2e-resp-meta-{marker}" - model_id = endpoints_client.create_model( + model_id = proxy.create_model( model, LiteLLMParamsBody( model="anthropic/claude-haiku-4-5-20251001", api_key="os.environ/ANTHROPIC_API_KEY", ), ) - resources.defer(lambda: endpoints_client.delete_model(model_id)) - key = resources.key() + resources.defer(lambda: proxy.delete_model(model_id)) + client = sdk.openai(resources.key()) - first = endpoints_client.proxy.transport.send( - "/v1/responses", - headers=endpoints_client.proxy.transport.bearer(key), - json=ResponsesMetadataBody( - model=model, - input=f"Remember marker {marker}. Reply with one word.", - metadata={"session_id": marker, "customer": "e2e"}, - ), + first = client.responses.create( + model=model, + input=f"Remember marker {marker}. Reply with one word.", + store=True, + metadata={"session_id": marker, "customer": "e2e"}, + instructions=INSTRUCTIONS, ) - require_successful_call(first) - parsed = ResponsesResult.model_validate_json(first.body) - assert parsed.id, f"responses must return an id: {first.body[:300]}" - assert parsed.text.strip(), f"responses returned empty text: {first.body[:300]}" + assert first.id, f"responses must return an id: {first!r}" + assert first.output_text.strip(), f"responses returned empty text: {first.output!r}" - second = endpoints_client.proxy.transport.send( - "/v1/responses", - headers=endpoints_client.proxy.transport.bearer(key), - json=ResponsesMetadataBody( - model=model, - input="Reply with the single word ok.", - previous_response_id=parsed.id, - metadata={"session_id": marker, "turn": "2"}, - ), + second = client.responses.create( + model=model, + input="Reply with the single word ok.", + store=True, + previous_response_id=first.id, + metadata={"session_id": marker, "turn": "2"}, + instructions=INSTRUCTIONS, ) - require_successful_call(second) - second_parsed = ResponsesResult.model_validate_json(second.body) - assert second_parsed.text.strip(), ( - f"previous_response_id follow-up returned empty text: {second.body[:300]}" + assert second.output_text.strip(), ( + f"previous_response_id follow-up returned empty text: {second.output!r}" ) time.sleep(1.0) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index b3ea9346180..722722f4e19 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -364,6 +364,25 @@ class EmbedResponse(BaseModel): model: str | None = None +# ---------- rerank ---------- + + +class RerankBody(BaseModel): + model: str + query: str + documents: list[str] + top_n: int + + +class RerankItem(BaseModel): + index: int | None = None + relevance_score: float | None = None + + +class RerankResponse(BaseModel): + results: list[RerankItem] = [] + + # ---------- ocr ---------- diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 6c6b948e29c..234c37d2f77 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -57,6 +57,8 @@ from models import ( ModelUpdateBody, OcrBody, OcrResponse, + RerankBody, + RerankResponse, SpendLogRow, SpendLogs, SpendLogsPage, @@ -291,6 +293,16 @@ class ProxyClient: response_type=OcrResponse, ) + def rerank(self, key: str, body: RerankBody) -> Result[RerankResponse]: + """POST /v1/rerank (Cohere-format). No official OpenAI/Anthropic SDK + covers this route, so it stays on the shared typed transport.""" + return self.transport.post( + "/v1/rerank", + headers=self.transport.bearer(key), + json=body, + response_type=RerankResponse, + ) + def count_tokens(self, key: str, body: CountTokensBody) -> Result[CountTokensResponse]: """POST /v1/messages/count_tokens (Anthropic-native). Sends the anthropic-version header so the native path accepts it; harmless on the diff --git a/uv.lock b/uv.lock index b3d6fccff26..caf976b9202 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-19T00:00:06.091071Z" +exclude-newer = "2026-07-20T03:20:37.107777782Z" exclude-newer-span = "P3D" [manifest] @@ -4300,6 +4300,7 @@ dev = [ { name = "vcrpy" }, ] e2e-dev = [ + { name = "anthropic" }, { name = "locust" }, { name = "playwright" }, { name = "websockets" }, @@ -4477,6 +4478,7 @@ dev = [ { name = "vcrpy", specifier = "==8.2.1" }, ] e2e-dev = [ + { name = "anthropic", specifier = "==0.84.0" }, { name = "locust", specifier = "==2.45.0" }, { name = "playwright", specifier = "==1.61.0" }, { name = "websockets", specifier = ">=15.0.1,<16.0" }, From 0e34c18d7ed9a846258fe6456117b8143134bfc6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:32:29 +0000 Subject: [PATCH 2/4] docs(e2e): name the e2e-dev group in the documented suite run commands The llm_translation suite needs the e2e-dev dependency group at collection time (websockets for the realtime folder, now also the anthropic SDK for the sdk fixture). make bootstrap installs the group, but the documented pytest command did not name it, so a default dev-group environment failed collection. Naming the group on uv run makes the command work from any environment state --- tests/e2e/CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index fa0174f686c..e542c123988 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -35,10 +35,10 @@ The suites run against a live proxy, so bring one up first by running the litell curl -fs http://localhost:4000/health/liveliness ``` -4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): +4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`). The suites' client dependencies (the provider SDKs, websockets) live in the `e2e-dev` dependency group; `make bootstrap` installs it, and naming the group on the run keeps the command working from any environment state: ```bash - uv run pytest tests/e2e/llm_translation/ -v + uv run --group e2e-dev pytest tests/e2e/llm_translation/ -v ``` The browser tests in the `management/` suite drive the dashboard the proxy serves at `/ui` through playwright, an optional dependency behind `importorskip` (the suite's API tests run without it). It lives in the `e2e-dev` dependency group; install it along with its browser: @@ -148,7 +148,7 @@ Before you push ```bash litellm --config .yml --port 4000 - uv run pytest tests/e2e// -v + uv run --group e2e-dev pytest tests/e2e// -v ``` 4. Capture screenshots of the test run and attach them to the PR as proof From 057c45f23f7db2ad52b5d316112742571848164a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:39:16 -0700 Subject: [PATCH 3/4] test(e2e): send no-cache on cacheable SDK calls and accept Bedrock's 400 on the responses leg The deleted wrapper put cache: {"no-cache": true} on every request body, so the gateway's response cache never answered a re-sent prompt. The SDKs send nothing of the sort, and the mid-conversation prompt-cache priming loop re-sends an identical body until the provider reports a warm cache, which a cached reply never does. NO_PROXY_CACHE in sdk_clients.py restores the field as extra_body on every messages, responses, completions, and embeddings call. The wrapper also returned a 4xx as a value where the SDKs raise. The Bedrock safety_identifier test judges the captured Converse body, and Claude on Bedrock rejects the forwarded field with a 400, so the /v1/responses leg now suppresses openai.BadRequestError the way the chat leg carries the same 400 as a Result. --- tests/e2e/llm_translation/sdk_clients.py | 9 +++ ...test_bedrock_web_search_server_tool_e2e.py | 3 +- .../test_completions_endpoint_e2e.py | 3 +- .../test_credential_messages_e2e.py | 7 +-- .../test_embeddings_endpoint_e2e.py | 20 +++--- .../test_messages_azure_foundry_e2e.py | 25 +++----- .../e2e/llm_translation/test_messages_e2e.py | 45 ++++++++----- ...st_messages_mid_conversation_system_e2e.py | 10 ++- ...onversation_system_native_providers_e2e.py | 10 ++- .../e2e/llm_translation/test_responses_e2e.py | 63 +++++++++++++------ 10 files changed, 125 insertions(+), 70 deletions(-) diff --git a/tests/e2e/llm_translation/sdk_clients.py b/tests/e2e/llm_translation/sdk_clients.py index 8a6b187f979..145efbdca98 100644 --- a/tests/e2e/llm_translation/sdk_clients.py +++ b/tests/e2e/llm_translation/sdk_clients.py @@ -13,12 +13,21 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType +from typing import Final from anthropic import Anthropic from openai import OpenAI from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT +NO_PROXY_CACHE: Final = MappingProxyType({"cache": {"no-cache": True}}) +"""``extra_body`` for every cacheable SDK call (messages, responses, completions, +embeddings): the gateway under test caches those call types, so an identical +re-send would otherwise be served from Redis instead of reaching the provider, +which hides provider-side behavior such as prompt-cache warm-up. The SDKs +themselves cannot bypass it (``Cache-Control`` only sets a TTL on the proxy).""" + def response_header(headers: Mapping[str, str], name: str) -> str | None: """Typed read of an SDK response header: httpx.Headers.get returns Any and diff --git a/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py b/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py index d44e9ba43f7..b4253a82dd8 100644 --- a/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py @@ -39,7 +39,7 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -83,6 +83,7 @@ class TestBedrockWebSearchServerTool: max_tokens=512, tools=[WEB_SEARCH_TOOL], messages=[{"role": "user", "content": SEARCH_PROMPT}], + extra_body=NO_PROXY_CACHE, ) assert response.content, f"no content blocks in response: {response!r}" diff --git a/tests/e2e/llm_translation/test_completions_endpoint_e2e.py b/tests/e2e/llm_translation/test_completions_endpoint_e2e.py index c6415ce7f29..63fcee3ce36 100644 --- a/tests/e2e/llm_translation/test_completions_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_completions_endpoint_e2e.py @@ -15,7 +15,7 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -40,6 +40,7 @@ class TestCompletionsEndpoint: model=model, prompt="Finish this sentence in a few words: the capital of France is", max_tokens=32, + extra_body=NO_PROXY_CACHE, ) assert completion.choices, f"/v1/completions returned no choices: {completion!r}" text = (completion.choices[0].text or "").strip() diff --git a/tests/e2e/llm_translation/test_credential_messages_e2e.py b/tests/e2e/llm_translation/test_credential_messages_e2e.py index d4bf4566cc0..52306ce3a7a 100644 --- a/tests/e2e/llm_translation/test_credential_messages_e2e.py +++ b/tests/e2e/llm_translation/test_credential_messages_e2e.py @@ -10,16 +10,14 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import CredentialCreateBody, LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e class TestCredentialBackedMessages: @pytest.mark.covers("mgmt.credential.new.serves_request") - def test_credential_backed_messages( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_credential_backed_messages(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: marker = unique_marker() credential_name = f"e2e-cred-{marker}" model = f"e2e-cred-messages-{marker}" @@ -48,6 +46,7 @@ class TestCredentialBackedMessages: model=model, max_tokens=64, messages=[{"role": "user", "content": "reply with one word"}], + extra_body=NO_PROXY_CACHE, ) assert message.role == "assistant", f"unexpected role: {message.role!r}" text = "".join(block.text for block in message.content if block.type == "text") diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index c46b10e25d2..41282260b7e 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -17,7 +17,7 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -39,7 +39,9 @@ def _openai_embeddings_params() -> LiteLLMParamsBody: ) -def _register(proxy: ProxyClient, resources: ResourceManager, prefix: str, params: LiteLLMParamsBody) -> tuple[str, str]: +def _register( + proxy: ProxyClient, resources: ResourceManager, prefix: str, params: LiteLLMParamsBody +) -> tuple[str, str]: model = f"{prefix}-{unique_marker()}" model_id = proxy.create_model(model, params) resources.defer(lambda: proxy.delete_model(model_id)) @@ -56,7 +58,7 @@ def _assert_embedding_vector( model, key = _register(proxy, resources, prefix, params) client = sdk.openai(key) - embeddings = client.embeddings.create(model=model, input="Say this is a test!") + embeddings = client.embeddings.create(model=model, input="Say this is a test!", extra_body=NO_PROXY_CACHE) assert embeddings.data, f"/embeddings returned no data: {embeddings!r}" vector = embeddings.data[0].embedding assert vector, f"/embeddings returned no vector: {embeddings!r}" @@ -66,9 +68,7 @@ def _assert_embedding_vector( class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") - def test_embeddings_returns_vector( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_embeddings_returns_vector(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: _assert_embedding_vector(proxy, resources, sdk, "e2e-embeddings", _openai_embeddings_params()) @pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works") @@ -118,11 +118,11 @@ class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") - def test_array_input_returns_vectors( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_array_input_returns_vectors(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources, "e2e-embeddings-array", _openai_embeddings_params()) - embeddings = sdk.openai(key).embeddings.create(model=model, input=["Hello", "World", "Test"]) + embeddings = sdk.openai(key).embeddings.create( + model=model, input=["Hello", "World", "Test"], extra_body=NO_PROXY_CACHE + ) assert len(embeddings.data) == 3, f"expected 3 vectors: {embeddings!r}" @pytest.mark.replayable diff --git a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py index 527deb8444a..8629cf12013 100644 --- a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py +++ b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py @@ -17,7 +17,7 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -55,9 +55,7 @@ class TestAzureFoundryMessages: return model @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") - def test_basic_nonstream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_basic_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -65,15 +63,14 @@ class TestAzureFoundryMessages: model=model, max_tokens=64, messages=[{"role": "user", "content": "Reply with one word."}], + extra_body=NO_PROXY_CACHE, ) assert message.content, f"no content blocks in response: {message!r}" text = "".join(block.text for block in message.content if block.type == "text") assert text.strip(), f"/v1/messages returned no text: {message.content!r}" @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") - def test_basic_stream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_basic_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -82,13 +79,12 @@ class TestAzureFoundryMessages: max_tokens=64, stream=True, messages=[{"role": "user", "content": "Count from one to three."}], + extra_body=NO_PROXY_CACHE, ) _assert_streamed_ok([event.type for event in stream]) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") - def test_tool_use_nonstream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_tool_use_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -97,6 +93,7 @@ class TestAzureFoundryMessages: max_tokens=256, tools=[WEATHER_TOOL], messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], + extra_body=NO_PROXY_CACHE, ) assert message.content, f"no content blocks in response: {message!r}" assert any(block.type == "tool_use" for block in message.content), ( @@ -104,9 +101,7 @@ class TestAzureFoundryMessages: ) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") - def test_tool_use_stream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_tool_use_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -116,12 +111,12 @@ class TestAzureFoundryMessages: stream=True, tools=[WEATHER_TOOL], messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], + extra_body=NO_PROXY_CACHE, ) events: list[RawMessageStreamEvent] = list(stream) event_types = [event.type for event in events] assert event_types, "stream produced no SSE events" assert any( - event.type == "content_block_start" and event.content_block.type == "tool_use" - for event in events + event.type == "content_block_start" and event.content_block.type == "tool_use" for event in events ), "stream carried no tool_use block" assert "message_stop" in event_types, "stream never reached message_stop" diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index b8007d7c475..d048d1343eb 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -36,7 +36,7 @@ from lifecycle import ResourceManager from models import ChatMessage, LiteLLMParamsBody, SpendLogRow from proxy_client import ProxyClient from pydantic import BaseModel, ConfigDict -from sdk_clients import SdkClients, response_header +from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header pytestmark = [pytest.mark.e2e, pytest.mark.replayable] @@ -96,13 +96,13 @@ def _user_turn(text: str) -> MessageParam: class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") - def test_messages_returns_completion( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_messages_returns_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources) client = sdk.anthropic(key) - message = client.messages.create(model=model, max_tokens=64, messages=[_user_turn("reply with one word")]) + message = client.messages.create( + model=model, max_tokens=64, messages=[_user_turn("reply with one word")], extra_body=NO_PROXY_CACHE + ) assert message.role == "assistant", f"unexpected role: {message.role!r}" assert _text(message).strip(), f"/v1/messages returned no text: {message.content!r}" @@ -117,6 +117,7 @@ class TestAnthropicMessages: model=model, max_tokens=64, messages=[_user_turn(f"reply with one word {unique_marker()}")], + extra_body=NO_PROXY_CACHE, ) message = raw.parse() assert message.role == "assistant" and _text(message).strip(), ( @@ -154,9 +155,7 @@ class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") @pytest.mark.provider_live - def test_messages_streams_completion( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_messages_streams_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: """Edge-wired like its non-streaming siblings, so record and replay both carry the streamed response. @@ -175,6 +174,7 @@ class TestAnthropicMessages: max_tokens=800, stream=True, messages=[_user_turn("Count from 1 to 200, one number per line.")], + extra_body=NO_PROXY_CACHE, ) arrivals: Final = tuple((event, time.monotonic() - started) for event in stream) assert arrivals, "stream produced no SSE events" @@ -221,13 +221,16 @@ class TestAnthropicMessages: max_tokens=256, tools=[WEATHER_TOOL], messages=[_user_turn("What is the weather in Paris? Use the tool.")], + extra_body=NO_PROXY_CACHE, ) assert message.content, f"no content blocks in response: {message!r}" assert any(isinstance(block, ToolUseBlock) for block in message.content), ( f"model did not call the tool: {message.content!r}" ) - @pytest.mark.skip(reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400") + @pytest.mark.skip( + reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400" + ) @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") def test_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) @@ -238,7 +241,9 @@ class TestAnthropicMessages: ) assert_client_error(result, "messages missing messages") - @pytest.mark.skip(reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400") + @pytest.mark.skip( + reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400" + ) @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") def test_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) @@ -292,9 +297,7 @@ def _tool_from_stream(events: tuple[RawMessageStreamEvent, ...]) -> ToolUseBlock terminal_positions: Final = tuple( index for index, event in enumerate(events) if isinstance(event, RawMessageDeltaEvent) ) - stop_reasons: Final = tuple( - event.delta.stop_reason for event in events if isinstance(event, RawMessageDeltaEvent) - ) + stop_reasons: Final = tuple(event.delta.stop_reason for event in events if isinstance(event, RawMessageDeltaEvent)) assert stop_reasons == ("tool_use",) assert len(terminal_positions) == 1 and stops[0] < terminal_positions[0] < len(events) - 1 assert tuple(index for index, event in enumerate(events) if event.type == "message_stop") == (len(events) - 1,), ( @@ -309,12 +312,23 @@ def _request_tool(client: Anthropic, model: str, question: MessageParam, tool: T if stream: events: Final = tuple( client.messages.create( - model=model, max_tokens=2048, messages=[question], tools=[tool], tool_choice=tool_choice, stream=True + model=model, + max_tokens=2048, + messages=[question], + tools=[tool], + tool_choice=tool_choice, + stream=True, + extra_body=NO_PROXY_CACHE, ) ) return _tool_from_stream(events) message: Final = client.messages.create( - model=model, max_tokens=2048, messages=[question], tools=[tool], tool_choice=tool_choice + model=model, + max_tokens=2048, + messages=[question], + tools=[tool], + tool_choice=tool_choice, + extra_body=NO_PROXY_CACHE, ) blocks: Final = tuple(block for block in message.content if isinstance(block, ToolUseBlock)) assert len(blocks) == 1 @@ -367,6 +381,7 @@ class TestOpenAIMessagesToolContinuation: }, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": emitted.id, "content": receipt}]}, ], + extra_body=NO_PROXY_CACHE, ) assert _text(continuation).strip() == receipt, "continuation did not consume the correlated tool result" assert all(not isinstance(block, ToolUseBlock) for block in continuation.content) diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index 730d19749ba..e9b4b394996 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -38,7 +38,7 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -63,7 +63,9 @@ def _cacheable_system_block(marker: str) -> TextBlockParam: def _user_turn(text: str, *, cached: bool = False) -> MessageParam: block: TextBlockParam = ( - {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} if cached else {"type": "text", "text": text} + {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} + if cached + else {"type": "text", "text": text} ) return {"role": "user", "content": [block]} @@ -87,7 +89,9 @@ def _text(message: Message) -> str: def _send(client: Anthropic, model: str, system_block: TextBlockParam, messages: Sequence[MessageParam]) -> Message: - return client.messages.create(model=model, max_tokens=64, system=[system_block], messages=messages) + return client.messages.create( + model=model, max_tokens=64, system=[system_block], messages=messages, extra_body=NO_PROXY_CACHE + ) def _register_invoke_deployment(proxy: ProxyClient, resources: ResourceManager, bedrock_model: str) -> str: diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index e95bc60d7cc..9f5ed8b05da 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -45,7 +45,7 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -83,7 +83,9 @@ def _cacheable_system_block(marker: str) -> TextBlockParam: def _user_turn(text: str, *, cached: bool = False) -> MessageParam: block: TextBlockParam = ( - {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} if cached else {"type": "text", "text": text} + {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} + if cached + else {"type": "text", "text": text} ) return {"role": "user", "content": [block]} @@ -107,7 +109,9 @@ def _text(message: Message) -> str: def _send(client: Anthropic, model: str, system_block: TextBlockParam, messages: Sequence[MessageParam]) -> Message: - return client.messages.create(model=model, max_tokens=64, system=[system_block], messages=messages) + return client.messages.create( + model=model, max_tokens=64, system=[system_block], messages=messages, extra_body=NO_PROXY_CACHE + ) def _register_deployment(proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody) -> str: diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 81314600125..9cc70da63b0 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -9,6 +9,7 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations +import contextlib import json import threading from collections.abc import Mapping @@ -16,6 +17,7 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, cast +import openai import pytest from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker from e2e_http import assert_client_error @@ -31,7 +33,7 @@ from provider_edge import LiveEdge, start_provider_edge from provider_edge_bedrock import bedrock_signer from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -136,7 +138,9 @@ class TestResponses: model = _register(proxy, resources, _openai_params()) client = sdk.openai(resources.key()) - response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.openai.basic.stream.works") @@ -147,7 +151,11 @@ class TestResponses: client = sdk.openai(resources.key()) stream = client.responses.create( - model=model, input="reply with one word", instructions=INSTRUCTIONS, stream=True + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + stream=True, + extra_body=NO_PROXY_CACHE, ) events = tuple(stream) assert events, "responses stream returned no events" @@ -158,14 +166,15 @@ class TestResponses: ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged") - def test_responses_logs_cost( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_responses_logs_cost(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = _register(proxy, resources, _openai_params()) client = sdk.openai(resources.key()) raw = client.responses.with_raw_response.create( - model=model, input=f"reply with one word {unique_marker()}", instructions=INSTRUCTIONS + model=model, + input=f"reply with one word {unique_marker()}", + instructions=INSTRUCTIONS, + extra_body=NO_PROXY_CACHE, ) response = raw.parse() assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @@ -193,6 +202,7 @@ class TestResponses: input="What is the weather in San Francisco? Use the get_weather tool.", instructions=INSTRUCTIONS, tools=[WEATHER_TOOL], + extra_body=NO_PROXY_CACHE, ) _assert_weather_call(response) @@ -216,7 +226,9 @@ class TestResponses: ], } ] - response = client.responses.create(model=model, input=vision_input, instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input=vision_input, instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) text = response.output_text.strip().lower() assert text, f"/responses vision returned no output text: {response.output!r}" assert any(keyword in text for keyword in ("cat", "feline")), ( @@ -230,7 +242,9 @@ class TestResponses: model = _register(proxy, resources, _anthropic_params()) client = sdk.openai(resources.key()) - response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") @@ -245,6 +259,7 @@ class TestResponses: input="What is the weather in San Francisco? Use the get_weather tool.", instructions=INSTRUCTIONS, tools=[WEATHER_TOOL], + extra_body=NO_PROXY_CACHE, ) _assert_weather_call(response) @@ -255,7 +270,9 @@ class TestResponses: model = _register(proxy, resources, _bedrock_params()) client = sdk.openai(resources.key()) - response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) assert response.output_text.strip(), f"/responses over bedrock returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") @@ -270,6 +287,7 @@ class TestResponses: input="What is the weather in San Francisco? Use the get_weather tool.", instructions=INSTRUCTIONS, tools=[WEATHER_TOOL], + extra_body=NO_PROXY_CACHE, ) _assert_weather_call(response) @@ -278,10 +296,15 @@ class TestResponses: def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, endpoint: str ) -> None: + """Judges the Converse bodies the edge captured, not the reply: Claude on + Bedrock rejects the forwarded field with a 400, which the chat leg's + ``Result`` carries as a value and the OpenAI SDK raises.""" capture: Final = ConverseRequestCapture() edge: Final = start_provider_edge( LiveEdge(observe_request=capture.observe, sign=bedrock_signer(BEDROCK_EDGE_REGION)), - mounts=MappingProxyType({BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"}), + mounts=MappingProxyType( + {BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"} + ), bind_host=PROVIDER_EDGE_BIND_HOST, advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, ) @@ -303,12 +326,14 @@ class TestResponses: safety_identifier: Final = f"end-user-{unique_marker()}" if endpoint == "/v1/responses": - sdk.openai(key).responses.create( - model=model, - input="reply with one word", - instructions=INSTRUCTIONS, - safety_identifier=safety_identifier, - ) + with contextlib.suppress(openai.BadRequestError): + sdk.openai(key).responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + safety_identifier=safety_identifier, + extra_body=NO_PROXY_CACHE, + ) else: proxy.chat( key, @@ -325,7 +350,9 @@ class TestResponses: f"{endpoint} did not forward safety_identifier to Bedrock Converse on every attempt: {capture.bodies}" ) - @pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400") + @pytest.mark.skip( + reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400" + ) @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") def test_missing_input_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = _register(proxy, resources, _openai_params(), prefix="e2e-responses-val") From bdbb4cf5272671f2bac0d02d4ad7b20a45fabee3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:20:06 -0700 Subject: [PATCH 4/4] test(e2e): send no-cache on rerank bodies like the other request models --- tests/e2e/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 066eb74749c..9440505e58c 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -705,6 +705,7 @@ class RerankBody(BaseModel): query: str documents: list[str] top_n: int + cache: dict[str, bool] | None = {"no-cache": True} class RerankItem(BaseModel):