diff --git a/tests/e2e/llm_translation/sdk_clients.py b/tests/e2e/llm_translation/sdk_clients.py index 8a6b187f979..145efbdca98 100644 --- a/tests/e2e/llm_translation/sdk_clients.py +++ b/tests/e2e/llm_translation/sdk_clients.py @@ -13,12 +13,21 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass +from types import MappingProxyType +from typing import Final from anthropic import Anthropic from openai import OpenAI from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT +NO_PROXY_CACHE: Final = MappingProxyType({"cache": {"no-cache": True}}) +"""``extra_body`` for every cacheable SDK call (messages, responses, completions, +embeddings): the gateway under test caches those call types, so an identical +re-send would otherwise be served from Redis instead of reaching the provider, +which hides provider-side behavior such as prompt-cache warm-up. The SDKs +themselves cannot bypass it (``Cache-Control`` only sets a TTL on the proxy).""" + def response_header(headers: Mapping[str, str], name: str) -> str | None: """Typed read of an SDK response header: httpx.Headers.get returns Any and diff --git a/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py b/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py index d44e9ba43f7..b4253a82dd8 100644 --- a/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py @@ -39,7 +39,7 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -83,6 +83,7 @@ class TestBedrockWebSearchServerTool: max_tokens=512, tools=[WEB_SEARCH_TOOL], messages=[{"role": "user", "content": SEARCH_PROMPT}], + extra_body=NO_PROXY_CACHE, ) assert response.content, f"no content blocks in response: {response!r}" diff --git a/tests/e2e/llm_translation/test_completions_endpoint_e2e.py b/tests/e2e/llm_translation/test_completions_endpoint_e2e.py index c6415ce7f29..63fcee3ce36 100644 --- a/tests/e2e/llm_translation/test_completions_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_completions_endpoint_e2e.py @@ -15,7 +15,7 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -40,6 +40,7 @@ class TestCompletionsEndpoint: model=model, prompt="Finish this sentence in a few words: the capital of France is", max_tokens=32, + extra_body=NO_PROXY_CACHE, ) assert completion.choices, f"/v1/completions returned no choices: {completion!r}" text = (completion.choices[0].text or "").strip() diff --git a/tests/e2e/llm_translation/test_credential_messages_e2e.py b/tests/e2e/llm_translation/test_credential_messages_e2e.py index d4bf4566cc0..52306ce3a7a 100644 --- a/tests/e2e/llm_translation/test_credential_messages_e2e.py +++ b/tests/e2e/llm_translation/test_credential_messages_e2e.py @@ -10,16 +10,14 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import CredentialCreateBody, LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e class TestCredentialBackedMessages: @pytest.mark.covers("mgmt.credential.new.serves_request") - def test_credential_backed_messages( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_credential_backed_messages(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: marker = unique_marker() credential_name = f"e2e-cred-{marker}" model = f"e2e-cred-messages-{marker}" @@ -48,6 +46,7 @@ class TestCredentialBackedMessages: model=model, max_tokens=64, messages=[{"role": "user", "content": "reply with one word"}], + extra_body=NO_PROXY_CACHE, ) assert message.role == "assistant", f"unexpected role: {message.role!r}" text = "".join(block.text for block in message.content if block.type == "text") diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index c46b10e25d2..41282260b7e 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -17,7 +17,7 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -39,7 +39,9 @@ def _openai_embeddings_params() -> LiteLLMParamsBody: ) -def _register(proxy: ProxyClient, resources: ResourceManager, prefix: str, params: LiteLLMParamsBody) -> tuple[str, str]: +def _register( + proxy: ProxyClient, resources: ResourceManager, prefix: str, params: LiteLLMParamsBody +) -> tuple[str, str]: model = f"{prefix}-{unique_marker()}" model_id = proxy.create_model(model, params) resources.defer(lambda: proxy.delete_model(model_id)) @@ -56,7 +58,7 @@ def _assert_embedding_vector( model, key = _register(proxy, resources, prefix, params) client = sdk.openai(key) - embeddings = client.embeddings.create(model=model, input="Say this is a test!") + embeddings = client.embeddings.create(model=model, input="Say this is a test!", extra_body=NO_PROXY_CACHE) assert embeddings.data, f"/embeddings returned no data: {embeddings!r}" vector = embeddings.data[0].embedding assert vector, f"/embeddings returned no vector: {embeddings!r}" @@ -66,9 +68,7 @@ def _assert_embedding_vector( class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") - def test_embeddings_returns_vector( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_embeddings_returns_vector(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: _assert_embedding_vector(proxy, resources, sdk, "e2e-embeddings", _openai_embeddings_params()) @pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works") @@ -118,11 +118,11 @@ class TestEmbeddingsEndpoint: @pytest.mark.replayable @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") - def test_array_input_returns_vectors( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_array_input_returns_vectors(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources, "e2e-embeddings-array", _openai_embeddings_params()) - embeddings = sdk.openai(key).embeddings.create(model=model, input=["Hello", "World", "Test"]) + embeddings = sdk.openai(key).embeddings.create( + model=model, input=["Hello", "World", "Test"], extra_body=NO_PROXY_CACHE + ) assert len(embeddings.data) == 3, f"expected 3 vectors: {embeddings!r}" @pytest.mark.replayable diff --git a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py index 527deb8444a..8629cf12013 100644 --- a/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py +++ b/tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py @@ -17,7 +17,7 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -55,9 +55,7 @@ class TestAzureFoundryMessages: return model @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works") - def test_basic_nonstream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_basic_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -65,15 +63,14 @@ class TestAzureFoundryMessages: model=model, max_tokens=64, messages=[{"role": "user", "content": "Reply with one word."}], + extra_body=NO_PROXY_CACHE, ) assert message.content, f"no content blocks in response: {message!r}" text = "".join(block.text for block in message.content if block.type == "text") assert text.strip(), f"/v1/messages returned no text: {message.content!r}" @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works") - def test_basic_stream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_basic_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -82,13 +79,12 @@ class TestAzureFoundryMessages: max_tokens=64, stream=True, messages=[{"role": "user", "content": "Count from one to three."}], + extra_body=NO_PROXY_CACHE, ) _assert_streamed_ok([event.type for event in stream]) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works") - def test_tool_use_nonstream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_tool_use_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -97,6 +93,7 @@ class TestAzureFoundryMessages: max_tokens=256, tools=[WEATHER_TOOL], messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], + extra_body=NO_PROXY_CACHE, ) assert message.content, f"no content blocks in response: {message!r}" assert any(block.type == "tool_use" for block in message.content), ( @@ -104,9 +101,7 @@ class TestAzureFoundryMessages: ) @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works") - def test_tool_use_stream( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_tool_use_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = self._register(proxy, resources) client = sdk.anthropic(resources.key(models=[model])) @@ -116,12 +111,12 @@ class TestAzureFoundryMessages: stream=True, tools=[WEATHER_TOOL], messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}], + extra_body=NO_PROXY_CACHE, ) events: list[RawMessageStreamEvent] = list(stream) event_types = [event.type for event in events] assert event_types, "stream produced no SSE events" assert any( - event.type == "content_block_start" and event.content_block.type == "tool_use" - for event in events + event.type == "content_block_start" and event.content_block.type == "tool_use" for event in events ), "stream carried no tool_use block" assert "message_stop" in event_types, "stream never reached message_stop" diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index b8007d7c475..d048d1343eb 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -36,7 +36,7 @@ from lifecycle import ResourceManager from models import ChatMessage, LiteLLMParamsBody, SpendLogRow from proxy_client import ProxyClient from pydantic import BaseModel, ConfigDict -from sdk_clients import SdkClients, response_header +from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header pytestmark = [pytest.mark.e2e, pytest.mark.replayable] @@ -96,13 +96,13 @@ def _user_turn(text: str) -> MessageParam: class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works") - def test_messages_returns_completion( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_messages_returns_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model, key = _register(proxy, resources) client = sdk.anthropic(key) - message = client.messages.create(model=model, max_tokens=64, messages=[_user_turn("reply with one word")]) + message = client.messages.create( + model=model, max_tokens=64, messages=[_user_turn("reply with one word")], extra_body=NO_PROXY_CACHE + ) assert message.role == "assistant", f"unexpected role: {message.role!r}" assert _text(message).strip(), f"/v1/messages returned no text: {message.content!r}" @@ -117,6 +117,7 @@ class TestAnthropicMessages: model=model, max_tokens=64, messages=[_user_turn(f"reply with one word {unique_marker()}")], + extra_body=NO_PROXY_CACHE, ) message = raw.parse() assert message.role == "assistant" and _text(message).strip(), ( @@ -154,9 +155,7 @@ class TestAnthropicMessages: @pytest.mark.covers("llm.messages.anthropic.basic.stream.works") @pytest.mark.provider_live - def test_messages_streams_completion( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_messages_streams_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: """Edge-wired like its non-streaming siblings, so record and replay both carry the streamed response. @@ -175,6 +174,7 @@ class TestAnthropicMessages: max_tokens=800, stream=True, messages=[_user_turn("Count from 1 to 200, one number per line.")], + extra_body=NO_PROXY_CACHE, ) arrivals: Final = tuple((event, time.monotonic() - started) for event in stream) assert arrivals, "stream produced no SSE events" @@ -221,13 +221,16 @@ class TestAnthropicMessages: max_tokens=256, tools=[WEATHER_TOOL], messages=[_user_turn("What is the weather in Paris? Use the tool.")], + extra_body=NO_PROXY_CACHE, ) assert message.content, f"no content blocks in response: {message!r}" assert any(isinstance(block, ToolUseBlock) for block in message.content), ( f"model did not call the tool: {message.content!r}" ) - @pytest.mark.skip(reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400") + @pytest.mark.skip( + reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400" + ) @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") def test_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) @@ -238,7 +241,9 @@ class TestAnthropicMessages: ) assert_client_error(result, "messages missing messages") - @pytest.mark.skip(reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400") + @pytest.mark.skip( + reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400" + ) @pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works") def test_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model, key = _register(proxy, resources) @@ -292,9 +297,7 @@ def _tool_from_stream(events: tuple[RawMessageStreamEvent, ...]) -> ToolUseBlock terminal_positions: Final = tuple( index for index, event in enumerate(events) if isinstance(event, RawMessageDeltaEvent) ) - stop_reasons: Final = tuple( - event.delta.stop_reason for event in events if isinstance(event, RawMessageDeltaEvent) - ) + stop_reasons: Final = tuple(event.delta.stop_reason for event in events if isinstance(event, RawMessageDeltaEvent)) assert stop_reasons == ("tool_use",) assert len(terminal_positions) == 1 and stops[0] < terminal_positions[0] < len(events) - 1 assert tuple(index for index, event in enumerate(events) if event.type == "message_stop") == (len(events) - 1,), ( @@ -309,12 +312,23 @@ def _request_tool(client: Anthropic, model: str, question: MessageParam, tool: T if stream: events: Final = tuple( client.messages.create( - model=model, max_tokens=2048, messages=[question], tools=[tool], tool_choice=tool_choice, stream=True + model=model, + max_tokens=2048, + messages=[question], + tools=[tool], + tool_choice=tool_choice, + stream=True, + extra_body=NO_PROXY_CACHE, ) ) return _tool_from_stream(events) message: Final = client.messages.create( - model=model, max_tokens=2048, messages=[question], tools=[tool], tool_choice=tool_choice + model=model, + max_tokens=2048, + messages=[question], + tools=[tool], + tool_choice=tool_choice, + extra_body=NO_PROXY_CACHE, ) blocks: Final = tuple(block for block in message.content if isinstance(block, ToolUseBlock)) assert len(blocks) == 1 @@ -367,6 +381,7 @@ class TestOpenAIMessagesToolContinuation: }, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": emitted.id, "content": receipt}]}, ], + extra_body=NO_PROXY_CACHE, ) assert _text(continuation).strip() == receipt, "continuation did not consume the correlated tool result" assert all(not isinstance(block, ToolUseBlock) for block in continuation.content) diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py index 730d19749ba..e9b4b394996 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py @@ -38,7 +38,7 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -63,7 +63,9 @@ def _cacheable_system_block(marker: str) -> TextBlockParam: def _user_turn(text: str, *, cached: bool = False) -> MessageParam: block: TextBlockParam = ( - {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} if cached else {"type": "text", "text": text} + {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} + if cached + else {"type": "text", "text": text} ) return {"role": "user", "content": [block]} @@ -87,7 +89,9 @@ def _text(message: Message) -> str: def _send(client: Anthropic, model: str, system_block: TextBlockParam, messages: Sequence[MessageParam]) -> Message: - return client.messages.create(model=model, max_tokens=64, system=[system_block], messages=messages) + return client.messages.create( + model=model, max_tokens=64, system=[system_block], messages=messages, extra_body=NO_PROXY_CACHE + ) def _register_invoke_deployment(proxy: ProxyClient, resources: ResourceManager, bedrock_model: str) -> str: diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py index e95bc60d7cc..9f5ed8b05da 100644 --- a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -45,7 +45,7 @@ from lifecycle import ResourceManager from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -83,7 +83,9 @@ def _cacheable_system_block(marker: str) -> TextBlockParam: def _user_turn(text: str, *, cached: bool = False) -> MessageParam: block: TextBlockParam = ( - {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} if cached else {"type": "text", "text": text} + {"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} + if cached + else {"type": "text", "text": text} ) return {"role": "user", "content": [block]} @@ -107,7 +109,9 @@ def _text(message: Message) -> str: def _send(client: Anthropic, model: str, system_block: TextBlockParam, messages: Sequence[MessageParam]) -> Message: - return client.messages.create(model=model, max_tokens=64, system=[system_block], messages=messages) + return client.messages.create( + model=model, max_tokens=64, system=[system_block], messages=messages, extra_body=NO_PROXY_CACHE + ) def _register_deployment(proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody) -> str: diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 81314600125..9cc70da63b0 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -9,6 +9,7 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations +import contextlib import json import threading from collections.abc import Mapping @@ -16,6 +17,7 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, cast +import openai import pytest from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker from e2e_http import assert_client_error @@ -31,7 +33,7 @@ from provider_edge import LiveEdge, start_provider_edge from provider_edge_bedrock import bedrock_signer from proxy_client import ProxyClient from pydantic import BaseModel -from sdk_clients import SdkClients +from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -136,7 +138,9 @@ class TestResponses: model = _register(proxy, resources, _openai_params()) client = sdk.openai(resources.key()) - response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.openai.basic.stream.works") @@ -147,7 +151,11 @@ class TestResponses: client = sdk.openai(resources.key()) stream = client.responses.create( - model=model, input="reply with one word", instructions=INSTRUCTIONS, stream=True + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + stream=True, + extra_body=NO_PROXY_CACHE, ) events = tuple(stream) assert events, "responses stream returned no events" @@ -158,14 +166,15 @@ class TestResponses: ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged") - def test_responses_logs_cost( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients - ) -> None: + def test_responses_logs_cost(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None: model = _register(proxy, resources, _openai_params()) client = sdk.openai(resources.key()) raw = client.responses.with_raw_response.create( - model=model, input=f"reply with one word {unique_marker()}", instructions=INSTRUCTIONS + model=model, + input=f"reply with one word {unique_marker()}", + instructions=INSTRUCTIONS, + extra_body=NO_PROXY_CACHE, ) response = raw.parse() assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @@ -193,6 +202,7 @@ class TestResponses: input="What is the weather in San Francisco? Use the get_weather tool.", instructions=INSTRUCTIONS, tools=[WEATHER_TOOL], + extra_body=NO_PROXY_CACHE, ) _assert_weather_call(response) @@ -216,7 +226,9 @@ class TestResponses: ], } ] - response = client.responses.create(model=model, input=vision_input, instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input=vision_input, instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) text = response.output_text.strip().lower() assert text, f"/responses vision returned no output text: {response.output!r}" assert any(keyword in text for keyword in ("cat", "feline")), ( @@ -230,7 +242,9 @@ class TestResponses: model = _register(proxy, resources, _anthropic_params()) client = sdk.openai(resources.key()) - response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") @@ -245,6 +259,7 @@ class TestResponses: input="What is the weather in San Francisco? Use the get_weather tool.", instructions=INSTRUCTIONS, tools=[WEATHER_TOOL], + extra_body=NO_PROXY_CACHE, ) _assert_weather_call(response) @@ -255,7 +270,9 @@ class TestResponses: model = _register(proxy, resources, _bedrock_params()) client = sdk.openai(resources.key()) - response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS) + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) assert response.output_text.strip(), f"/responses over bedrock returned no output text: {response.output!r}" @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") @@ -270,6 +287,7 @@ class TestResponses: input="What is the weather in San Francisco? Use the get_weather tool.", instructions=INSTRUCTIONS, tools=[WEATHER_TOOL], + extra_body=NO_PROXY_CACHE, ) _assert_weather_call(response) @@ -278,10 +296,15 @@ class TestResponses: def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, endpoint: str ) -> None: + """Judges the Converse bodies the edge captured, not the reply: Claude on + Bedrock rejects the forwarded field with a 400, which the chat leg's + ``Result`` carries as a value and the OpenAI SDK raises.""" capture: Final = ConverseRequestCapture() edge: Final = start_provider_edge( LiveEdge(observe_request=capture.observe, sign=bedrock_signer(BEDROCK_EDGE_REGION)), - mounts=MappingProxyType({BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"}), + mounts=MappingProxyType( + {BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"} + ), bind_host=PROVIDER_EDGE_BIND_HOST, advertise_host=PROVIDER_EDGE_ADVERTISE_HOST, ) @@ -303,12 +326,14 @@ class TestResponses: safety_identifier: Final = f"end-user-{unique_marker()}" if endpoint == "/v1/responses": - sdk.openai(key).responses.create( - model=model, - input="reply with one word", - instructions=INSTRUCTIONS, - safety_identifier=safety_identifier, - ) + with contextlib.suppress(openai.BadRequestError): + sdk.openai(key).responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + safety_identifier=safety_identifier, + extra_body=NO_PROXY_CACHE, + ) else: proxy.chat( key, @@ -325,7 +350,9 @@ class TestResponses: f"{endpoint} did not forward safety_identifier to Bedrock Converse on every attempt: {capture.bodies}" ) - @pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400") + @pytest.mark.skip( + reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400" + ) @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") def test_missing_input_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = _register(proxy, resources, _openai_params(), prefix="e2e-responses-val")