scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

test(e2e): send no-cache on cacheable SDK calls and accept Bedrock's 400 on the responses leg

Browse source 
The deleted wrapper put cache: {"no-cache": true} on every request body, so the
gateway's response cache never answered a re-sent prompt. The SDKs send nothing
of the sort, and the mid-conversation prompt-cache priming loop re-sends an
identical body until the provider reports a warm cache, which a cached reply
never does. NO_PROXY_CACHE in sdk_clients.py restores the field as extra_body
on every messages, responses, completions, and embeddings call.

The wrapper also returned a 4xx as a value where the SDKs raise. The Bedrock
safety_identifier test judges the captured Converse body, and Claude on Bedrock
rejects the forwarded field with a 400, so the /v1/responses leg now suppresses
openai.BadRequestError the way the chat leg carries the same 400 as a Result.
This commit is contained in:
mateo-berri 2026-09-21 12:39:16 -07:00
parent 4c50710c02
commit 057c45f23f
10 changed files with 125 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
tests/e2e/llm_translation/sdk_clients.py
View file

@ -13,12 +13,21 @@ from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from anthropic import Anthropic
from openai import OpenAI
from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT
NO_PROXY_CACHE: Final = MappingProxyType({"cache": {"no-cache": True}})
"""``extra_body`` for every cacheable SDK call (messages, responses, completions,
embeddings): the gateway under test caches those call types, so an identical
re-send would otherwise be served from Redis instead of reaching the provider,
which hides provider-side behavior such as prompt-cache warm-up. The SDKs
themselves cannot bypass it (``Cache-Control`` only sets a TTL on the proxy)."""
def response_header(headers: Mapping[str, str], name: str) -> str | None:
"""Typed read of an SDK response header: httpx.Headers.get returns Any and

3
tests/e2e/llm_translation/test_bedrock_web_search_server_tool_e2e.py
View file

@ -39,7 +39,7 @@ from e2e_config import unique_marker
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -83,6 +83,7 @@ class TestBedrockWebSearchServerTool:
max_tokens=512,
tools=[WEB_SEARCH_TOOL],
messages=[{"role": "user", "content": SEARCH_PROMPT}],
extra_body=NO_PROXY_CACHE,
)
assert response.content, f"no content blocks in response: {response!r}"

3
tests/e2e/llm_translation/test_completions_endpoint_e2e.py
View file

@ -15,7 +15,7 @@ from e2e_config import unique_marker
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -40,6 +40,7 @@ class TestCompletionsEndpoint:
model=model,
prompt="Finish this sentence in a few words: the capital of France is",
max_tokens=32,
extra_body=NO_PROXY_CACHE,
)
assert completion.choices, f"/v1/completions returned no choices: {completion!r}"
text = (completion.choices[0].text or "").strip()

7
tests/e2e/llm_translation/test_credential_messages_e2e.py
View file

@ -10,16 +10,14 @@ from e2e_config import unique_marker
from lifecycle import ResourceManager
from models import CredentialCreateBody, LiteLLMParamsBody
from proxy_client import ProxyClient
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
class TestCredentialBackedMessages:
@pytest.mark.covers("mgmt.credential.new.serves_request")
def test_credential_backed_messages(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_credential_backed_messages(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
marker = unique_marker()
credential_name = f"e2e-cred-{marker}"
model = f"e2e-cred-messages-{marker}"
@ -48,6 +46,7 @@ class TestCredentialBackedMessages:
model=model,
max_tokens=64,
messages=[{"role": "user", "content": "reply with one word"}],
extra_body=NO_PROXY_CACHE,
)
assert message.role == "assistant", f"unexpected role: {message.role!r}"
text = "".join(block.text for block in message.content if block.type == "text")

20
tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py
View file

@ -17,7 +17,7 @@ from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -39,7 +39,9 @@ def _openai_embeddings_params() -> LiteLLMParamsBody:
)
def _register(proxy: ProxyClient, resources: ResourceManager, prefix: str, params: LiteLLMParamsBody) -> tuple[str, str]:
def _register(
proxy: ProxyClient, resources: ResourceManager, prefix: str, params: LiteLLMParamsBody
) -> tuple[str, str]:
model = f"{prefix}-{unique_marker()}"
model_id = proxy.create_model(model, params)
resources.defer(lambda: proxy.delete_model(model_id))
@ -56,7 +58,7 @@ def _assert_embedding_vector(
model, key = _register(proxy, resources, prefix, params)
client = sdk.openai(key)
embeddings = client.embeddings.create(model=model, input="Say this is a test!")
embeddings = client.embeddings.create(model=model, input="Say this is a test!", extra_body=NO_PROXY_CACHE)
assert embeddings.data, f"/embeddings returned no data: {embeddings!r}"
vector = embeddings.data[0].embedding
assert vector, f"/embeddings returned no vector: {embeddings!r}"
@ -66,9 +68,7 @@ def _assert_embedding_vector(
class TestEmbeddingsEndpoint:
@pytest.mark.replayable
@pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works")
def test_embeddings_returns_vector(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_embeddings_returns_vector(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
_assert_embedding_vector(proxy, resources, sdk, "e2e-embeddings", _openai_embeddings_params())
@pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works")
@ -118,11 +118,11 @@ class TestEmbeddingsEndpoint:
@pytest.mark.replayable
@pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works")
def test_array_input_returns_vectors(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_array_input_returns_vectors(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model, key = _register(proxy, resources, "e2e-embeddings-array", _openai_embeddings_params())
embeddings = sdk.openai(key).embeddings.create(model=model, input=["Hello", "World", "Test"])
embeddings = sdk.openai(key).embeddings.create(
model=model, input=["Hello", "World", "Test"], extra_body=NO_PROXY_CACHE
)
assert len(embeddings.data) == 3, f"expected 3 vectors: {embeddings!r}"
@pytest.mark.replayable

25
tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py
View file

@ -17,7 +17,7 @@ from e2e_config import unique_marker
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -55,9 +55,7 @@ class TestAzureFoundryMessages:
return model
@pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works")
def test_basic_nonstream(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_basic_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model = self._register(proxy, resources)
client = sdk.anthropic(resources.key(models=[model]))
@ -65,15 +63,14 @@ class TestAzureFoundryMessages:
model=model,
max_tokens=64,
messages=[{"role": "user", "content": "Reply with one word."}],
extra_body=NO_PROXY_CACHE,
)
assert message.content, f"no content blocks in response: {message!r}"
text = "".join(block.text for block in message.content if block.type == "text")
assert text.strip(), f"/v1/messages returned no text: {message.content!r}"
@pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works")
def test_basic_stream(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_basic_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model = self._register(proxy, resources)
client = sdk.anthropic(resources.key(models=[model]))
@ -82,13 +79,12 @@ class TestAzureFoundryMessages:
max_tokens=64,
stream=True,
messages=[{"role": "user", "content": "Count from one to three."}],
extra_body=NO_PROXY_CACHE,
)
_assert_streamed_ok([event.type for event in stream])
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works")
def test_tool_use_nonstream(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_tool_use_nonstream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model = self._register(proxy, resources)
client = sdk.anthropic(resources.key(models=[model]))
@ -97,6 +93,7 @@ class TestAzureFoundryMessages:
max_tokens=256,
tools=[WEATHER_TOOL],
messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}],
extra_body=NO_PROXY_CACHE,
)
assert message.content, f"no content blocks in response: {message!r}"
assert any(block.type == "tool_use" for block in message.content), (
@ -104,9 +101,7 @@ class TestAzureFoundryMessages:
)
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works")
def test_tool_use_stream(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_tool_use_stream(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model = self._register(proxy, resources)
client = sdk.anthropic(resources.key(models=[model]))
@ -116,12 +111,12 @@ class TestAzureFoundryMessages:
stream=True,
tools=[WEATHER_TOOL],
messages=[{"role": "user", "content": "What is the weather in Paris? Use the tool."}],
extra_body=NO_PROXY_CACHE,
)
events: list[RawMessageStreamEvent] = list(stream)
event_types = [event.type for event in events]
assert event_types, "stream produced no SSE events"
assert any(
event.type == "content_block_start" and event.content_block.type == "tool_use"
for event in events
event.type == "content_block_start" and event.content_block.type == "tool_use" for event in events
), "stream carried no tool_use block"
assert "message_stop" in event_types, "stream never reached message_stop"

45
tests/e2e/llm_translation/test_messages_e2e.py
View file

@ -36,7 +36,7 @@ from lifecycle import ResourceManager
from models import ChatMessage, LiteLLMParamsBody, SpendLogRow
from proxy_client import ProxyClient
from pydantic import BaseModel, ConfigDict
from sdk_clients import SdkClients, response_header
from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header
pytestmark = [pytest.mark.e2e, pytest.mark.replayable]
@ -96,13 +96,13 @@ def _user_turn(text: str) -> MessageParam:
class TestAnthropicMessages:
@pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works")
def test_messages_returns_completion(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_messages_returns_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model, key = _register(proxy, resources)
client = sdk.anthropic(key)
message = client.messages.create(model=model, max_tokens=64, messages=[_user_turn("reply with one word")])
message = client.messages.create(
model=model, max_tokens=64, messages=[_user_turn("reply with one word")], extra_body=NO_PROXY_CACHE
)
assert message.role == "assistant", f"unexpected role: {message.role!r}"
assert _text(message).strip(), f"/v1/messages returned no text: {message.content!r}"
@ -117,6 +117,7 @@ class TestAnthropicMessages:
model=model,
max_tokens=64,
messages=[_user_turn(f"reply with one word {unique_marker()}")],
extra_body=NO_PROXY_CACHE,
)
message = raw.parse()
assert message.role == "assistant" and _text(message).strip(), (
@ -154,9 +155,7 @@ class TestAnthropicMessages:
@pytest.mark.covers("llm.messages.anthropic.basic.stream.works")
@pytest.mark.provider_live
def test_messages_streams_completion(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_messages_streams_completion(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
"""Edge-wired like its non-streaming siblings, so record and replay both
carry the streamed response.
@ -175,6 +174,7 @@ class TestAnthropicMessages:
max_tokens=800,
stream=True,
messages=[_user_turn("Count from 1 to 200, one number per line.")],
extra_body=NO_PROXY_CACHE,
)
arrivals: Final = tuple((event, time.monotonic() - started) for event in stream)
assert arrivals, "stream produced no SSE events"
@ -221,13 +221,16 @@ class TestAnthropicMessages:
max_tokens=256,
tools=[WEATHER_TOOL],
messages=[_user_turn("What is the weather in Paris? Use the tool.")],
extra_body=NO_PROXY_CACHE,
)
assert message.content, f"no content blocks in response: {message!r}"
assert any(isinstance(block, ToolUseBlock) for block in message.content), (
f"model did not call the tool: {message.content!r}"
)
@pytest.mark.skip(reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400")
@pytest.mark.skip(
reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing messages instead of 400"
)
@pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works")
def test_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
model, key = _register(proxy, resources)
@ -238,7 +241,9 @@ class TestAnthropicMessages:
)
assert_client_error(result, "messages missing messages")
@pytest.mark.skip(reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400")
@pytest.mark.skip(
reason="stage red: product gap, /v1/messages 500s (anthropic_messages TypeError) on missing max_tokens instead of 400"
)
@pytest.mark.covers("llm.messages.anthropic.input_validation.nonstream.works")
def test_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
model, key = _register(proxy, resources)
@ -292,9 +297,7 @@ def _tool_from_stream(events: tuple[RawMessageStreamEvent, ...]) -> ToolUseBlock
terminal_positions: Final = tuple(
index for index, event in enumerate(events) if isinstance(event, RawMessageDeltaEvent)
)
stop_reasons: Final = tuple(
event.delta.stop_reason for event in events if isinstance(event, RawMessageDeltaEvent)
)
stop_reasons: Final = tuple(event.delta.stop_reason for event in events if isinstance(event, RawMessageDeltaEvent))
assert stop_reasons == ("tool_use",)
assert len(terminal_positions) == 1 and stops[0] < terminal_positions[0] < len(events) - 1
assert tuple(index for index, event in enumerate(events) if event.type == "message_stop") == (len(events) - 1,), (
@ -309,12 +312,23 @@ def _request_tool(client: Anthropic, model: str, question: MessageParam, tool: T
if stream:
events: Final = tuple(
client.messages.create(
model=model, max_tokens=2048, messages=[question], tools=[tool], tool_choice=tool_choice, stream=True
model=model,
max_tokens=2048,
messages=[question],
tools=[tool],
tool_choice=tool_choice,
stream=True,
extra_body=NO_PROXY_CACHE,
)
)
return _tool_from_stream(events)
message: Final = client.messages.create(
model=model, max_tokens=2048, messages=[question], tools=[tool], tool_choice=tool_choice
model=model,
max_tokens=2048,
messages=[question],
tools=[tool],
tool_choice=tool_choice,
extra_body=NO_PROXY_CACHE,
)
blocks: Final = tuple(block for block in message.content if isinstance(block, ToolUseBlock))
assert len(blocks) == 1
@ -367,6 +381,7 @@ class TestOpenAIMessagesToolContinuation:
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": emitted.id, "content": receipt}]},
],
extra_body=NO_PROXY_CACHE,
)
assert _text(continuation).strip() == receipt, "continuation did not consume the correlated tool result"
assert all(not isinstance(block, ToolUseBlock) for block in continuation.content)

10
tests/e2e/llm_translation/test_messages_mid_conversation_system_e2e.py
View file

@ -38,7 +38,7 @@ from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -63,7 +63,9 @@ def _cacheable_system_block(marker: str) -> TextBlockParam:
def _user_turn(text: str, *, cached: bool = False) -> MessageParam:
block: TextBlockParam = (
{"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} if cached else {"type": "text", "text": text}
{"type": "text", "text": text, "cache_control": {"type": "ephemeral"}}
if cached
else {"type": "text", "text": text}
)
return {"role": "user", "content": [block]}
@ -87,7 +89,9 @@ def _text(message: Message) -> str:
def _send(client: Anthropic, model: str, system_block: TextBlockParam, messages: Sequence[MessageParam]) -> Message:
return client.messages.create(model=model, max_tokens=64, system=[system_block], messages=messages)
return client.messages.create(
model=model, max_tokens=64, system=[system_block], messages=messages, extra_body=NO_PROXY_CACHE
)
def _register_invoke_deployment(proxy: ProxyClient, resources: ResourceManager, bedrock_model: str) -> str:

10
tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py
View file

@ -45,7 +45,7 @@ from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -83,7 +83,9 @@ def _cacheable_system_block(marker: str) -> TextBlockParam:
def _user_turn(text: str, *, cached: bool = False) -> MessageParam:
block: TextBlockParam = (
{"type": "text", "text": text, "cache_control": {"type": "ephemeral"}} if cached else {"type": "text", "text": text}
{"type": "text", "text": text, "cache_control": {"type": "ephemeral"}}
if cached
else {"type": "text", "text": text}
)
return {"role": "user", "content": [block]}
@ -107,7 +109,9 @@ def _text(message: Message) -> str:
def _send(client: Anthropic, model: str, system_block: TextBlockParam, messages: Sequence[MessageParam]) -> Message:
return client.messages.create(model=model, max_tokens=64, system=[system_block], messages=messages)
return client.messages.create(
model=model, max_tokens=64, system=[system_block], messages=messages, extra_body=NO_PROXY_CACHE
)
def _register_deployment(proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody) -> str:

63
tests/e2e/llm_translation/test_responses_e2e.py
View file

@ -9,6 +9,7 @@ litellm-regression-tests/tests/test_inference_endpoints.py.
from __future__ import annotations
import contextlib
import json
import threading
from collections.abc import Mapping
@ -16,6 +17,7 @@ from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, cast
import openai
import pytest
from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker
from e2e_http import assert_client_error
@ -31,7 +33,7 @@ from provider_edge import LiveEdge, start_provider_edge
from provider_edge_bedrock import bedrock_signer
from proxy_client import ProxyClient
from pydantic import BaseModel
from sdk_clients import SdkClients
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -136,7 +138,9 @@ class TestResponses:
model = _register(proxy, resources, _openai_params())
client = sdk.openai(resources.key())
response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS)
response = client.responses.create(
model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE
)
assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}"
@pytest.mark.covers("llm.responses.openai.basic.stream.works")
@ -147,7 +151,11 @@ class TestResponses:
client = sdk.openai(resources.key())
stream = client.responses.create(
model=model, input="reply with one word", instructions=INSTRUCTIONS, stream=True
model=model,
input="reply with one word",
instructions=INSTRUCTIONS,
stream=True,
extra_body=NO_PROXY_CACHE,
)
events = tuple(stream)
assert events, "responses stream returned no events"
@ -158,14 +166,15 @@ class TestResponses:
)
@pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged")
def test_responses_logs_cost(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
def test_responses_logs_cost(self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients) -> None:
model = _register(proxy, resources, _openai_params())
client = sdk.openai(resources.key())
raw = client.responses.with_raw_response.create(
model=model, input=f"reply with one word {unique_marker()}", instructions=INSTRUCTIONS
model=model,
input=f"reply with one word {unique_marker()}",
instructions=INSTRUCTIONS,
extra_body=NO_PROXY_CACHE,
)
response = raw.parse()
assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}"
@ -193,6 +202,7 @@ class TestResponses:
input="What is the weather in San Francisco? Use the get_weather tool.",
instructions=INSTRUCTIONS,
tools=[WEATHER_TOOL],
extra_body=NO_PROXY_CACHE,
)
_assert_weather_call(response)
@ -216,7 +226,9 @@ class TestResponses:
],
}
]
response = client.responses.create(model=model, input=vision_input, instructions=INSTRUCTIONS)
response = client.responses.create(
model=model, input=vision_input, instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE
)
text = response.output_text.strip().lower()
assert text, f"/responses vision returned no output text: {response.output!r}"
assert any(keyword in text for keyword in ("cat", "feline")), (
@ -230,7 +242,9 @@ class TestResponses:
model = _register(proxy, resources, _anthropic_params())
client = sdk.openai(resources.key())
response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS)
response = client.responses.create(
model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE
)
assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}"
@pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works")
@ -245,6 +259,7 @@ class TestResponses:
input="What is the weather in San Francisco? Use the get_weather tool.",
instructions=INSTRUCTIONS,
tools=[WEATHER_TOOL],
extra_body=NO_PROXY_CACHE,
)
_assert_weather_call(response)
@ -255,7 +270,9 @@ class TestResponses:
model = _register(proxy, resources, _bedrock_params())
client = sdk.openai(resources.key())
response = client.responses.create(model=model, input="reply with one word", instructions=INSTRUCTIONS)
response = client.responses.create(
model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE
)
assert response.output_text.strip(), f"/responses over bedrock returned no output text: {response.output!r}"
@pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
@ -270,6 +287,7 @@ class TestResponses:
input="What is the weather in San Francisco? Use the get_weather tool.",
instructions=INSTRUCTIONS,
tools=[WEATHER_TOOL],
extra_body=NO_PROXY_CACHE,
)
_assert_weather_call(response)
@ -278,10 +296,15 @@ class TestResponses:
def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, endpoint: str
) -> None:
"""Judges the Converse bodies the edge captured, not the reply: Claude on
Bedrock rejects the forwarded field with a 400, which the chat leg's
``Result`` carries as a value and the OpenAI SDK raises."""
capture: Final = ConverseRequestCapture()
edge: Final = start_provider_edge(
LiveEdge(observe_request=capture.observe, sign=bedrock_signer(BEDROCK_EDGE_REGION)),
mounts=MappingProxyType({BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"}),
mounts=MappingProxyType(
{BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"}
),
bind_host=PROVIDER_EDGE_BIND_HOST,
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
)
@ -303,12 +326,14 @@ class TestResponses:
safety_identifier: Final = f"end-user-{unique_marker()}"
if endpoint == "/v1/responses":
sdk.openai(key).responses.create(
model=model,
input="reply with one word",
instructions=INSTRUCTIONS,
safety_identifier=safety_identifier,
)
with contextlib.suppress(openai.BadRequestError):
sdk.openai(key).responses.create(
model=model,
input="reply with one word",
instructions=INSTRUCTIONS,
safety_identifier=safety_identifier,
extra_body=NO_PROXY_CACHE,
)
else:
proxy.chat(
key,
@ -325,7 +350,9 @@ class TestResponses:
f"{endpoint} did not forward safety_identifier to Bedrock Converse on every attempt: {capture.bodies}"
)
@pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400")
@pytest.mark.skip(
reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400"
)
@pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works")
def test_missing_input_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
model = _register(proxy, resources, _openai_params(), prefix="e2e-responses-val")