mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
test(rust/messages): pin the remaining python messages behaviors on the native route
Ports the anthropic provider /v1/messages python unit tests that had no rust twin yet: the max effort tier on a 4.6 model, 401 and 403 provider errors with their failure logging payload, safeguards and safeguard_results round trips, the body field allowlist with None values dropped, the per-turn-control beta reaching the wire, the auto-summary env var in shaping, and idempotent flattening of failed web searches Malformed content that Anthropic rejects (a bare string message or non-string text) is pinned as a local 400 with no provider call
This commit is contained in:
parent
02fd287a0b
commit
0861f60c34
5 changed files with 140 additions and 14 deletions
|
|
@ -1224,6 +1224,7 @@ mod tests {
|
|||
#[rstest]
|
||||
#[case::with_results(json!([{"type": "web_search_result", "url": "u", "title": "Rome", "snippet": "s", "page_age": null}]))]
|
||||
#[case::without_results(json!([]))]
|
||||
#[case::failed_search(json!({"type": "web_search_tool_result_error", "error_code": "max_uses_exceeded"}))]
|
||||
fn flatten_unencrypted_web_search_results_is_idempotent(#[case] results: Value) {
|
||||
let input = replayed_search_turn(results);
|
||||
let once = apply(flatten_unencrypted_web_search_results, input.clone());
|
||||
|
|
|
|||
|
|
@ -476,6 +476,7 @@ mod tests {
|
|||
#[case::max(opus_4_7(), "max", "max")]
|
||||
#[case::minimal_maps_to_low_on_4_6(sonnet_4_6(), "minimal", "low")]
|
||||
#[case::low_on_4_6(sonnet_4_6(), "low", "low")]
|
||||
#[case::max_on_4_6(sonnet_4_6(), "max", "max")]
|
||||
#[case::max_without_max_tier_is_allowed_on_adaptive_models(newfamily_6(), "max", "max")]
|
||||
fn reasoning_effort_on_adaptive_model_becomes_summarized_adaptive_thinking_and_effort(
|
||||
#[case] capabilities: AnthropicModelCapabilities,
|
||||
|
|
|
|||
|
|
@ -71,6 +71,25 @@ def test_drop_params_merges_the_global_flag_with_the_request(
|
|||
assert route_host.shaping("anthropic/not-a-real-model", None, kwargs)["drop_params"] is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("global_flag", "env", "expected"),
|
||||
[
|
||||
(False, "false", False),
|
||||
(True, "false", True),
|
||||
(False, "true", True),
|
||||
(False, "TRUE", True),
|
||||
(False, "1", False),
|
||||
],
|
||||
)
|
||||
def test_reasoning_auto_summary_follows_the_global_flag_or_the_env_var(
|
||||
monkeypatch: pytest.MonkeyPatch, global_flag: bool, env: str, expected: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "reasoning_auto_summary", global_flag)
|
||||
monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", env)
|
||||
|
||||
assert route_host.shaping("anthropic/not-a-real-model", None, {})["reasoning_auto_summary"] is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -89,27 +89,45 @@ async def test_native_messages_pre_call_body_edit_reaches_the_provider(messages_
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("status", "error_type", "expected"),
|
||||
[
|
||||
(400, "invalid_request_error", litellm.BadRequestError),
|
||||
(401, "authentication_error", litellm.AuthenticationError),
|
||||
(403, "permission_error", litellm.PermissionDeniedError),
|
||||
],
|
||||
)
|
||||
async def test_native_messages_provider_error_reaches_caller_and_failure_callbacks_as_one_public_error(
|
||||
messages_server: RecordingServer,
|
||||
messages_server: RecordingServer, status: int, error_type: str, expected: type[litellm.APIError]
|
||||
) -> None:
|
||||
messages_server.enqueue(
|
||||
ResponseSpec(body={"type": "error", "error": {"type": "invalid_request_error", "message": "bad"}}, status=400)
|
||||
ResponseSpec(
|
||||
body={"type": "error", "error": {"type": error_type, "message": "rejected upstream"}}, status=status
|
||||
)
|
||||
)
|
||||
observed: Final = []
|
||||
|
||||
class Observe(CustomLogger):
|
||||
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
observed.append(("sync", kwargs["exception"]))
|
||||
observed.append(("sync", kwargs["exception"], kwargs["standard_logging_object"]["error_information"]))
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
observed.append(("async", kwargs["exception"]))
|
||||
observed.append(("async", kwargs["exception"], kwargs["standard_logging_object"]["error_information"]))
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
with pytest.raises(expected) as raised:
|
||||
await litellm.anthropic.messages.acreate(**arguments(messages_server, callbacks=[Observe()]))
|
||||
|
||||
assert_served_natively(messages_server)
|
||||
assert [phase for phase, _ in observed] == ["sync", "async"]
|
||||
assert all(error is raised.value for _, error in observed)
|
||||
assert raised.value.status_code == status
|
||||
assert raised.value.llm_provider == "anthropic"
|
||||
assert "AnthropicException" in raised.value.message
|
||||
assert f'"{error_type}"' in raised.value.message
|
||||
assert [phase for phase, _, _ in observed] == ["sync", "async"]
|
||||
assert all(error is raised.value for _, error, _ in observed)
|
||||
assert all(
|
||||
(info["error_class"], info["llm_provider"], info["error_code"]) == (expected.__name__, "anthropic", str(status))
|
||||
for _, _, info in observed
|
||||
)
|
||||
|
||||
|
||||
def sse_payload() -> bytes:
|
||||
|
|
|
|||
|
|
@ -141,17 +141,39 @@ async def test_replayed_history_is_sanitized_before_it_reaches_the_provider(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feature_betas_merge_into_the_forwarded_beta_header(messages_server: RecordingServer) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("feature", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
{"output_format": {"type": "json_schema", "schema": {"type": "object"}}},
|
||||
"structured-outputs-2025-11-13,web-search-2025-03-05",
|
||||
id="structured_output",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"messages": [
|
||||
*MESSAGES,
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "# Environment"}],
|
||||
"output_config": {"effort": "high"},
|
||||
},
|
||||
]
|
||||
},
|
||||
"per-turn-control-2026-07-01,web-search-2025-03-05",
|
||||
id="per_turn_control",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_feature_betas_merge_into_the_forwarded_beta_header(
|
||||
messages_server: RecordingServer, feature: dict[str, object], expected: str
|
||||
) -> None:
|
||||
await litellm.anthropic.messages.acreate(
|
||||
**arguments(
|
||||
messages_server,
|
||||
output_format={"type": "json_schema", "schema": {"type": "object"}},
|
||||
extra_headers={"anthropic-beta": "web-search-2025-03-05"},
|
||||
)
|
||||
**arguments(messages_server, extra_headers={"anthropic-beta": "web-search-2025-03-05"}, **feature)
|
||||
)
|
||||
|
||||
_, headers = sent(messages_server)
|
||||
assert headers["anthropic-beta"] == "structured-outputs-2025-11-13,web-search-2025-03-05"
|
||||
assert headers["anthropic-beta"] == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -235,3 +257,68 @@ async def test_non_string_metadata_user_id_is_rejected_before_the_provider_call(
|
|||
await litellm.anthropic.messages.acreate(**arguments(messages_server, metadata={"user_id": 123}))
|
||||
|
||||
assert messages_server.requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_known_request_fields_with_a_value_reach_the_wire(messages_server: RecordingServer) -> None:
|
||||
await litellm.anthropic.messages.acreate(
|
||||
**arguments(
|
||||
messages_server,
|
||||
system="be brief",
|
||||
stop_sequences=["###"],
|
||||
temperature=None,
|
||||
top_p=None,
|
||||
not_a_messages_field="dropped",
|
||||
)
|
||||
)
|
||||
|
||||
body, _ = sent(messages_server)
|
||||
assert body == {
|
||||
"model": "claude-sonnet-5",
|
||||
"messages": list(MESSAGES),
|
||||
"max_tokens": 8192,
|
||||
"system": "be brief",
|
||||
"stop_sequences": ["###"],
|
||||
"stream": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_safeguards_and_client_betas_reach_anthropic_and_safeguard_results_reach_the_caller(
|
||||
messages_server: RecordingServer,
|
||||
) -> None:
|
||||
safeguards: Final = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]
|
||||
safeguard_results: Final = [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": {}}}]
|
||||
messages_server.enqueue(ResponseSpec(body={**MESSAGES_RESPONSE, "safeguard_results": safeguard_results}))
|
||||
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
**arguments(
|
||||
messages_server,
|
||||
safeguards=safeguards,
|
||||
extra_headers={"anthropic-beta": "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"},
|
||||
)
|
||||
)
|
||||
|
||||
body, headers = sent(messages_server)
|
||||
assert body["safeguards"] == safeguards
|
||||
assert headers["anthropic-beta"] == "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14"
|
||||
assert response["safeguard_results"] == safeguard_results
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"messages",
|
||||
[
|
||||
pytest.param(["not a message", {"role": "user", "content": "hi"}], id="bare_string_message"),
|
||||
pytest.param([{"role": "user", "content": [{"type": "text", "text": 123}]}], id="non_string_text"),
|
||||
],
|
||||
)
|
||||
async def test_messages_anthropic_would_reject_are_a_bad_request_without_a_provider_call(
|
||||
messages_server: RecordingServer, messages: list[object]
|
||||
) -> None:
|
||||
messages_server.expected_requests = 0
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await litellm.anthropic.messages.acreate(**arguments(messages_server, messages=messages))
|
||||
|
||||
assert messages_server.requests == []
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue