scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-26 01:12:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 98

test(integration): cover customer reported cache key, cache_control, bedrock request id, responses schema, scim and tag budget contracts (#42785)

Browse source 
* test(integration): cover prompt cache key, system cache_control, bedrock request id, responses schema, scim and tag budget contracts

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): drop invented instructions shape and unused imports

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): register scim placeholder cleanup before asserting the patch

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-09-23 13:19:38 -07:00 • committed by GitHub
parent 5db8543817
commit 35225709ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 400 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
tests/integration/compatibility/test_responses_openapi_schema.py Normal file
View file

@ -0,0 +1,21 @@
import pytest
from integration._support.client import Gateway, object_value
from pydantic import JsonValue
def _assert_responses_post_is_documented(openapi: dict[str, JsonValue]) -> None:
post: dict[str, JsonValue] = object_value(object_value(object_value(openapi["paths"])["/v1/responses"])["post"])
body: dict[str, JsonValue] = object_value(post["requestBody"])
schema: dict[str, JsonValue] = object_value(
object_value(object_value(body["content"])["application/json"])["schema"]
)
properties: dict[str, JsonValue] = object_value(schema.get("properties"))
assert "model" in properties and "input" in properties, schema
ok: dict[str, JsonValue] = object_value(object_value(object_value(post)["responses"])["200"])
assert "schema" in object_value(object_value(ok["content"])["application/json"]), ok
def test_v1_responses_post_declares_a_request_body_and_response_schema(gateway: Gateway) -> None:
pytest.skip("BUG: POST /v1/responses takes a raw Request, so /openapi.json documents no body or response schema")
openapi: dict[str, JsonValue] = gateway.get("/openapi.json")
_assert_responses_post_is_documented(openapi)

30
tests/integration/management/test_scim_group_member_not_yet_provisioned.py Normal file
View file

@ -0,0 +1,30 @@
import uuid
from typing import Final
from integration._support.client import Gateway, object_value, string_value
from pydantic import JsonValue
def test_scim_group_patch_add_member_provisions_the_missing_user(gateway: Gateway) -> None:
missing_user: Final = f"scim-pending-{uuid.uuid4().hex}"
with gateway.scenario() as scenario:
team: Final = scenario.team()
response: Final = gateway.request(
"PATCH",
f"/scim/v2/Groups/{team}",
{
"schemas": ["urn:ietf:params:scim:api:messages:2.0:PatchOp"],
"Operations": [
{"op": "add", "path": "members", "value": [{"value": missing_user}]}
],
},
)
scenario.cleanups.callback(scenario.delete_user, missing_user)
assert response.status_code == 200, response.text
team_info: dict[str, JsonValue] = gateway.get("/team/info", {"team_id": team})
members: Final = object_value(team_info["team_info"]).get("members_with_roles") or []
member_ids: Final = [
string_value(object_value(member)["user_id"]) for member in members if isinstance(member, dict)
]
assert missing_user in member_ids, members

56
tests/integration/observability/test_bedrock_error_request_id.py Normal file
View file

@ -0,0 +1,56 @@
import json
import uuid
from typing import Final
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
_MODEL: Final = "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0"
_TOKEN: Final = "synthetic-bedrock-bearer"
def test_bedrock_500_keeps_amzn_request_id_on_error_headers_and_failure_log(gateway: Gateway) -> None:
identity: Final = f"bedrock-request-id-{uuid.uuid4().hex}"
amzn_request_id: Final = str(uuid.uuid4())
prompt: Final = f"failure probe {identity}"
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/model/anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse", request.target
return Reply(
status=500,
headers={"x-amzn-RequestId": amzn_request_id},
body=b'{"message":"synthetic bedrock failure"}',
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=_MODEL,
api_key=_TOKEN,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
num_retries=0,
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": prompt}]},
)
assert response.status_code >= 400, response.text
assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers)
call_id: Final = response.headers["x-litellm-call-id"]
assert len(wire.drain()) == 1
rows: Final = eventually(
lambda: read_rows(
'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)
),
lambda values: len(values) == 1,
seconds=70,
)
row: Final = rows[0]
assert row["status"] == "failure", row
metadata: Final = row["metadata"]
parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata)
error_information: Final = object_value(parsed["error_information"])
assert error_information["error_provider_request_id"] == amzn_request_id, error_information

105
tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py Normal file
View file

@ -0,0 +1,105 @@
import json
import uuid
from typing import Final
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "gpt-5.4-mini"
_API_KEY: Final = "synthetic-openai-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _claude_code_user_id(device_id: str, session_id: str) -> str:
return json.dumps({"device_id": device_id, "account_uuid": "", "session_id": session_id})
def _responses_reply(identity: str) -> bytes:
return json.dumps(
{
"id": f"resp_{identity}",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": _BACKEND,
"output": [
{
"type": "message",
"id": f"msg_{identity}",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
],
"usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12},
}
).encode()
def test_prompt_cache_key_is_derived_from_claude_code_session_id_not_device_id(gateway: Gateway) -> None:
identity: Final = f"claude-code-cache-key-{uuid.uuid4().hex}"
device_one: Final = "a" * 64
device_two: Final = "b" * 64
session_one: Final = str(uuid.uuid4())
session_two: Final = str(uuid.uuid4())
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/responses"
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
return Reply(body=_responses_reply(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
def send(user_id: str, probe: str) -> None:
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 16,
"metadata": {"user_id": user_id},
"messages": [{"role": "user", "content": probe}],
},
)
assert response.status_code == 200, response.text
send(_claude_code_user_id(device_one, session_one), f"probe one {identity}")
send(_claude_code_user_id(device_one, session_two), f"probe two {identity}")
send(_claude_code_user_id(device_two, session_two), f"probe three {identity}")
keys: Final = [
_JSON_OBJECT.validate_json(request.body).get("prompt_cache_key") for request in wire.drain()
]
assert keys[0] == session_one, keys
assert keys[1] == session_two, keys
assert keys[2] == session_two, keys
assert keys[0] != keys[1] and keys[1] == keys[2]
def test_explicit_prompt_cache_key_wins_over_derived_session_key(gateway: Gateway) -> None:
identity: Final = f"claude-code-explicit-key-{uuid.uuid4().hex}"
explicit: Final = "explicit-client-cache-key"
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/responses"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["prompt_cache_key"] == explicit, body
return Reply(body=_responses_reply(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 16,
"prompt_cache_key": explicit,
"metadata": {"user_id": _claude_code_user_id("c" * 64, str(uuid.uuid4()))},
"messages": [{"role": "user", "content": f"explicit key probe {identity}"}],
},
)
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1

128
tests/integration/providers/test_anthropic_system_cache_control_wire.py Normal file
View file

@ -0,0 +1,128 @@
import json
import uuid
from typing import Final
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_MODEL: Final = "claude-sonnet-4-5-20250929"
_API_KEY: Final = "synthetic-anthropic-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _anthropic_reply(identity: str, text: str) -> bytes:
return json.dumps(
{
"id": identity,
"type": "message",
"role": "assistant",
"model": _MODEL,
"content": [{"type": "text", "text": text}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 3, "cache_creation_input_tokens": 12},
}
).encode()
def _assert_system_block(body: dict[str, JsonValue], policy: str) -> None:
assert body["model"] == _MODEL, body
assert body["system"] == [{"type": "text", "text": policy, "cache_control": {"type": "ephemeral"}}], body
def test_chat_completions_system_block_list_carries_cache_control_to_anthropic_system(gateway: Gateway) -> None:
identity: Final = f"anthropic-system-cc-{uuid.uuid4().hex}"
policy: Final = f"policy {identity}"
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/messages"
assert request.headers["x-api-key"] == _API_KEY
_assert_system_block(_JSON_OBJECT.validate_json(request.body), policy)
return Reply(body=_anthropic_reply(identity, "done"))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"max_tokens": 16,
"messages": [
{
"role": "system",
"content": [
{"type": "text", "text": policy, "cache_control": {"type": "ephemeral"}}
],
},
{"role": "user", "content": "hi"},
],
},
)
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1
def test_chat_completions_system_string_with_message_cache_control_reaches_anthropic_system(
gateway: Gateway,
) -> None:
identity: Final = f"anthropic-system-str-{uuid.uuid4().hex}"
policy: Final = f"policy {identity}"
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/messages"
_assert_system_block(_JSON_OBJECT.validate_json(request.body), policy)
return Reply(body=_anthropic_reply(identity, "done"))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"max_tokens": 16,
"messages": [
{"role": "system", "content": policy, "cache_control": {"type": "ephemeral"}},
{"role": "user", "content": "hi"},
],
},
)
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1
def test_responses_system_input_item_carries_cache_control_to_anthropic_system(gateway: Gateway) -> None:
identity: Final = f"responses-system-cc-{uuid.uuid4().hex}"
policy: Final = f"policy {identity}"
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/messages"
_assert_system_block(_JSON_OBJECT.validate_json(request.body), policy)
return Reply(body=_anthropic_reply(identity, "done"))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/responses",
{
"model": model,
"input": [
{
"role": "system",
"content": [
{"type": "input_text", "text": policy, "cache_control": {"type": "ephemeral"}}
],
},
{"role": "user", "content": "hi"},
],
},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["status"] == "completed", response.text
assert any(item.get("type") == "message" for item in payload.get("output", []) if isinstance(item, dict))
assert len(wire.drain()) == 1

60
tests/integration/spend/test_tag_budget_enforcement.py Normal file
View file

@ -0,0 +1,60 @@
import uuid
from typing import Final
from integration._support.client import Gateway, eventually
def test_spend_over_a_tag_max_budget_rejects_the_next_request(gateway: Gateway) -> None:
tag: Final = f"tag-budget-{uuid.uuid4().hex}"
def delete_tag() -> None:
gateway.post("/tag/delete", {"name": tag})
with gateway.scenario() as scenario:
model: Final = scenario.model(input_cost_per_token=0.01, output_cost_per_token=0.01)
gateway.post("/tag/new", {"name": tag, "max_budget": 0.0001})
scenario.cleanups.callback(delete_tag)
first: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"tag spend {tag}"}],
"metadata": {"tags": [tag]},
},
)
assert first.status_code == 200, first.text
def rejection() -> int:
return gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"tag budget probe {tag}"}],
"metadata": {"tags": [tag]},
},
).status_code
status: Final = eventually(rejection, lambda code: code != 200, seconds=70)
assert status in (400, 422, 429), status
blocked: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"tag budget probe {tag}"}],
"metadata": {"tags": [tag]},
},
)
assert "budget" in blocked.text.lower(), blocked.text
control: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"untagged probe {tag}"}],
"metadata": {"tags": [f"other-{tag}"]},
},
)
assert control.status_code == 200, control.text