From 35225709ed34bdf8a907dc233c19a8ceea67c947 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:19:38 -0700 Subject: [PATCH] test(integration): cover customer reported cache key, cache_control, bedrock request id, responses schema, scim and tag budget contracts (#42785) * test(integration): cover prompt cache key, system cache_control, bedrock request id, responses schema, scim and tag budget contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop invented instructions shape and unused imports Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): register scim placeholder cleanup before asserting the patch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_responses_openapi_schema.py | 21 +++ ...t_scim_group_member_not_yet_provisioned.py | 30 ++++ .../test_bedrock_error_request_id.py | 56 ++++++++ ...pic_messages_claude_code_cache_key_wire.py | 105 ++++++++++++++ ...est_anthropic_system_cache_control_wire.py | 128 ++++++++++++++++++ .../spend/test_tag_budget_enforcement.py | 60 ++++++++ 6 files changed, 400 insertions(+) create mode 100644 tests/integration/compatibility/test_responses_openapi_schema.py create mode 100644 tests/integration/management/test_scim_group_member_not_yet_provisioned.py create mode 100644 tests/integration/observability/test_bedrock_error_request_id.py create mode 100644 tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py create mode 100644 tests/integration/providers/test_anthropic_system_cache_control_wire.py create mode 100644 tests/integration/spend/test_tag_budget_enforcement.py diff --git a/tests/integration/compatibility/test_responses_openapi_schema.py b/tests/integration/compatibility/test_responses_openapi_schema.py new file mode 100644 index 00000000000..65039bb5f2e --- /dev/null +++ b/tests/integration/compatibility/test_responses_openapi_schema.py @@ -0,0 +1,21 @@ +import pytest +from integration._support.client import Gateway, object_value +from pydantic import JsonValue + + +def _assert_responses_post_is_documented(openapi: dict[str, JsonValue]) -> None: + post: dict[str, JsonValue] = object_value(object_value(object_value(openapi["paths"])["/v1/responses"])["post"]) + body: dict[str, JsonValue] = object_value(post["requestBody"]) + schema: dict[str, JsonValue] = object_value( + object_value(object_value(body["content"])["application/json"])["schema"] + ) + properties: dict[str, JsonValue] = object_value(schema.get("properties")) + assert "model" in properties and "input" in properties, schema + ok: dict[str, JsonValue] = object_value(object_value(object_value(post)["responses"])["200"]) + assert "schema" in object_value(object_value(ok["content"])["application/json"]), ok + + +def test_v1_responses_post_declares_a_request_body_and_response_schema(gateway: Gateway) -> None: + pytest.skip("BUG: POST /v1/responses takes a raw Request, so /openapi.json documents no body or response schema") + openapi: dict[str, JsonValue] = gateway.get("/openapi.json") + _assert_responses_post_is_documented(openapi) diff --git a/tests/integration/management/test_scim_group_member_not_yet_provisioned.py b/tests/integration/management/test_scim_group_member_not_yet_provisioned.py new file mode 100644 index 00000000000..8e339b711d3 --- /dev/null +++ b/tests/integration/management/test_scim_group_member_not_yet_provisioned.py @@ -0,0 +1,30 @@ +import uuid +from typing import Final + +from integration._support.client import Gateway, object_value, string_value +from pydantic import JsonValue + + +def test_scim_group_patch_add_member_provisions_the_missing_user(gateway: Gateway) -> None: + missing_user: Final = f"scim-pending-{uuid.uuid4().hex}" + + with gateway.scenario() as scenario: + team: Final = scenario.team() + response: Final = gateway.request( + "PATCH", + f"/scim/v2/Groups/{team}", + { + "schemas": ["urn:ietf:params:scim:api:messages:2.0:PatchOp"], + "Operations": [ + {"op": "add", "path": "members", "value": [{"value": missing_user}]} + ], + }, + ) + scenario.cleanups.callback(scenario.delete_user, missing_user) + assert response.status_code == 200, response.text + team_info: dict[str, JsonValue] = gateway.get("/team/info", {"team_id": team}) + members: Final = object_value(team_info["team_info"]).get("members_with_roles") or [] + member_ids: Final = [ + string_value(object_value(member)["user_id"]) for member in members if isinstance(member, dict) + ] + assert missing_user in member_ids, members diff --git a/tests/integration/observability/test_bedrock_error_request_id.py b/tests/integration/observability/test_bedrock_error_request_id.py new file mode 100644 index 00000000000..230619a2863 --- /dev/null +++ b/tests/integration/observability/test_bedrock_error_request_id.py @@ -0,0 +1,56 @@ +import json +import uuid +from typing import Final + +from integration._support.client import Gateway, eventually, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server + +_MODEL: Final = "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0" +_TOKEN: Final = "synthetic-bedrock-bearer" + + +def test_bedrock_500_keeps_amzn_request_id_on_error_headers_and_failure_log(gateway: Gateway) -> None: + identity: Final = f"bedrock-request-id-{uuid.uuid4().hex}" + amzn_request_id: Final = str(uuid.uuid4()) + prompt: Final = f"failure probe {identity}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target == "/model/anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse", request.target + return Reply( + status=500, + headers={"x-amzn-RequestId": amzn_request_id}, + body=b'{"message":"synthetic bedrock failure"}', + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=_MODEL, + api_key=_TOKEN, + aws_region_name="us-east-1", + aws_bedrock_runtime_endpoint=wire.url, + num_retries=0, + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": prompt}]}, + ) + assert response.status_code >= 400, response.text + assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers) + call_id: Final = response.headers["x-litellm-call-id"] + assert len(wire.drain()) == 1 + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,) + ), + lambda values: len(values) == 1, + seconds=70, + ) + row: Final = rows[0] + assert row["status"] == "failure", row + metadata: Final = row["metadata"] + parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata) + error_information: Final = object_value(parsed["error_information"]) + assert error_information["error_provider_request_id"] == amzn_request_id, error_information diff --git a/tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py b/tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py new file mode 100644 index 00000000000..c9fb3a7ae16 --- /dev/null +++ b/tests/integration/providers/test_anthropic_messages_claude_code_cache_key_wire.py @@ -0,0 +1,105 @@ +import json +import uuid +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "gpt-5.4-mini" +_API_KEY: Final = "synthetic-openai-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _claude_code_user_id(device_id: str, session_id: str) -> str: + return json.dumps({"device_id": device_id, "account_uuid": "", "session_id": session_id}) + + +def _responses_reply(identity: str) -> bytes: + return json.dumps( + { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": _BACKEND, + "output": [ + { + "type": "message", + "id": f"msg_{identity}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, + } + ).encode() + + +def test_prompt_cache_key_is_derived_from_claude_code_session_id_not_device_id(gateway: Gateway) -> None: + identity: Final = f"claude-code-cache-key-{uuid.uuid4().hex}" + device_one: Final = "a" * 64 + device_two: Final = "b" * 64 + session_one: Final = str(uuid.uuid4()) + session_two: Final = str(uuid.uuid4()) + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + return Reply(body=_responses_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + + def send(user_id: str, probe: str) -> None: + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "metadata": {"user_id": user_id}, + "messages": [{"role": "user", "content": probe}], + }, + ) + assert response.status_code == 200, response.text + + send(_claude_code_user_id(device_one, session_one), f"probe one {identity}") + send(_claude_code_user_id(device_one, session_two), f"probe two {identity}") + send(_claude_code_user_id(device_two, session_two), f"probe three {identity}") + + keys: Final = [ + _JSON_OBJECT.validate_json(request.body).get("prompt_cache_key") for request in wire.drain() + ] + assert keys[0] == session_one, keys + assert keys[1] == session_two, keys + assert keys[2] == session_two, keys + assert keys[0] != keys[1] and keys[1] == keys[2] + + +def test_explicit_prompt_cache_key_wins_over_derived_session_key(gateway: Gateway) -> None: + identity: Final = f"claude-code-explicit-key-{uuid.uuid4().hex}" + explicit: Final = "explicit-client-cache-key" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["prompt_cache_key"] == explicit, body + return Reply(body=_responses_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "prompt_cache_key": explicit, + "metadata": {"user_id": _claude_code_user_id("c" * 64, str(uuid.uuid4()))}, + "messages": [{"role": "user", "content": f"explicit key probe {identity}"}], + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 diff --git a/tests/integration/providers/test_anthropic_system_cache_control_wire.py b/tests/integration/providers/test_anthropic_system_cache_control_wire.py new file mode 100644 index 00000000000..cdac76158e6 --- /dev/null +++ b/tests/integration/providers/test_anthropic_system_cache_control_wire.py @@ -0,0 +1,128 @@ +import json +import uuid +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "claude-sonnet-4-5-20250929" +_API_KEY: Final = "synthetic-anthropic-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _anthropic_reply(identity: str, text: str) -> bytes: + return json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 3, "cache_creation_input_tokens": 12}, + } + ).encode() + + +def _assert_system_block(body: dict[str, JsonValue], policy: str) -> None: + assert body["model"] == _MODEL, body + assert body["system"] == [{"type": "text", "text": policy, "cache_control": {"type": "ephemeral"}}], body + + +def test_chat_completions_system_block_list_carries_cache_control_to_anthropic_system(gateway: Gateway) -> None: + identity: Final = f"anthropic-system-cc-{uuid.uuid4().hex}" + policy: Final = f"policy {identity}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages" + assert request.headers["x-api-key"] == _API_KEY + _assert_system_block(_JSON_OBJECT.validate_json(request.body), policy) + return Reply(body=_anthropic_reply(identity, "done")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [ + { + "role": "system", + "content": [ + {"type": "text", "text": policy, "cache_control": {"type": "ephemeral"}} + ], + }, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_chat_completions_system_string_with_message_cache_control_reaches_anthropic_system( + gateway: Gateway, +) -> None: + identity: Final = f"anthropic-system-str-{uuid.uuid4().hex}" + policy: Final = f"policy {identity}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages" + _assert_system_block(_JSON_OBJECT.validate_json(request.body), policy) + return Reply(body=_anthropic_reply(identity, "done")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 16, + "messages": [ + {"role": "system", "content": policy, "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_responses_system_input_item_carries_cache_control_to_anthropic_system(gateway: Gateway) -> None: + identity: Final = f"responses-system-cc-{uuid.uuid4().hex}" + policy: Final = f"policy {identity}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages" + _assert_system_block(_JSON_OBJECT.validate_json(request.body), policy) + return Reply(body=_anthropic_reply(identity, "done")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/responses", + { + "model": model, + "input": [ + { + "role": "system", + "content": [ + {"type": "input_text", "text": policy, "cache_control": {"type": "ephemeral"}} + ], + }, + {"role": "user", "content": "hi"}, + ], + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["status"] == "completed", response.text + assert any(item.get("type") == "message" for item in payload.get("output", []) if isinstance(item, dict)) + assert len(wire.drain()) == 1 + diff --git a/tests/integration/spend/test_tag_budget_enforcement.py b/tests/integration/spend/test_tag_budget_enforcement.py new file mode 100644 index 00000000000..e8d4c6438a5 --- /dev/null +++ b/tests/integration/spend/test_tag_budget_enforcement.py @@ -0,0 +1,60 @@ +import uuid +from typing import Final + +from integration._support.client import Gateway, eventually + + +def test_spend_over_a_tag_max_budget_rejects_the_next_request(gateway: Gateway) -> None: + tag: Final = f"tag-budget-{uuid.uuid4().hex}" + + def delete_tag() -> None: + gateway.post("/tag/delete", {"name": tag}) + + with gateway.scenario() as scenario: + model: Final = scenario.model(input_cost_per_token=0.01, output_cost_per_token=0.01) + gateway.post("/tag/new", {"name": tag, "max_budget": 0.0001}) + scenario.cleanups.callback(delete_tag) + first: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tag spend {tag}"}], + "metadata": {"tags": [tag]}, + }, + ) + assert first.status_code == 200, first.text + + def rejection() -> int: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tag budget probe {tag}"}], + "metadata": {"tags": [tag]}, + }, + ).status_code + + status: Final = eventually(rejection, lambda code: code != 200, seconds=70) + assert status in (400, 422, 429), status + blocked: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tag budget probe {tag}"}], + "metadata": {"tags": [tag]}, + }, + ) + assert "budget" in blocked.text.lower(), blocked.text + control: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"untagged probe {tag}"}], + "metadata": {"tags": [f"other-{tag}"]}, + }, + ) + assert control.status_code == 200, control.text