scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

test(integration): cover claude-fable-5-2 capability flags and prompt-cache pricing on the scripted Anthropic upstream (#50722)

Browse source 
* test(e2e): cover claude-fable-5-2 capability flags and prompt-cache pricing over Anthropic direct

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): replace the fable-5-2 e2e suite with scripted-upstream wire and cost cases

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): add claude-fable-5-2 wire and cost cases

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-11-09 13:24:37 -08:00 • committed by GitHub
parent 07f5baba73
commit d7dc5d7fa3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 808 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

594
tests/integration/cost_calculation/cases/anthropic.py
View file

@ -4,8 +4,8 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.
from typing import Final
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
from integration.cost_calculation.cost_tracking_case import (
with_usage,
CostTrackingTestCase,
ExactExpected,
FailureDetails,
@ -14,11 +14,10 @@ from integration.cost_calculation.cost_tracking_case import (
RecountExpected,
RecountRates,
SseResponse,
with_usage,
)
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
# claude-haiku-4-5
CLAUDE_HAIKU_4_5_INPUT_TEXT: Final = CostTrackingTestCase(
name="claude-haiku-4-5-input_text",
@ -5344,6 +5343,66 @@ CLAUDE_FABLE_5_1_INPUT_TEXT: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_INPUT_TEXT: Final = CostTrackingTestCase(
name="claude-fable-5-2-input_text",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "7d8c0a2d1ecc summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted answer 7d8c0a2d1ecc"}],
"container": None,
"stop_reason": "end_turn",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 1840,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
"output_tokens": 412,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
"speed": "standard",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412
),
)
CLAUDE_FABLE_5_1_STREAM: Final = CostTrackingTestCase(
name="claude-fable-5-1-stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
@ -5429,6 +5488,91 @@ CLAUDE_FABLE_5_1_STREAM: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_STREAM: Final = CostTrackingTestCase(
name="claude-fable-5-2-stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "459736de6b4a summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": True,
"stream_options": {"include_usage": True},
"allowed_openai_params": [],
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"type": "message_start",
"message": {
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [],
"container": None,
"stop_reason": None,
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 1840,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
"output_tokens": 1,
"service_tier": "standard",
"inference_geo": "global",
"speed": "standard",
},
"diagnostics": None,
},
},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "ping"},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "scripted answer 459736de6b4a"},
},
{"type": "content_block_stop", "index": 0},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None, "stop_details": None, "container": None},
"usage": {
"input_tokens": 1840,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"output_tokens": 412,
"output_tokens_details": {"thinking_tokens": 0},
},
},
{"type": "message_stop"},
),
),
expected=ExactExpected(
spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412
),
)
CLAUDE_FABLE_5_1_CACHE_READ: Final = CostTrackingTestCase(
name="claude-fable-5-1-cache_read",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -5490,6 +5634,67 @@ CLAUDE_FABLE_5_1_CACHE_READ: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_CACHE_READ: Final = CostTrackingTestCase(
name="claude-fable-5-2-cache_read",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
"cache_control": {"type": "ephemeral"},
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "c5a9841fc31b summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted answer c5a9841fc31b"}],
"container": None,
"stop_reason": "end_turn",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 640,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 12288,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
"output_tokens": 380,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
"speed": "standard",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.02505536, input_cost=0.00833536, output_cost=0.01672, prompt_tokens=12928, completion_tokens=380
),
)
CLAUDE_FABLE_5_1_CACHE_WRITE_5M: Final = CostTrackingTestCase(
name="claude-fable-5-1-cache_write_5m",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -5550,6 +5755,66 @@ CLAUDE_FABLE_5_1_CACHE_WRITE_5M: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_CACHE_WRITE_5M: Final = CostTrackingTestCase(
name="claude-fable-5-2-cache_write_5m",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
"cache_control": {"type": "ephemeral"},
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "95e59856831e summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted answer 95e59856831e"}],
"container": None,
"stop_reason": "end_turn",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 512,
"cache_creation_input_tokens": 9216,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 9216, "ephemeral_1h_input_tokens": 0},
"output_tokens": 350,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.1212816, input_cost=0.1058816, output_cost=0.0154, prompt_tokens=9728, completion_tokens=350
),
)
CLAUDE_FABLE_5_1_CACHE_WRITE_1H: Final = CostTrackingTestCase(
name="claude-fable-5-1-cache_write_1h",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -5610,6 +5875,66 @@ CLAUDE_FABLE_5_1_CACHE_WRITE_1H: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_CACHE_WRITE_1H: Final = CostTrackingTestCase(
name="claude-fable-5-2-cache_write_1h",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "0057eda38d9c summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted answer 0057eda38d9c"}],
"container": None,
"stop_reason": "end_turn",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 512,
"cache_creation_input_tokens": 9216,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9216},
"output_tokens": 350,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.1821072, input_cost=0.1667072, output_cost=0.0154, prompt_tokens=9728, completion_tokens=350
),
)
CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
name="claude-fable-5-1-messages_input_text",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -5651,6 +5976,47 @@ CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase(
name="claude-fable-5-2-messages_input_text",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
endpoint="/v1/messages",
request={
"model": "$MODEL",
"max_tokens": 412,
"messages": [{"role": "user", "content": "summarize the attached material in one line"}],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted response"}],
"container": None,
"stop_reason": "end_turn",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 1840,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
"output_tokens": 412,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
"speed": "standard",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412
),
)
CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase(
name="claude-fable-5-1-messages_stream",
@ -5680,6 +6046,35 @@ CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase(
spend=0.0624, input_cost=0.02944, output_cost=0.03296, prompt_tokens=1840, completion_tokens=412
),
)
CLAUDE_FABLE_5_2_MESSAGES_STREAM: Final = CostTrackingTestCase(
name="claude-fable-5-2-messages_stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="claude-fable-5-2",
endpoint="/v1/messages",
request={
"model": "$MODEL",
"max_tokens": 412,
"messages": [{"role": "user", "content": "summarize the attached material in one line"}],
"stream": True,
},
response=SseResponse(
content_type="text/event-stream",
frames=(
'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-2","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}',
'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}',
'event: ping\ndata: {"type":"ping"}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted "}}',
'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"response"}}',
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":412,"output_tokens_details":{"thinking_tokens":0}}}',
'event: message_stop\ndata: {"type":"message_stop"}',
),
),
expected=ExactExpected(
spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412
),
)
CLAUDE_OPUS_5_5_STREAM: Final = CostTrackingTestCase(
name="claude-opus-5-5-stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
@ -6560,6 +6955,92 @@ CLAUDE_FABLE_5_1_TOOL_CALL: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_TOOL_CALL: Final = CostTrackingTestCase(
name="claude-fable-5-2-tool_call",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "6c08522a6809 summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather and a short forecast for a city.",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string", "description": "City name"},
"days": {"type": "integer", "description": "Forecast horizon in days"},
"units": {"type": "string", "enum": ["metric", "imperial"]},
},
"required": ["city"],
},
},
}
],
"tool_choice": "auto",
"allowed_openai_params": ["tool_choice"],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [
{
"type": "tool_use",
"id": "toolu_$REQUEST_ID",
"name": "get_weather",
"input": {"city": "Berlin", "days": 7, "units": "metric"},
"caller": {"type": "direct"},
}
],
"container": None,
"stop_reason": "tool_use",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 1840,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
"output_tokens": 412,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412
),
)
CLAUDE_FABLE_5_1_THINKING: Final = CostTrackingTestCase(
name="claude-fable-5-1-thinking",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -6621,6 +7102,67 @@ CLAUDE_FABLE_5_1_THINKING: Final = CostTrackingTestCase(
),
)
CLAUDE_FABLE_5_2_THINKING: Final = CostTrackingTestCase(
name="claude-fable-5-2-thinking",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="claude-fable-5-2",
request={
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "697b4f57a5df summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"reasoning_effort": "medium",
"allowed_openai_params": ["reasoning_effort"],
},
response=JsonResponse(
content_type="application/json",
body={
"model": "claude-fable-5-2",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "scripted answer 697b4f57a5df"}],
"container": None,
"stop_reason": "end_turn",
"stop_sequence": None,
"stop_details": None,
"usage": {
"input_tokens": 1240,
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
"output_tokens": 412,
"output_tokens_details": {"thinking_tokens": 0},
"service_tier": "standard",
"inference_geo": "global",
"speed": "standard",
},
"diagnostics": None,
},
),
expected=ExactExpected(
spend=0.02904, input_cost=0.010912, output_cost=0.018128, prompt_tokens=1240, completion_tokens=412
),
)
CLAUDE_OPUS_5_5_TOOL_CALL: Final = CostTrackingTestCase(
name="claude-opus-5-5-tool_call",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -6895,6 +7437,14 @@ CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON: Final = CLAUDE_FABLE_5_1_MESSAGES_INPUT
}
)
CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON: Final = CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT.model_copy(
update={
"name": "claude-fable-5-2-responses_bridged_json",
"endpoint": "/v1/responses",
"request": {"model": "$MODEL", "input": "summarize the attached material in one line"},
}
)
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT.model_copy(
update={
"name": "claude-opus-5-5-responses_bridged_json",
@ -6903,6 +7453,7 @@ CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_T
}
)
def _responses_bridged_stream_case(
sibling: CostTrackingTestCase, stream_sibling: CostTrackingTestCase, name: str
) -> CostTrackingTestCase:
@ -6916,19 +7467,29 @@ def _responses_bridged_stream_case(
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_MESSAGES_STREAM,
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON,
CLAUDE_FABLE_5_1_MESSAGES_STREAM,
"claude-fable-5-1-responses_bridged_stream",
)
CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON,
CLAUDE_FABLE_5_2_MESSAGES_STREAM,
"claude-fable-5-2-responses_bridged_stream",
)
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON,
CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
"claude-haiku-5-5-responses_bridged_stream",
)
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_OPUS_5_5_MESSAGES_STREAM,
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON,
CLAUDE_OPUS_5_5_MESSAGES_STREAM,
"claude-opus-5-5-responses_bridged_stream",
)
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_SONNET_5_5_MESSAGES_STREAM,
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON,
CLAUDE_SONNET_5_5_MESSAGES_STREAM,
"claude-sonnet-5-5-responses_bridged_stream",
)
@ -7028,11 +7589,13 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON,
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON,
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM,
CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_STREAM,
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM,
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM,
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM,
CLAUDE_FABLE_5_RESPONSES_BRIDGED_JSON,
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON,
CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON,
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON,
CLAUDE_HAIKU_4_5_INPUT_TEXT,
CLAUDE_HAIKU_4_5_CACHE_READ,
@ -7129,12 +7692,19 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
CLAUDE_FABLE_5_CACHE_WRITE_1H,
CLAUDE_FABLE_5_MESSAGES_INPUT_TEXT,
CLAUDE_FABLE_5_1_INPUT_TEXT,
CLAUDE_FABLE_5_2_INPUT_TEXT,
CLAUDE_FABLE_5_1_STREAM,
CLAUDE_FABLE_5_2_STREAM,
CLAUDE_FABLE_5_1_CACHE_READ,
CLAUDE_FABLE_5_2_CACHE_READ,
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
CLAUDE_FABLE_5_2_CACHE_WRITE_5M,
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
CLAUDE_FABLE_5_2_CACHE_WRITE_1H,
CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT,
CLAUDE_FABLE_5_1_MESSAGES_STREAM,
CLAUDE_FABLE_5_2_MESSAGES_STREAM,
CLAUDE_OPUS_5_5_STREAM,
CLAUDE_OPUS_5_5_CACHE_READ,
CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
@ -7148,7 +7718,9 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
CLAUDE_FABLE_5_TOOL_CALL,
CLAUDE_FABLE_5_THINKING,
CLAUDE_FABLE_5_1_TOOL_CALL,
CLAUDE_FABLE_5_2_TOOL_CALL,
CLAUDE_FABLE_5_1_THINKING,
CLAUDE_FABLE_5_2_THINKING,
CLAUDE_OPUS_5_5_TOOL_CALL,
CLAUDE_OPUS_5_5_THINKING,
CLAUDE_OPUS_5_5_UPSTREAM_404_ZERO_SPEND,
@ -7159,6 +7731,9 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
StreamParityTestCase(
plain=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM
),
StreamParityTestCase(
plain=CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_STREAM
),
StreamParityTestCase(
plain=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM
),
@ -7190,11 +7765,13 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
StreamParityTestCase(plain=CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=CLAUDE_HAIKU_5_5_STREAM),
StreamParityTestCase(plain=CLAUDE_FABLE_5_INPUT_TEXT, streamed=CLAUDE_FABLE_5_STREAM),
StreamParityTestCase(plain=CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=CLAUDE_FABLE_5_1_STREAM),
StreamParityTestCase(plain=CLAUDE_FABLE_5_2_INPUT_TEXT, streamed=CLAUDE_FABLE_5_2_STREAM),
StreamParityTestCase(plain=CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=CLAUDE_OPUS_5_5_STREAM),
StreamParityTestCase(plain=CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_OPUS_5_5_MESSAGES_STREAM),
StreamParityTestCase(plain=CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_SONNET_5_5_MESSAGES_STREAM),
StreamParityTestCase(plain=CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_HAIKU_5_5_MESSAGES_STREAM),
StreamParityTestCase(plain=CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, streamed=CLAUDE_FABLE_5_1_MESSAGES_STREAM),
StreamParityTestCase(plain=CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT, streamed=CLAUDE_FABLE_5_2_MESSAGES_STREAM),
)
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
@ -7220,6 +7797,9 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
BridgeCostParityTestCase(
native=CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, bridged=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON
),
BridgeCostParityTestCase(
native=CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT, bridged=CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON
),
BridgeCostParityTestCase(
native=CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, bridged=CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON
),

13
tests/integration/cost_calculation/cost_map.py
View file

@ -1398,6 +1398,19 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"output_cost_per_token": 8e-05,
"supports_function_calling": True,
},
"claude-fable-5-2": {
"cache_creation_input_token_cost": 1.1e-05,
"cache_creation_input_token_cost_above_1hr": 1.76e-05,
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 8.8e-06,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4.4e-05,
"supports_function_calling": True,
},
"us.anthropic.claude-sonnet-5": {
"cache_creation_input_token_cost": 4.5e-06,
"cache_creation_input_token_cost_above_1hr": 7.2e-06,

208
tests/integration/providers/test_anthropic_fable_capability_flags_wire.py Normal file
View file

@ -0,0 +1,208 @@
"""Claude Fable 5.2 over Anthropic direct. Its cost-map row says `supports_sampling_params: false`,
`supports_forced_tool_use: false` and `thinking_always_on: true`, and Anthropic answers 400 to a
request that ignores any of them, so the proxy has to rewrite or refuse those requests before they
leave. The scripted upstream here is what proves the outbound body, a leaked field is invisible in
the response."""
import json
import uuid
from typing import Final
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_MODEL: Final = "claude-fable-5-2"
_API_KEY: Final = "synthetic-anthropic-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_WEATHER_TOOL: Final[JsonValue] = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
_NAMED_TOOL_CHOICE: Final[JsonValue] = {"type": "function", "function": {"name": "get_weather"}}
def _anthropic_reply(identity: str, usage: dict[str, JsonValue] | None = None) -> bytes:
return json.dumps(
{
"id": identity,
"type": "message",
"role": "assistant",
"model": _MODEL,
"content": [
{"type": "thinking", "thinking": "scripted thought", "signature": "scripted-signature"},
{"type": "text", "text": "done"},
],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": usage or {"input_tokens": 12, "output_tokens": 3},
}
).encode()
def _chat_body(model: str, **extra: JsonValue) -> dict[str, JsonValue]:
return {
"model": model,
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}],
"cache": {"no-cache": True},
**extra,
}
def _outbound(request: Request) -> dict[str, JsonValue]:
assert request.method == "POST" and request.target == "/v1/messages", (request.method, request.target)
assert request.headers["x-api-key"] == _API_KEY
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["model"] == _MODEL, body
return body
def test_drop_params_removes_temperature_and_top_p_before_anthropic(gateway: Gateway) -> None:
identity: Final = f"fable-sampling-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
body: Final = _outbound(request)
assert "temperature" not in body and "top_p" not in body, body
return Reply(body=_anthropic_reply(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, drop_params=True
)
response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, temperature=0.2, top_p=0.5))
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1
def test_sampling_params_are_refused_before_reaching_anthropic_without_drop_params(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
raise AssertionError(f"a request with sampling params reached Anthropic: {request.body!r}")
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, temperature=0.2))
assert response.status_code == 400, response.text
assert "temperature" in response.text and "drop_params" in response.text, response.text
assert wire.drain() == ()
def test_drop_params_downgrades_forced_and_named_tool_choice_to_auto(gateway: Gateway) -> None:
identity: Final = f"fable-tool-choice-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
body: Final = _outbound(request)
assert body["tool_choice"] == {"type": "auto"}, body
tools: Final = body["tools"]
assert isinstance(tools, list) and len(tools) == 1, body
first: Final = tools[0]
assert isinstance(first, dict) and first["name"] == "get_weather", body
return Reply(body=_anthropic_reply(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, drop_params=True
)
responses: Final = tuple(
gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, tools=[_WEATHER_TOOL], tool_choice=tool_choice)
)
for tool_choice in ("required", _NAMED_TOOL_CHOICE)
)
assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses]
assert len(wire.drain()) == 2
def test_forced_tool_choice_is_refused_before_reaching_anthropic_without_drop_params(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
raise AssertionError(f"a forced tool_choice reached Anthropic: {request.body!r}")
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, tools=[_WEATHER_TOOL], tool_choice="required")
)
assert response.status_code == 400, response.text
assert "tool" in response.text and "drop_params" in response.text, response.text
assert wire.drain() == ()
def test_thinking_disabled_is_dropped_for_an_always_on_model(gateway: Gateway) -> None:
identity: Final = f"fable-thinking-off-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
body: Final = _outbound(request)
assert "thinking" not in body, body
return Reply(body=_anthropic_reply(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST", "/v1/chat/completions", _chat_body(model, thinking={"type": "disabled"})
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
choices: Final = payload["choices"]
assert isinstance(choices, list) and isinstance(choices[0], dict), response.text
message: Final = choices[0]["message"]
assert isinstance(message, dict) and message["reasoning_content"] == "scripted thought", response.text
assert len(wire.drain()) == 1
def test_reasoning_effort_becomes_adaptive_thinking_with_output_config_effort(gateway: Gateway) -> None:
identity: Final = f"fable-effort-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
body: Final = _outbound(request)
assert body["thinking"] == {"type": "adaptive", "display": "summarized"}, body
assert body["output_config"] == {"effort": "max"}, body
assert "budget_tokens" not in json.dumps(body), body
return Reply(body=_anthropic_reply(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, reasoning_effort="max"))
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1
def test_messages_one_hour_cache_control_reaches_anthropic_and_its_usage_comes_back(gateway: Gateway) -> None:
identity: Final = f"fable-cache-1h-{uuid.uuid4().hex}"
policy: Final = f"policy {identity}"
upstream_usage: Final[dict[str, JsonValue]] = {
"input_tokens": 12,
"output_tokens": 3,
"cache_creation_input_tokens": 9216,
"cache_read_input_tokens": 0,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9216},
}
def respond(request: Request) -> Reply:
body: Final = _outbound(request)
assert body["system"] == [
{"type": "text", "text": policy, "cache_control": {"type": "ephemeral", "ttl": "1h"}}
], body
return Reply(body=_anthropic_reply(identity, upstream_usage))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 16,
"system": [{"type": "text", "text": policy, "cache_control": {"type": "ephemeral", "ttl": "1h"}}],
"messages": [{"role": "user", "content": "hi"}],
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
usage: Final = _JSON_OBJECT.validate_json(response.content)["usage"]
assert isinstance(usage, dict), response.text
assert usage["cache_creation_input_tokens"] == 9216, usage
assert usage["cache_creation"] == {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9216}, usage
assert len(wire.drain()) == 1