diff --git a/tests/integration/cost_calculation/cases/anthropic.py b/tests/integration/cost_calculation/cases/anthropic.py index 776c5b51dd4..565d521aa95 100644 --- a/tests/integration/cost_calculation/cases/anthropic.py +++ b/tests/integration/cost_calculation/cases/anthropic.py @@ -4,8 +4,8 @@ PARITY pairs the streamed case with its plain twin where both bill the same row. from typing import Final +from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cost_tracking_case import ( - with_usage, CostTrackingTestCase, ExactExpected, FailureDetails, @@ -14,11 +14,10 @@ from integration.cost_calculation.cost_tracking_case import ( RecountExpected, RecountRates, SseResponse, + with_usage, ) -from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames - # claude-haiku-4-5 CLAUDE_HAIKU_4_5_INPUT_TEXT: Final = CostTrackingTestCase( name="claude-haiku-4-5-input_text", @@ -5344,6 +5343,66 @@ CLAUDE_FABLE_5_1_INPUT_TEXT: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_INPUT_TEXT: Final = CostTrackingTestCase( + name="claude-fable-5-2-input_text", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7d8c0a2d1ecc summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted answer 7d8c0a2d1ecc"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412 + ), +) + CLAUDE_FABLE_5_1_STREAM: Final = CostTrackingTestCase( name="claude-fable-5-1-stream", covers="quota_management.spend_tracking.scripted_wire.logs_cost", @@ -5429,6 +5488,91 @@ CLAUDE_FABLE_5_1_STREAM: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_STREAM: Final = CostTrackingTestCase( + name="claude-fable-5-2-stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "459736de6b4a summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": True, + "stream_options": {"include_usage": True}, + "allowed_openai_params": [], + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "message_start", + "message": { + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [], + "container": None, + "stop_reason": None, + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 1, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "ping"}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "scripted answer 459736de6b4a"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None, "stop_details": None, "container": None}, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + {"type": "message_stop"}, + ), + ), + expected=ExactExpected( + spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412 + ), +) + CLAUDE_FABLE_5_1_CACHE_READ: Final = CostTrackingTestCase( name="claude-fable-5-1-cache_read", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -5490,6 +5634,67 @@ CLAUDE_FABLE_5_1_CACHE_READ: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_CACHE_READ: Final = CostTrackingTestCase( + name="claude-fable-5-2-cache_read", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c5a9841fc31b summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted answer c5a9841fc31b"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 640, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 12288, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 380, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.02505536, input_cost=0.00833536, output_cost=0.01672, prompt_tokens=12928, completion_tokens=380 + ), +) + CLAUDE_FABLE_5_1_CACHE_WRITE_5M: Final = CostTrackingTestCase( name="claude-fable-5-1-cache_write_5m", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -5550,6 +5755,66 @@ CLAUDE_FABLE_5_1_CACHE_WRITE_5M: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_CACHE_WRITE_5M: Final = CostTrackingTestCase( + name="claude-fable-5-2-cache_write_5m", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "95e59856831e summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted answer 95e59856831e"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 512, + "cache_creation_input_tokens": 9216, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 9216, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 350, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.1212816, input_cost=0.1058816, output_cost=0.0154, prompt_tokens=9728, completion_tokens=350 + ), +) + CLAUDE_FABLE_5_1_CACHE_WRITE_1H: Final = CostTrackingTestCase( name="claude-fable-5-1-cache_write_1h", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -5610,6 +5875,66 @@ CLAUDE_FABLE_5_1_CACHE_WRITE_1H: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_CACHE_WRITE_1H: Final = CostTrackingTestCase( + name="claude-fable-5-2-cache_write_1h", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0057eda38d9c summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "allowed_openai_params": [], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted answer 0057eda38d9c"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 512, + "cache_creation_input_tokens": 9216, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9216}, + "output_tokens": 350, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.1821072, input_cost=0.1667072, output_cost=0.0154, prompt_tokens=9728, completion_tokens=350 + ), +) + CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( name="claude-fable-5-1-messages_input_text", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -5651,6 +5976,47 @@ CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT: Final = CostTrackingTestCase( + name="claude-fable-5-2-messages_input_text", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "summarize the attached material in one line"}], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted response"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412 + ), +) + CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase( name="claude-fable-5-1-messages_stream", @@ -5680,6 +6046,35 @@ CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CostTrackingTestCase( spend=0.0624, input_cost=0.02944, output_cost=0.03296, prompt_tokens=1840, completion_tokens=412 ), ) + +CLAUDE_FABLE_5_2_MESSAGES_STREAM: Final = CostTrackingTestCase( + name="claude-fable-5-2-messages_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="claude-fable-5-2", + endpoint="/v1/messages", + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "summarize the attached material in one line"}], + "stream": True, + }, + response=SseResponse( + content_type="text/event-stream", + frames=( + 'event: message_start\ndata: {"type":"message_start","message":{"model":"claude-fable-5-2","id":"msg_$REQUEST_ID","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"global","speed":"standard"},"diagnostics":null}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: ping\ndata: {"type":"ping"}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"scripted "}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"response"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":1840,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":412,"output_tokens_details":{"thinking_tokens":0}}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ), + ), + expected=ExactExpected( + spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412 + ), +) CLAUDE_OPUS_5_5_STREAM: Final = CostTrackingTestCase( name="claude-opus-5-5-stream", covers="quota_management.spend_tracking.scripted_wire.logs_cost", @@ -6560,6 +6955,92 @@ CLAUDE_FABLE_5_1_TOOL_CALL: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_TOOL_CALL: Final = CostTrackingTestCase( + name="claude-fable-5-2-tool_call", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6c08522a6809 summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + "days": {"type": "integer", "description": "Forecast horizon in days"}, + "units": {"type": "string", "enum": ["metric", "imperial"]}, + }, + "required": ["city"], + }, + }, + } + ], + "tool_choice": "auto", + "allowed_openai_params": ["tool_choice"], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_$REQUEST_ID", + "name": "get_weather", + "input": {"city": "Berlin", "days": 7, "units": "metric"}, + "caller": {"type": "direct"}, + } + ], + "container": None, + "stop_reason": "tool_use", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.03432, input_cost=0.016192, output_cost=0.018128, prompt_tokens=1840, completion_tokens=412 + ), +) + CLAUDE_FABLE_5_1_THINKING: Final = CostTrackingTestCase( name="claude-fable-5-1-thinking", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -6621,6 +7102,67 @@ CLAUDE_FABLE_5_1_THINKING: Final = CostTrackingTestCase( ), ) +CLAUDE_FABLE_5_2_THINKING: Final = CostTrackingTestCase( + name="claude-fable-5-2-thinking", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="claude-fable-5-2", + request={ + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "697b4f57a5df summarize the attached material in one line and name the city weather", + } + ], + }, + ], + "stream": False, + "reasoning_effort": "medium", + "allowed_openai_params": ["reasoning_effort"], + }, + response=JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-2", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "scripted answer 697b4f57a5df"}], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1240, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + "speed": "standard", + }, + "diagnostics": None, + }, + ), + expected=ExactExpected( + spend=0.02904, input_cost=0.010912, output_cost=0.018128, prompt_tokens=1240, completion_tokens=412 + ), +) + CLAUDE_OPUS_5_5_TOOL_CALL: Final = CostTrackingTestCase( name="claude-opus-5-5-tool_call", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -6895,6 +7437,14 @@ CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON: Final = CLAUDE_FABLE_5_1_MESSAGES_INPUT } ) +CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON: Final = CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT.model_copy( + update={ + "name": "claude-fable-5-2-responses_bridged_json", + "endpoint": "/v1/responses", + "request": {"model": "$MODEL", "input": "summarize the attached material in one line"}, + } +) + CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT.model_copy( update={ "name": "claude-opus-5-5-responses_bridged_json", @@ -6903,6 +7453,7 @@ CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_T } ) + def _responses_bridged_stream_case( sibling: CostTrackingTestCase, stream_sibling: CostTrackingTestCase, name: str ) -> CostTrackingTestCase: @@ -6916,19 +7467,29 @@ def _responses_bridged_stream_case( CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( - CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_MESSAGES_STREAM, + CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, + CLAUDE_FABLE_5_1_MESSAGES_STREAM, "claude-fable-5-1-responses_bridged_stream", ) + +CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( + CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON, + CLAUDE_FABLE_5_2_MESSAGES_STREAM, + "claude-fable-5-2-responses_bridged_stream", +) CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( - CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, + CLAUDE_HAIKU_5_5_MESSAGES_STREAM, "claude-haiku-5-5-responses_bridged_stream", ) CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( - CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_OPUS_5_5_MESSAGES_STREAM, + CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, + CLAUDE_OPUS_5_5_MESSAGES_STREAM, "claude-opus-5-5-responses_bridged_stream", ) CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( - CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_SONNET_5_5_MESSAGES_STREAM, + CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, + CLAUDE_SONNET_5_5_MESSAGES_STREAM, "claude-sonnet-5-5-responses_bridged_stream", ) @@ -7028,11 +7589,13 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM, + CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_STREAM, CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM, CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM, CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM, CLAUDE_FABLE_5_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, + CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON, CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_4_5_INPUT_TEXT, CLAUDE_HAIKU_4_5_CACHE_READ, @@ -7129,12 +7692,19 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_FABLE_5_CACHE_WRITE_1H, CLAUDE_FABLE_5_MESSAGES_INPUT_TEXT, CLAUDE_FABLE_5_1_INPUT_TEXT, + CLAUDE_FABLE_5_2_INPUT_TEXT, CLAUDE_FABLE_5_1_STREAM, + CLAUDE_FABLE_5_2_STREAM, CLAUDE_FABLE_5_1_CACHE_READ, + CLAUDE_FABLE_5_2_CACHE_READ, CLAUDE_FABLE_5_1_CACHE_WRITE_5M, + CLAUDE_FABLE_5_2_CACHE_WRITE_5M, CLAUDE_FABLE_5_1_CACHE_WRITE_1H, + CLAUDE_FABLE_5_2_CACHE_WRITE_1H, CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT, CLAUDE_FABLE_5_1_MESSAGES_STREAM, + CLAUDE_FABLE_5_2_MESSAGES_STREAM, CLAUDE_OPUS_5_5_STREAM, CLAUDE_OPUS_5_5_CACHE_READ, CLAUDE_OPUS_5_5_CACHE_WRITE_5M, @@ -7148,7 +7718,9 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_FABLE_5_TOOL_CALL, CLAUDE_FABLE_5_THINKING, CLAUDE_FABLE_5_1_TOOL_CALL, + CLAUDE_FABLE_5_2_TOOL_CALL, CLAUDE_FABLE_5_1_THINKING, + CLAUDE_FABLE_5_2_THINKING, CLAUDE_OPUS_5_5_TOOL_CALL, CLAUDE_OPUS_5_5_THINKING, CLAUDE_OPUS_5_5_UPSTREAM_404_ZERO_SPEND, @@ -7159,6 +7731,9 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM ), + StreamParityTestCase( + plain=CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_STREAM + ), StreamParityTestCase( plain=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM ), @@ -7190,11 +7765,13 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=CLAUDE_HAIKU_5_5_STREAM), StreamParityTestCase(plain=CLAUDE_FABLE_5_INPUT_TEXT, streamed=CLAUDE_FABLE_5_STREAM), StreamParityTestCase(plain=CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=CLAUDE_FABLE_5_1_STREAM), + StreamParityTestCase(plain=CLAUDE_FABLE_5_2_INPUT_TEXT, streamed=CLAUDE_FABLE_5_2_STREAM), StreamParityTestCase(plain=CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=CLAUDE_OPUS_5_5_STREAM), StreamParityTestCase(plain=CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_OPUS_5_5_MESSAGES_STREAM), StreamParityTestCase(plain=CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_SONNET_5_5_MESSAGES_STREAM), StreamParityTestCase(plain=CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, streamed=CLAUDE_HAIKU_5_5_MESSAGES_STREAM), StreamParityTestCase(plain=CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, streamed=CLAUDE_FABLE_5_1_MESSAGES_STREAM), + StreamParityTestCase(plain=CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT, streamed=CLAUDE_FABLE_5_2_MESSAGES_STREAM), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( @@ -7220,6 +7797,9 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( BridgeCostParityTestCase( native=CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, bridged=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON ), + BridgeCostParityTestCase( + native=CLAUDE_FABLE_5_2_MESSAGES_INPUT_TEXT, bridged=CLAUDE_FABLE_5_2_RESPONSES_BRIDGED_JSON + ), BridgeCostParityTestCase( native=CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, bridged=CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON ), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index ffdb6752bf2..37dc86e2601 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -1398,6 +1398,19 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "output_cost_per_token": 8e-05, "supports_function_calling": True, }, + "claude-fable-5-2": { + "cache_creation_input_token_cost": 1.1e-05, + "cache_creation_input_token_cost_above_1hr": 1.76e-05, + "cache_read_input_token_cost": 2.2e-07, + "input_cost_per_token": 8.8e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-05, + "supports_function_calling": True, + }, "us.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, diff --git a/tests/integration/providers/test_anthropic_fable_capability_flags_wire.py b/tests/integration/providers/test_anthropic_fable_capability_flags_wire.py new file mode 100644 index 00000000000..0b140e4626f --- /dev/null +++ b/tests/integration/providers/test_anthropic_fable_capability_flags_wire.py @@ -0,0 +1,208 @@ +"""Claude Fable 5.2 over Anthropic direct. Its cost-map row says `supports_sampling_params: false`, +`supports_forced_tool_use: false` and `thinking_always_on: true`, and Anthropic answers 400 to a +request that ignores any of them, so the proxy has to rewrite or refuse those requests before they +leave. The scripted upstream here is what proves the outbound body, a leaked field is invisible in +the response.""" + +import json +import uuid +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_MODEL: Final = "claude-fable-5-2" +_API_KEY: Final = "synthetic-anthropic-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_WEATHER_TOOL: Final[JsonValue] = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +_NAMED_TOOL_CHOICE: Final[JsonValue] = {"type": "function", "function": {"name": "get_weather"}} + + +def _anthropic_reply(identity: str, usage: dict[str, JsonValue] | None = None) -> bytes: + return json.dumps( + { + "id": identity, + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted-signature"}, + {"type": "text", "text": "done"}, + ], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": usage or {"input_tokens": 12, "output_tokens": 3}, + } + ).encode() + + +def _chat_body(model: str, **extra: JsonValue) -> dict[str, JsonValue]: + return { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "cache": {"no-cache": True}, + **extra, + } + + +def _outbound(request: Request) -> dict[str, JsonValue]: + assert request.method == "POST" and request.target == "/v1/messages", (request.method, request.target) + assert request.headers["x-api-key"] == _API_KEY + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == _MODEL, body + return body + + +def test_drop_params_removes_temperature_and_top_p_before_anthropic(gateway: Gateway) -> None: + identity: Final = f"fable-sampling-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _outbound(request) + assert "temperature" not in body and "top_p" not in body, body + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, drop_params=True + ) + response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, temperature=0.2, top_p=0.5)) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_sampling_params_are_refused_before_reaching_anthropic_without_drop_params(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + raise AssertionError(f"a request with sampling params reached Anthropic: {request.body!r}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, temperature=0.2)) + assert response.status_code == 400, response.text + assert "temperature" in response.text and "drop_params" in response.text, response.text + assert wire.drain() == () + + +def test_drop_params_downgrades_forced_and_named_tool_choice_to_auto(gateway: Gateway) -> None: + identity: Final = f"fable-tool-choice-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _outbound(request) + assert body["tool_choice"] == {"type": "auto"}, body + tools: Final = body["tools"] + assert isinstance(tools, list) and len(tools) == 1, body + first: Final = tools[0] + assert isinstance(first, dict) and first["name"] == "get_weather", body + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, drop_params=True + ) + responses: Final = tuple( + gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, tools=[_WEATHER_TOOL], tool_choice=tool_choice) + ) + for tool_choice in ("required", _NAMED_TOOL_CHOICE) + ) + assert [response.status_code for response in responses] == [200, 200], [r.text for r in responses] + assert len(wire.drain()) == 2 + + +def test_forced_tool_choice_is_refused_before_reaching_anthropic_without_drop_params(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + raise AssertionError(f"a forced tool_choice reached Anthropic: {request.body!r}") + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, tools=[_WEATHER_TOOL], tool_choice="required") + ) + assert response.status_code == 400, response.text + assert "tool" in response.text and "drop_params" in response.text, response.text + assert wire.drain() == () + + +def test_thinking_disabled_is_dropped_for_an_always_on_model(gateway: Gateway) -> None: + identity: Final = f"fable-thinking-off-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _outbound(request) + assert "thinking" not in body, body + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", "/v1/chat/completions", _chat_body(model, thinking={"type": "disabled"}) + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + choices: Final = payload["choices"] + assert isinstance(choices, list) and isinstance(choices[0], dict), response.text + message: Final = choices[0]["message"] + assert isinstance(message, dict) and message["reasoning_content"] == "scripted thought", response.text + assert len(wire.drain()) == 1 + + +def test_reasoning_effort_becomes_adaptive_thinking_with_output_config_effort(gateway: Gateway) -> None: + identity: Final = f"fable-effort-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _outbound(request) + assert body["thinking"] == {"type": "adaptive", "display": "summarized"}, body + assert body["output_config"] == {"effort": "max"}, body + assert "budget_tokens" not in json.dumps(body), body + return Reply(body=_anthropic_reply(identity)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, reasoning_effort="max")) + assert response.status_code == 200, response.text + assert len(wire.drain()) == 1 + + +def test_messages_one_hour_cache_control_reaches_anthropic_and_its_usage_comes_back(gateway: Gateway) -> None: + identity: Final = f"fable-cache-1h-{uuid.uuid4().hex}" + policy: Final = f"policy {identity}" + upstream_usage: Final[dict[str, JsonValue]] = { + "input_tokens": 12, + "output_tokens": 3, + "cache_creation_input_tokens": 9216, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9216}, + } + + def respond(request: Request) -> Reply: + body: Final = _outbound(request) + assert body["system"] == [ + {"type": "text", "text": policy, "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ], body + return Reply(body=_anthropic_reply(identity, upstream_usage)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 16, + "system": [{"type": "text", "text": policy, "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + "messages": [{"role": "user", "content": "hi"}], + "cache": {"no-cache": True}, + }, + ) + assert response.status_code == 200, response.text + usage: Final = _JSON_OBJECT.validate_json(response.content)["usage"] + assert isinstance(usage, dict), response.text + assert usage["cache_creation_input_tokens"] == 9216, usage + assert usage["cache_creation"] == {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 9216}, usage + assert len(wire.drain()) == 1