mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-12 03:48:56 +00:00
test(integration): streamed request bills the same SpendLogs row as the non-streamed request (#45730)
* test(integration): streamed request bills the same SpendLogs row as the non-streamed request Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): write the stream parity cells as StreamParityTestCase literals Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): express stream parity cells as CostTrackingTestCase pairs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): stream parity asserts exactly one SpendLogs row per leg after a settle window Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): stream parity forces a spend-log flush with a /v1/responses row instead of a settle sleep Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): stream parity covers messages -> chat and messages -> converse bridges Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): stream parity bridges use xai/grok-4.7 and us.anthropic.claude-opus-5-5 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): use latest frontier models in stream parity cells (gpt-5.6, claude-opus-5-5) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): switch OpenAI parity cells to gpt-6.1-sol and the chat -> responses bridge to gpt-5.5-pro Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
6fe46522d3
commit
ff60d27b60
11 changed files with 857 additions and 11 deletions
|
|
@ -642,20 +642,20 @@
|
|||
"gpt-realtime-mini-2025-12-15": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "realtime",
|
||||
"input_cost_per_token": 6.0e-07,
|
||||
"input_cost_per_token": 6e-07,
|
||||
"output_cost_per_token": 2.4e-06,
|
||||
"input_cost_per_audio_token": 1.0e-05,
|
||||
"cache_read_input_token_cost": 6.0e-08,
|
||||
"cache_read_input_audio_token_cost": 3.0e-07,
|
||||
"output_cost_per_audio_token": 2.0e-05
|
||||
"input_cost_per_audio_token": 1e-05,
|
||||
"cache_read_input_token_cost": 6e-08,
|
||||
"cache_read_input_audio_token_cost": 3e-07,
|
||||
"output_cost_per_audio_token": 2e-05
|
||||
},
|
||||
"gpt-realtime-2.1": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "realtime",
|
||||
"input_cost_per_token": 4.0e-06,
|
||||
"input_cost_per_token": 4e-06,
|
||||
"input_cost_per_audio_token": 3.2e-05,
|
||||
"cache_read_input_token_cost": 4.0e-07,
|
||||
"cache_read_input_audio_token_cost": 4.0e-07,
|
||||
"cache_read_input_token_cost": 4e-07,
|
||||
"cache_read_input_audio_token_cost": 4e-07,
|
||||
"output_cost_per_token": 2.4e-05,
|
||||
"output_cost_per_audio_token": 6.4e-05
|
||||
},
|
||||
|
|
@ -769,6 +769,52 @@
|
|||
"search_context_size_medium": 0.014
|
||||
},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"xai/grok-4.7": {
|
||||
"litellm_provider": "xai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 6e-06,
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"max_tokens": 500000,
|
||||
"max_input_tokens": 500000,
|
||||
"max_output_tokens": 500000,
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"us.anthropic.claude-opus-5-5": {
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 4.4e-06,
|
||||
"output_cost_per_token": 2.2e-05,
|
||||
"cache_read_input_token_cost": 2.2e-07,
|
||||
"cache_creation_input_token_cost": 5.5e-06,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"claude-opus-5-5": {
|
||||
"litellm_provider": "anthropic",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 4e-06,
|
||||
"output_cost_per_token": 2e-05,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"supports_function_calling": true
|
||||
},
|
||||
"gpt-6.1-sol": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
"cache_read_input_token_cost": 1e-07,
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"supports_function_calling": true
|
||||
}
|
||||
},
|
||||
"cases": [
|
||||
|
|
@ -29687,7 +29733,13 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"expected": {"spend": 0.0025118, "input_cost": 0.0020792, "output_cost": 0.0004326, "prompt_tokens": 1840, "completion_tokens": 412}
|
||||
"expected": {
|
||||
"spend": 0.0025118,
|
||||
"input_cost": 0.0020792,
|
||||
"output_cost": 0.0004326,
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "deepseek-deepseek-v4-chat-prompt_cache_hit",
|
||||
|
|
@ -32102,6 +32154,190 @@
|
|||
"prompt_tokens": 100,
|
||||
"completion_tokens": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "xai-grok-4.7-input_text",
|
||||
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
"model": "xai/grok-4.7",
|
||||
"request": {
|
||||
"model": "$MODEL",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Say hello."
|
||||
}
|
||||
],
|
||||
"stream": false,
|
||||
"allowed_openai_params": []
|
||||
},
|
||||
"response": {
|
||||
"content_type": "application/json",
|
||||
"body": {
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion",
|
||||
"model": "grok-4.7",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Hello."
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412,
|
||||
"total_tokens": 2252
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"spend": 0.006152,
|
||||
"input_cost": 0.00368,
|
||||
"output_cost": 0.002472,
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "us.anthropic.claude-opus-5-5-input_text",
|
||||
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
"model": "us.anthropic.claude-opus-5-5",
|
||||
"request": {
|
||||
"model": "$MODEL",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Say hello."
|
||||
}
|
||||
],
|
||||
"stream": false,
|
||||
"allowed_openai_params": []
|
||||
},
|
||||
"response": {
|
||||
"content_type": "application/json",
|
||||
"body": {
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"text": "Hello."
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {
|
||||
"inputTokens": 1840,
|
||||
"outputTokens": 412,
|
||||
"totalTokens": 2252
|
||||
},
|
||||
"metrics": {
|
||||
"latencyMs": 42
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"spend": 0.01716,
|
||||
"input_cost": 0.008096,
|
||||
"output_cost": 0.009064,
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "claude-opus-5-5-input_text",
|
||||
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
"model": "claude-opus-5-5",
|
||||
"request": {
|
||||
"model": "$MODEL",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Say hello."
|
||||
}
|
||||
],
|
||||
"stream": false,
|
||||
"allowed_openai_params": []
|
||||
},
|
||||
"response": {
|
||||
"content_type": "application/json",
|
||||
"body": {
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-5-5",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Hello."
|
||||
}
|
||||
],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": null,
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"output_tokens": 412
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"spend": 0.0156,
|
||||
"input_cost": 0.00736,
|
||||
"output_cost": 0.00824,
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gpt-6.1-sol-input_text",
|
||||
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
"model": "gpt-6.1-sol",
|
||||
"request": {
|
||||
"model": "$MODEL",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Say hello."
|
||||
}
|
||||
],
|
||||
"stream": false,
|
||||
"allowed_openai_params": []
|
||||
},
|
||||
"response": {
|
||||
"content_type": "application/json",
|
||||
"body": {
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-6.1-sol",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Hello."
|
||||
},
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412,
|
||||
"total_tokens": 2252
|
||||
}
|
||||
}
|
||||
},
|
||||
"expected": {
|
||||
"spend": 0.0078,
|
||||
"input_cost": 0.00368,
|
||||
"output_cost": 0.00412,
|
||||
"prompt_tokens": 1840,
|
||||
"completion_tokens": 412
|
||||
}
|
||||
}
|
||||
],
|
||||
"batch_cases": [
|
||||
|
|
@ -32149,8 +32385,8 @@
|
|||
],
|
||||
"expected": {
|
||||
"spend": 0.000126,
|
||||
"input_cost": 0.000056,
|
||||
"output_cost": 0.00007,
|
||||
"input_cost": 5.6e-05,
|
||||
"output_cost": 7e-05,
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 10,
|
||||
"cost_header": false
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
SseResponse,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.case import COVERS, MODEL, REQUEST_ID, StreamParityTestCase, sse_frames
|
||||
|
||||
CLAUDE_OPUS_5_5_MESSAGES: Final = CostTrackingTestCase(
|
||||
name="claude-opus-5-5-messages-parity",
|
||||
covers=COVERS,
|
||||
model="claude-opus-5-5",
|
||||
endpoint="/v1/messages",
|
||||
deployment=Deployment(input_cost_per_token=0.001, output_cost_per_token=0.002),
|
||||
request={
|
||||
"model": MODEL,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": f"msg_{REQUEST_ID}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-5-5",
|
||||
"content": [{"type": "text", "text": "Hello."}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 30, "output_tokens": 40},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(spend=0.11, input_cost=0.03, output_cost=0.08, prompt_tokens=30, completion_tokens=40),
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_STREAM_RESPONSE: Final = SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": f"msg_{REQUEST_ID}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-5-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 30, "output_tokens": 1},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Hello."}},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 40},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
),
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_MESSAGES_PARITY: Final = StreamParityTestCase(
|
||||
plain=CLAUDE_OPUS_5_5_MESSAGES,
|
||||
streamed=CLAUDE_OPUS_5_5_MESSAGES.model_copy(
|
||||
update={
|
||||
"name": "claude-opus-5-5-messages-parity-stream",
|
||||
"request": {**CLAUDE_OPUS_5_5_MESSAGES.request, "stream": True},
|
||||
"response": CLAUDE_OPUS_5_5_STREAM_RESPONSE,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_CHAT_COMPLETIONS: Final = CLAUDE_OPUS_5_5_MESSAGES.model_copy(
|
||||
update={
|
||||
"name": "claude-opus-5-5-chat-parity",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"request": {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_CHAT_COMPLETIONS_PARITY: Final = StreamParityTestCase(
|
||||
plain=CLAUDE_OPUS_5_5_CHAT_COMPLETIONS,
|
||||
streamed=CLAUDE_OPUS_5_5_CHAT_COMPLETIONS.model_copy(
|
||||
update={
|
||||
"name": "claude-opus-5-5-chat-parity-stream",
|
||||
"request": {
|
||||
**CLAUDE_OPUS_5_5_CHAT_COMPLETIONS.request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
"response": CLAUDE_OPUS_5_5_STREAM_RESPONSE,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_RESPONSES: Final = CLAUDE_OPUS_5_5_MESSAGES.model_copy(
|
||||
update={
|
||||
"name": "claude-opus-5-5-responses-parity",
|
||||
"endpoint": "/v1/responses",
|
||||
"request": {"model": MODEL, "input": "Say hello.", "cache": {"no-cache": True}},
|
||||
}
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_RESPONSES_PARITY: Final = StreamParityTestCase(
|
||||
plain=CLAUDE_OPUS_5_5_RESPONSES,
|
||||
streamed=CLAUDE_OPUS_5_5_RESPONSES.model_copy(
|
||||
update={
|
||||
"name": "claude-opus-5-5-responses-parity-stream",
|
||||
"request": {**CLAUDE_OPUS_5_5_RESPONSES.request, "stream": True},
|
||||
"response": CLAUDE_OPUS_5_5_STREAM_RESPONSE,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
"""/v1/messages on a Bedrock Converse deployment: the messages -> converse bridge."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
EventStreamEvent,
|
||||
EventStreamResponse,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.case import COVERS, MODEL, StreamParityTestCase
|
||||
|
||||
_USAGE: Final = {"inputTokens": 30, "outputTokens": 40, "totalTokens": 70}
|
||||
|
||||
CLAUDE_OPUS_5_5_CONVERSE_MESSAGES: Final = CostTrackingTestCase(
|
||||
name="bedrock-converse-claude-opus-5-5-messages-parity",
|
||||
covers=COVERS,
|
||||
model="us.anthropic.claude-opus-5-5",
|
||||
endpoint="/v1/messages",
|
||||
deployment=Deployment(input_cost_per_token=0.001, output_cost_per_token=0.002),
|
||||
request={
|
||||
"model": MODEL,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "Hello."}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": _USAGE,
|
||||
"metrics": {"latencyMs": 1},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(spend=0.11, input_cost=0.03, output_cost=0.08, prompt_tokens=30, completion_tokens=40),
|
||||
)
|
||||
|
||||
CLAUDE_OPUS_5_5_CONVERSE_MESSAGES_PARITY: Final = StreamParityTestCase(
|
||||
plain=CLAUDE_OPUS_5_5_CONVERSE_MESSAGES,
|
||||
streamed=CLAUDE_OPUS_5_5_CONVERSE_MESSAGES.model_copy(
|
||||
update={
|
||||
"name": "bedrock-converse-claude-opus-5-5-messages-parity-stream",
|
||||
"request": {**CLAUDE_OPUS_5_5_CONVERSE_MESSAGES.request, "stream": True},
|
||||
"response": EventStreamResponse(
|
||||
content_type="application/vnd.amazon.eventstream",
|
||||
framing="converse",
|
||||
events=(
|
||||
EventStreamEvent(event_type="messageStart", payload={"role": "assistant"}),
|
||||
EventStreamEvent(
|
||||
event_type="contentBlockDelta", payload={"delta": {"text": "Hello."}, "contentBlockIndex": 0}
|
||||
),
|
||||
EventStreamEvent(event_type="contentBlockStop", payload={"contentBlockIndex": 0}),
|
||||
EventStreamEvent(event_type="messageStop", payload={"stopReason": "end_turn"}),
|
||||
EventStreamEvent(event_type="metadata", payload={"usage": _USAGE, "metrics": {"latencyMs": 1}}),
|
||||
),
|
||||
),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
SseResponse,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.bases.openai_responses import (
|
||||
GPT_6_1_SOL_RESPONSES,
|
||||
GPT_6_1_SOL_RESPONSES_PARITY_STREAM_RESPONSE,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.case import COVERS, MODEL, REQUEST_ID, StreamParityTestCase, sse_frames
|
||||
|
||||
_USAGE: Final = {"prompt_tokens": 30, "completion_tokens": 40, "total_tokens": 70}
|
||||
_CHUNK: Final = {
|
||||
"id": f"chatcmpl-{REQUEST_ID}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-6.1-sol",
|
||||
}
|
||||
|
||||
GPT_6_1_SOL_CHAT_COMPLETIONS: Final = CostTrackingTestCase(
|
||||
name="gpt-6.1-sol-chat-parity",
|
||||
covers=COVERS,
|
||||
model="gpt-6.1-sol",
|
||||
endpoint="/v1/chat/completions",
|
||||
deployment=Deployment(input_cost_per_token=0.001, output_cost_per_token=0.002),
|
||||
request={"model": MODEL, "messages": [{"role": "user", "content": "Say hello."}], "cache": {"no-cache": True}},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": f"chatcmpl-{REQUEST_ID}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-6.1-sol",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "finish_reason": "stop"}],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(spend=0.11, input_cost=0.03, output_cost=0.08, prompt_tokens=30, completion_tokens=40),
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_CHAT_COMPLETIONS_PARITY: Final = StreamParityTestCase(
|
||||
plain=GPT_6_1_SOL_CHAT_COMPLETIONS,
|
||||
streamed=GPT_6_1_SOL_CHAT_COMPLETIONS.model_copy(
|
||||
update={
|
||||
"name": "gpt-6.1-sol-chat-parity-stream",
|
||||
"request": {
|
||||
**GPT_6_1_SOL_CHAT_COMPLETIONS.request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
"response": SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{**_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello."}}]},
|
||||
{**_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**_CHUNK, "choices": [], "usage": _USAGE},
|
||||
done=True,
|
||||
),
|
||||
),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_MESSAGES: Final = GPT_6_1_SOL_RESPONSES.model_copy(
|
||||
update={
|
||||
"name": "gpt-6.1-sol-messages-parity",
|
||||
"model": "gpt-6.1-sol",
|
||||
"endpoint": "/v1/messages",
|
||||
"request": {
|
||||
"model": MODEL,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_MESSAGES_PARITY: Final = StreamParityTestCase(
|
||||
plain=GPT_6_1_SOL_MESSAGES,
|
||||
streamed=GPT_6_1_SOL_MESSAGES.model_copy(
|
||||
update={
|
||||
"name": "gpt-6.1-sol-messages-parity-stream",
|
||||
"request": {**GPT_6_1_SOL_MESSAGES.request, "stream": True},
|
||||
"response": GPT_6_1_SOL_RESPONSES_PARITY_STREAM_RESPONSE,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
SseResponse,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.case import COVERS, MODEL, REQUEST_ID, StreamParityTestCase, sse_frames
|
||||
|
||||
_ITEM: Final = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Hello.", "annotations": []}],
|
||||
}
|
||||
_RESPONSE: Final = {
|
||||
"id": f"resp_{REQUEST_ID}",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-6.1-sol",
|
||||
"output": [_ITEM],
|
||||
"usage": {"input_tokens": 30, "output_tokens": 40, "total_tokens": 70},
|
||||
}
|
||||
|
||||
GPT_6_1_SOL_RESPONSES: Final = CostTrackingTestCase(
|
||||
name="gpt-6.1-sol-responses-parity",
|
||||
covers=COVERS,
|
||||
model="gpt-6.1-sol",
|
||||
endpoint="/v1/responses",
|
||||
deployment=Deployment(input_cost_per_token=0.001, output_cost_per_token=0.002),
|
||||
request={"model": MODEL, "input": "Say hello.", "cache": {"no-cache": True}},
|
||||
response=JsonResponse(content_type="application/json", body=_RESPONSE),
|
||||
expected=ExactExpected(spend=0.11, input_cost=0.03, output_cost=0.08, prompt_tokens=30, completion_tokens=40),
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_RESPONSES_PARITY_STREAM_RESPONSE: Final = SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"type": "response.created",
|
||||
"sequence_number": 0,
|
||||
"response": {**_RESPONSE, "status": "in_progress", "output": []},
|
||||
},
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"sequence_number": 1,
|
||||
"output_index": 0,
|
||||
"item": {**_ITEM, "status": "in_progress", "content": []},
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 2,
|
||||
"item_id": "msg_1",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "Hello.",
|
||||
},
|
||||
{"type": "response.output_item.done", "sequence_number": 3, "output_index": 0, "item": _ITEM},
|
||||
{"type": "response.completed", "sequence_number": 4, "response": _RESPONSE},
|
||||
),
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_RESPONSES_PARITY: Final = StreamParityTestCase(
|
||||
plain=GPT_6_1_SOL_RESPONSES,
|
||||
streamed=GPT_6_1_SOL_RESPONSES.model_copy(
|
||||
update={
|
||||
"name": "gpt-6.1-sol-responses-parity-stream",
|
||||
"request": {**GPT_6_1_SOL_RESPONSES.request, "stream": True},
|
||||
"response": GPT_6_1_SOL_RESPONSES_PARITY_STREAM_RESPONSE,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
_PRO_RESPONSE: Final = {**_RESPONSE, "model": "gpt-5.5-pro"}
|
||||
GPT_5_5_PRO_STREAM_RESPONSE: Final = GPT_6_1_SOL_RESPONSES_PARITY_STREAM_RESPONSE.model_copy(
|
||||
update={
|
||||
"frames": tuple(
|
||||
frame.replace('"model": "gpt-6.1-sol"', '"model": "gpt-5.5-pro"')
|
||||
for frame in GPT_6_1_SOL_RESPONSES_PARITY_STREAM_RESPONSE.frames
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
GPT_5_5_PRO_CHAT_COMPLETIONS: Final = GPT_6_1_SOL_RESPONSES.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.5-pro-chat-parity",
|
||||
"model": "gpt-5.5-pro",
|
||||
"response": JsonResponse(content_type="application/json", body=_PRO_RESPONSE),
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"request": {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
GPT_5_5_PRO_CHAT_COMPLETIONS_PARITY: Final = StreamParityTestCase(
|
||||
plain=GPT_5_5_PRO_CHAT_COMPLETIONS,
|
||||
streamed=GPT_5_5_PRO_CHAT_COMPLETIONS.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.5-pro-chat-parity-stream",
|
||||
"request": {
|
||||
**GPT_5_5_PRO_CHAT_COMPLETIONS.request,
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
"response": GPT_5_5_PRO_STREAM_RESPONSE,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
"""/v1/messages on a chat-completions provider: the messages -> chat/completions bridge."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
SseResponse,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.case import COVERS, MODEL, REQUEST_ID, StreamParityTestCase, sse_frames
|
||||
|
||||
_USAGE: Final = {"prompt_tokens": 30, "completion_tokens": 40, "total_tokens": 70}
|
||||
_CHUNK: Final = {
|
||||
"id": f"chatcmpl-{REQUEST_ID}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "grok-4.7",
|
||||
}
|
||||
|
||||
GROK_4_7_MESSAGES: Final = CostTrackingTestCase(
|
||||
name="grok-4.7-messages-parity",
|
||||
covers=COVERS,
|
||||
model="xai/grok-4.7",
|
||||
endpoint="/v1/messages",
|
||||
deployment=Deployment(input_cost_per_token=0.001, output_cost_per_token=0.002),
|
||||
request={
|
||||
"model": MODEL,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "Say hello."}],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": f"chatcmpl-{REQUEST_ID}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "grok-4.7",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "finish_reason": "stop"}],
|
||||
"usage": _USAGE,
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(spend=0.11, input_cost=0.03, output_cost=0.08, prompt_tokens=30, completion_tokens=40),
|
||||
)
|
||||
|
||||
GROK_4_7_MESSAGES_PARITY: Final = StreamParityTestCase(
|
||||
plain=GROK_4_7_MESSAGES,
|
||||
streamed=GROK_4_7_MESSAGES.model_copy(
|
||||
update={
|
||||
"name": "grok-4.7-messages-parity-stream",
|
||||
"request": {**GROK_4_7_MESSAGES.request, "stream": True},
|
||||
"response": SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{**_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello."}}]},
|
||||
{**_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**_CHUNK, "choices": [], "usage": _USAGE},
|
||||
done=True,
|
||||
),
|
||||
),
|
||||
}
|
||||
),
|
||||
)
|
||||
44
tests/integration/cost_calculation/stream_parity/case.py
Normal file
44
tests/integration/cost_calculation/stream_parity/case.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase
|
||||
from pydantic import JsonValue
|
||||
|
||||
COVERS: Final = "quota_management.spend_tracking.scripted_wire.logs_cost"
|
||||
REQUEST_ID: Final = "$REQUEST_ID"
|
||||
MODEL: Final = "$MODEL"
|
||||
|
||||
|
||||
def sse_frames(*events: Mapping[str, JsonValue], done: bool = False) -> tuple[str, ...]:
|
||||
frames: Final = tuple(
|
||||
f"event: {event['type']}\ndata: {json.dumps(event)}" if "type" in event else f"data: {json.dumps(event)}"
|
||||
for event in events
|
||||
)
|
||||
return (*frames, "data: [DONE]") if done else frames
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class StreamParityTestCase:
|
||||
"""The same request as two CostTrackingTestCases, `plain` answered with JSON and `streamed` answered with SSE,
|
||||
that must both bill the one `expected` row."""
|
||||
|
||||
plain: CostTrackingTestCase
|
||||
streamed: CostTrackingTestCase
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.plain.expected != self.streamed.expected:
|
||||
raise ValueError(f"{self.id}: plain and streamed cases expect different rows")
|
||||
if self.streamed.request.get("stream") is not True or self.plain.request.get("stream") is True:
|
||||
raise ValueError(f"{self.id}: streamed case must set stream: true and the plain case must not")
|
||||
if (self.plain.model, self.plain.endpoint, self.plain.deployment) != (
|
||||
self.streamed.model,
|
||||
self.streamed.endpoint,
|
||||
self.streamed.deployment,
|
||||
):
|
||||
raise ValueError(f"{self.id}: plain and streamed cases must share model, endpoint and deployment")
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return self.plain.name
|
||||
70
tests/integration/cost_calculation/stream_parity/runner.py
Normal file
70
tests/integration/cost_calculation/stream_parity/runner.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
from collections import Counter
|
||||
from hashlib import sha256
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway, Scenario
|
||||
from integration.cost_calculation.assertions import assert_exact
|
||||
from integration.cost_calculation.conftest import (
|
||||
poll_rows,
|
||||
poll_rows_where,
|
||||
read_rows_now,
|
||||
register_scenario_deployment,
|
||||
)
|
||||
from integration.cost_calculation.cost_tracking_case import ExactExpected
|
||||
from integration.cost_calculation.stream_parity.bases import openai_responses
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase
|
||||
|
||||
FLUSH_CASE: Final = openai_responses.GPT_6_1_SOL_RESPONSES
|
||||
|
||||
|
||||
def flush_spend_logs(scenario: Scenario, gateway: Gateway, key: str) -> str:
|
||||
"""A /v1/responses row wakes the spend-log writer at once, and the queue is FIFO, so every row queued before it lands."""
|
||||
flush: Final = register_scenario_deployment(scenario, FLUSH_CASE, "flush", key)
|
||||
response: Final = gateway.request(
|
||||
"POST", FLUSH_CASE.endpoint, {**FLUSH_CASE.request, "model": flush.model_name}, key=key
|
||||
)
|
||||
assert response.status_code == 200, ("flush", response.text[:400])
|
||||
poll_rows_where(key, 1, lambda row: row.model_id == flush.identity)
|
||||
return flush.identity
|
||||
|
||||
|
||||
def assert_stream_parity(case: StreamParityTestCase, gateway: Gateway) -> None:
|
||||
expected: Final = case.plain.expected
|
||||
assert isinstance(expected, ExactExpected), case.id
|
||||
with gateway.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
deployments: Final = {
|
||||
leg.name: register_scenario_deployment(scenario, leg, sha256(leg.name.encode()).hexdigest()[:12], key)
|
||||
for leg in (case.plain, case.streamed)
|
||||
}
|
||||
responses: Final = {
|
||||
leg.name: gateway.request(
|
||||
"POST", leg.endpoint, {**leg.request, "model": deployments[leg.name].model_name}, key=key
|
||||
)
|
||||
for leg in (case.plain, case.streamed)
|
||||
}
|
||||
for leg in (case.plain, case.streamed):
|
||||
assert responses[leg.name].status_code == 200, (leg.name, responses[leg.name].text[:400])
|
||||
|
||||
poll_rows(key, 2)
|
||||
flush_identity: Final = flush_spend_logs(scenario, gateway, key)
|
||||
rows: Final = read_rows_now(key)
|
||||
rows_per_deployment: Final = Counter(row.model_id for row in rows)
|
||||
expected_rows: Final = {deployments[leg.name].identity: 1 for leg in (case.plain, case.streamed)}
|
||||
assert rows_per_deployment == {**expected_rows, flush_identity: 1}, (
|
||||
f"{case.id}: expected exactly one SpendLogs row per leg plus the flush row, got {dict(rows_per_deployment)}",
|
||||
rows,
|
||||
)
|
||||
billed: Final = {
|
||||
leg.name: next(row for row in rows if row.model_id == deployments[leg.name].identity)
|
||||
for leg in (case.plain, case.streamed)
|
||||
}
|
||||
for leg in (case.plain, case.streamed):
|
||||
assert_exact(leg.name, leg.response.content_type, expected, billed[leg.name], responses[leg.name])
|
||||
plain: Final = billed[case.plain.name]
|
||||
streamed: Final = billed[case.streamed.name]
|
||||
assert (streamed.spend, streamed.prompt_tokens, streamed.completion_tokens) == (
|
||||
plain.spend,
|
||||
plain.prompt_tokens,
|
||||
plain.completion_tokens,
|
||||
), (plain, streamed)
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
"""A streamed request bills exactly what the same request bills without streaming.
|
||||
|
||||
Streaming reconstructs the response, its usage and its model name from chunks after the fact, in a separate code
|
||||
path from the non-streamed one (LIT-6241, LIT-6872, LIT-7632, LIT-7729, LIT-9044, LIT-9065). Each case is a plain and
|
||||
a streamed CostTrackingTestCase for one request, sent through the same key, and both LiteLLM_SpendLogs rows must
|
||||
bill the case's expected row.
|
||||
|
||||
Three cells are the native routes (chat, /v1/messages, /v1/responses); the rest are the bridges that translate one
|
||||
API into another before the provider call, each with its own stream reassembly
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration.cost_calculation.stream_parity.bases import anthropic, bedrock, openai, openai_responses, xai
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase
|
||||
from integration.cost_calculation.stream_parity.runner import assert_stream_parity
|
||||
|
||||
NATIVE: Final = (
|
||||
openai.GPT_6_1_SOL_CHAT_COMPLETIONS_PARITY,
|
||||
anthropic.CLAUDE_OPUS_5_5_MESSAGES_PARITY,
|
||||
openai_responses.GPT_6_1_SOL_RESPONSES_PARITY,
|
||||
)
|
||||
BRIDGES: Final = (
|
||||
xai.GROK_4_7_MESSAGES_PARITY,
|
||||
openai.GPT_6_1_SOL_MESSAGES_PARITY,
|
||||
bedrock.CLAUDE_OPUS_5_5_CONVERSE_MESSAGES_PARITY,
|
||||
openai_responses.GPT_5_5_PRO_CHAT_COMPLETIONS_PARITY,
|
||||
anthropic.CLAUDE_OPUS_5_5_RESPONSES_PARITY,
|
||||
)
|
||||
CASES: Final = (*NATIVE, *BRIDGES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=[case.id for case in CASES])
|
||||
@pytest.mark.timeout(180)
|
||||
def test_streamed_request_bills_the_same_row_as_the_non_streamed_request(
|
||||
case: StreamParityTestCase, gateway: Gateway
|
||||
) -> None:
|
||||
assert_stream_parity(case, gateway)
|
||||
Loading…
Add table
Reference in a new issue