mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
test(cost): cover custom deployment aliases, off-peak pricing, bridge usage shapes and duplicate spend rows (#46708)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu
GitHub
parent
ee63cb91d0
commit
c3da442a25
16 changed files with 311 additions and 26 deletions
|
|
@ -24,7 +24,7 @@ from _fake_openai_endpoint_server import (
|
|||
moderations,
|
||||
triton_embeddings,
|
||||
)
|
||||
from integration.cost_calculation.cost_map import COST_MAP_ENTRIES
|
||||
from integration.cost_calculation.cost_map import COST_MAP_ENTRIES, FALLBACK_GENERALIZATIONS
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
BinaryResponse,
|
||||
EventStreamEvent,
|
||||
|
|
@ -275,7 +275,7 @@ class Provider:
|
|||
return JSONResponse({"deleted": deleted}, status_code=200 if deleted else 404)
|
||||
|
||||
async def cost_map(self, _request: Request) -> Response:
|
||||
return JSONResponse(COST_MAP_ENTRIES)
|
||||
return JSONResponse({**COST_MAP_ENTRIES, "fallback_generalizations": FALLBACK_GENERALIZATIONS})
|
||||
|
||||
async def oauth_token(self, _request: Request) -> Response:
|
||||
return JSONResponse(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase, ExactExpected
|
||||
from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase, Deployment, ExactExpected
|
||||
from pydantic import JsonValue
|
||||
|
||||
ROW_FIELDS: Final = frozenset({"spend", "prompt_tokens", "completion_tokens"})
|
||||
_EXPECTED_FIELDS: Final = frozenset(ExactExpected.model_fields)
|
||||
|
|
@ -32,6 +36,7 @@ class BridgeCostParityTestCase:
|
|||
native: CostTrackingTestCase
|
||||
bridged: CostTrackingTestCase
|
||||
quirk: ProviderCostQuirk | None = None
|
||||
known_bug: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.native.endpoint == self.bridged.endpoint:
|
||||
|
|
@ -61,3 +66,75 @@ class BridgeCostParityTestCase:
|
|||
@property
|
||||
def id(self) -> str:
|
||||
return self.bridged.name
|
||||
|
||||
|
||||
BridgeEndpoint = Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
|
||||
_ENDPOINT_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"/v1/chat/completions": "chat_bridged", "/v1/responses": "responses_bridged", "/v1/messages": "messages_bridged"}
|
||||
)
|
||||
|
||||
|
||||
def _prompt_text(request: Mapping[str, JsonValue]) -> str:
|
||||
"""The user prompt of a plain base request, whichever client shape it was written in."""
|
||||
prompt: Final = request.get("input")
|
||||
if isinstance(prompt, str):
|
||||
return prompt
|
||||
messages: Final = request.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
raise ValueError("a bridge base request needs a string input or a messages list")
|
||||
for message in reversed(messages):
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
continue
|
||||
content: Final = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(str(part["text"]) for part in content if isinstance(part, dict) and "text" in part)
|
||||
raise ValueError("a bridge base request needs a user message")
|
||||
|
||||
|
||||
def _bridged_request(base: CostTrackingTestCase, endpoint: BridgeEndpoint) -> dict[str, JsonValue]:
|
||||
streamed: Final = base.request.get("stream") is True
|
||||
text: Final = _prompt_text(base.request)
|
||||
if endpoint == "/v1/responses":
|
||||
return {"model": "$MODEL", "input": text, **({"stream": True} if streamed else {})}
|
||||
messages: Final[list[JsonValue]] = [{"role": "user", "content": text}]
|
||||
if endpoint == "/v1/messages":
|
||||
return {"model": "$MODEL", "max_tokens": 412, "messages": messages, **({"stream": True} if streamed else {})}
|
||||
return {
|
||||
"model": "$MODEL",
|
||||
"messages": messages,
|
||||
**({"stream": True, "stream_options": {"include_usage": True}} if streamed else {}),
|
||||
}
|
||||
|
||||
|
||||
def bridge_pairs(
|
||||
bases: Sequence[CostTrackingTestCase],
|
||||
endpoints: Sequence[BridgeEndpoint],
|
||||
*,
|
||||
deployment: Deployment | None = None,
|
||||
known_bug: str | None = None,
|
||||
) -> tuple[BridgeCostParityTestCase, ...]:
|
||||
"""One parity pair per base and bridged endpoint: the same stored provider response and deployment, billed through
|
||||
the base's own endpoint and through each client endpoint LiteLLM bridges into it. `deployment` pins both legs to a
|
||||
deployment the base does not carry (an `openai/responses/` deployment for the chat -> Responses bridge)."""
|
||||
pinned: Final = tuple(
|
||||
base if deployment is None else base.model_copy(update={"deployment": deployment}) for base in bases
|
||||
)
|
||||
return tuple(
|
||||
BridgeCostParityTestCase(
|
||||
native=base,
|
||||
bridged=base.model_copy(
|
||||
update={
|
||||
"name": f"{base.name}-{_ENDPOINT_SUFFIX[endpoint]}",
|
||||
"endpoint": endpoint,
|
||||
"request": _bridged_request(base, endpoint),
|
||||
}
|
||||
),
|
||||
known_bug=known_bug,
|
||||
)
|
||||
for base in pinned
|
||||
for endpoint in endpoints
|
||||
if endpoint != base.endpoint
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ provider's own API and once through a client API LiteLLM bridges into it. Both L
|
|||
case's row, and they must agree with each other except where a documented provider quirk says otherwise.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
|
||||
|
|
@ -13,7 +15,15 @@ from integration.cost_calculation.bridge_cost_parity.runner import assert_bridge
|
|||
from integration.cost_calculation.catalog import BRIDGE_COST_PARITY_CASES
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", BRIDGE_COST_PARITY_CASES, ids=[case.id for case in BRIDGE_COST_PARITY_CASES])
|
||||
def _marks(case: BridgeCostParityTestCase) -> tuple[pytest.MarkDecorator, ...]:
|
||||
"""A known bridge bug is intermittent (it billed the native row in 1 of 4 local runs), so the xfail is not strict."""
|
||||
return (pytest.mark.xfail(strict=False, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else ()
|
||||
|
||||
|
||||
_CASES: Final = tuple(pytest.param(case, marks=_marks(case), id=case.id) for case in BRIDGE_COST_PARITY_CASES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", _CASES)
|
||||
@pytest.mark.timeout(180)
|
||||
def test_bridged_request_bills_the_same_row_as_the_native_request(
|
||||
case: BridgeCostParityTestCase, gateway: Gateway
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
RecountRates,
|
||||
SseResponse,
|
||||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
|
|
@ -6954,7 +6954,22 @@ CLAUDE_HAIKU_5_5_TIERED_CACHE_WRITE_1H_ABOVE_100K: Final = with_usage(
|
|||
),
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_GENERALIZED_ALIAS: Final = CLAUDE_SONNET_5_INPUT_TEXT.model_copy(
|
||||
update={"name": "claude-sonnet-5-generalized_alias", "alias": "claude-sonnet-5-$MARKER"}
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_GENERALIZED_ALIAS_STREAM: Final = CLAUDE_SONNET_5_STREAM.model_copy(
|
||||
update={"name": "claude-sonnet-5-generalized_alias_stream", "alias": "claude-sonnet-5-$MARKER"}
|
||||
)
|
||||
|
||||
CLAUDE_SONNET_5_SLASH_ALIAS_STREAM: Final = CLAUDE_SONNET_5_STREAM.model_copy(
|
||||
update={"name": "claude-sonnet-5-slash_alias_stream", "alias": "anthropic/claude-sonnet-5-$MARKER"}
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
CLAUDE_SONNET_5_GENERALIZED_ALIAS,
|
||||
CLAUDE_SONNET_5_GENERALIZED_ALIAS_STREAM,
|
||||
CLAUDE_SONNET_5_SLASH_ALIAS_STREAM,
|
||||
CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K,
|
||||
CLAUDE_HAIKU_5_5_TIERED_AT_100K_BOUNDARY,
|
||||
CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K,
|
||||
|
|
@ -7118,6 +7133,15 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs(
|
||||
(
|
||||
CLAUDE_SONNET_5_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_SONNET_5_MESSAGES_CACHE_READ,
|
||||
CLAUDE_SONNET_5_MESSAGES_STREAM,
|
||||
CLAUDE_SONNET_5_MESSAGES_STREAM_CACHE_READ,
|
||||
),
|
||||
("/v1/responses", "/v1/chat/completions"),
|
||||
),
|
||||
BridgeCostParityTestCase(
|
||||
native=CLAUDE_SONNET_5_MESSAGES_INPUT_TEXT, bridged=CLAUDE_SONNET_5_RESPONSES_BRIDGED_JSON
|
||||
),
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
SseResponse,
|
||||
WavUpload,
|
||||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
|
|
@ -6170,7 +6170,9 @@ AZURE_GPT_6_LUNA_CACHE_WRITE_5M: Final = with_response_usage(
|
|||
"total_tokens": 13308,
|
||||
},
|
||||
ExactExpected(spend=0.002327, input_cost=0.00208, output_cost=0.000247, prompt_tokens=12928, completion_tokens=380),
|
||||
choice_updates={"content_filter_result": {"error": {"code": "content_filter_error", "message": "The contents are not filtered"}}},
|
||||
choice_updates={
|
||||
"content_filter_result": {"error": {"code": "content_filter_error", "message": "The contents are not filtered"}}
|
||||
},
|
||||
)
|
||||
AZURE_GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage(
|
||||
AZURE_GPT_6_ASTRA_CACHE_READ,
|
||||
|
|
@ -6317,6 +6319,16 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs((AZURE_GPT_5_6_INPUT_TEXT, AZURE_GPT_5_6_CACHE_READ, AZURE_GPT_5_6_STREAM), ("/v1/messages",)),
|
||||
*bridge_pairs(
|
||||
(AZURE_GPT_5_6_REASONING,),
|
||||
("/v1/messages",),
|
||||
known_bug=(
|
||||
"the /v1/messages -> chat bridge usually drops completion_tokens_details.reasoning_tokens on azure, so"
|
||||
" reasoning tokens bill at output_cost_per_token instead of output_cost_per_reasoning_token"
|
||||
" (output_cost 0.058176 instead of 0.065484)"
|
||||
),
|
||||
),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_SOL_INPUT_TEXT, bridged=AZURE_GPT_6_SOL_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_LUNA_INPUT_TEXT, bridged=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_ASTRA_INPUT_TEXT, bridged=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_JSON),
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
RecountExpected,
|
||||
RecountRates,
|
||||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase
|
||||
|
||||
# anthropic.claude-sonnet-5-v1:0
|
||||
|
|
@ -6238,6 +6238,14 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs(
|
||||
(
|
||||
ANTHROPIC_CLAUDE_SONNET_5_V1_0_INPUT_TEXT,
|
||||
ANTHROPIC_CLAUDE_SONNET_5_V1_0_CACHE_READ,
|
||||
ANTHROPIC_CLAUDE_SONNET_5_V1_0_STREAM,
|
||||
),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
),
|
||||
BridgeCostParityTestCase(
|
||||
native=US_ANTHROPIC_CLAUDE_SONNET_5_INPUT_TEXT, bridged=US_ANTHROPIC_CLAUDE_SONNET_5_RESPONSES_BRIDGED_JSON
|
||||
),
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.
|
|||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
ALWAYS_OPEN_OFF_PEAK_WINDOW,
|
||||
Deployment,
|
||||
OffPeakPricing,
|
||||
CostTrackingTestCase,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
|
|
@ -3089,7 +3092,39 @@ FIREWORKS_EMBEDDINGS_V1: Final = CostTrackingTestCase(
|
|||
)
|
||||
|
||||
|
||||
_DEEPSEEK_V4P1_FLASH_OFF_PEAK: Final = Deployment(
|
||||
off_peak_pricing=OffPeakPricing(
|
||||
hours_utc=ALWAYS_OPEN_OFF_PEAK_WINDOW, input_cost_per_token=3e-07, output_cost_per_token=1.2e-06
|
||||
)
|
||||
)
|
||||
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_OFF_PEAK: Final = (
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-off_peak",
|
||||
"deployment": _DEEPSEEK_V4P1_FLASH_OFF_PEAK,
|
||||
"expected": ExactExpected(
|
||||
spend=0.0010464, input_cost=0.000552, output_cost=0.0004944, prompt_tokens=1840, completion_tokens=412
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_OFF_PEAK_STREAM: Final = (
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_STREAM.model_copy(
|
||||
update={
|
||||
"name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-off_peak_stream",
|
||||
"deployment": _DEEPSEEK_V4P1_FLASH_OFF_PEAK,
|
||||
"expected": ExactExpected(
|
||||
spend=0.0010464, input_cost=0.000552, output_cost=0.0004944, prompt_tokens=1840, completion_tokens=412
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_OFF_PEAK,
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_OFF_PEAK_STREAM,
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_INPUT_TEXT,
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_FALLBACK_CACHE_READ_AT_HALF_INPUT_RATE,
|
||||
FIREWORKS_AI_ACCOUNTS_FIREWORKS_MODELS_DEEPSEEK_V4P1_FLASH_STREAM,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.
|
|||
|
||||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
ExactExpected,
|
||||
|
|
@ -3626,3 +3627,15 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
),
|
||||
StreamParityTestCase(plain=GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, streamed=GEMINI_GEMINI_3_8_FLASH_STREAM_TOOL_CALL),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs(
|
||||
(
|
||||
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT,
|
||||
GEMINI_GEMINI_3_8_FLASH_CACHE_READ,
|
||||
GEMINI_GEMINI_3_8_FLASH_REASONING,
|
||||
GEMINI_GEMINI_3_8_FLASH_STREAM,
|
||||
),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.
|
|||
from typing import Final
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
ALWAYS_OPEN_OFF_PEAK_WINDOW,
|
||||
OffPeakPricing,
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
EventStreamEvent,
|
||||
|
|
@ -1697,7 +1699,22 @@ AZURE_AI_CLAUDE_HAIKU_5_5_STREAM: Final = CLAUDE_HAIKU_5_5_STREAM.model_copy(
|
|||
)
|
||||
|
||||
|
||||
PERPLEXITY_SONAR_NEXT_OFF_PEAK: Final = PERPLEXITY_SONAR_NEXT_NO_SEARCH.model_copy(
|
||||
update={
|
||||
"name": "perplexity-sonar-next-off_peak",
|
||||
"deployment": Deployment(
|
||||
off_peak_pricing=OffPeakPricing(
|
||||
hours_utc=ALWAYS_OPEN_OFF_PEAK_WINDOW, input_cost_per_token=2.26e-06, output_cost_per_token=2.1e-06
|
||||
)
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0050236, input_cost=0.0041584, output_cost=0.0008652, prompt_tokens=1840, completion_tokens=412
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
PERPLEXITY_SONAR_NEXT_OFF_PEAK,
|
||||
NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND,
|
||||
AMAZON_NOVA_CANVAS_NEXT_IMAGES_ONE,
|
||||
COHERE_EMBEDDINGS_V5,
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
)
|
||||
from pydantic import JsonValue
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
|
||||
|
||||
# gpt-5.3-codex
|
||||
|
|
@ -11184,7 +11184,17 @@ GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage(
|
|||
)
|
||||
|
||||
|
||||
GPT_5_6_SLASH_ALIAS: Final = GPT_5_6_INPUT_TEXT.model_copy(
|
||||
update={"name": "gpt-5.6-slash_alias", "alias": "openai/custom/gpt-5.6-$MARKER"}
|
||||
)
|
||||
|
||||
GPT_5_6_SLASH_ALIAS_STREAM: Final = GPT_5_6_STREAM.model_copy(
|
||||
update={"name": "gpt-5.6-slash_alias_stream", "alias": "openai/custom/gpt-5.6-$MARKER"}
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GPT_5_6_SLASH_ALIAS,
|
||||
GPT_5_6_SLASH_ALIAS_STREAM,
|
||||
GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON,
|
||||
GPT_6_SOL_CHAT_BRIDGED_JSON,
|
||||
GPT_6_LUNA_RESPONSES_DEPLOYMENT_JSON,
|
||||
|
|
@ -11424,6 +11434,27 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs(
|
||||
(
|
||||
GPT_5_6_RESPONSES_NATIVE_JSON,
|
||||
GPT_5_6_RESPONSES_CACHE_READ,
|
||||
GPT_5_6_RESPONSES_REASONING,
|
||||
GPT_5_6_RESPONSES_STREAM,
|
||||
GPT_5_6_RESPONSES_STREAM_CACHE_READ,
|
||||
),
|
||||
("/v1/messages",),
|
||||
),
|
||||
*bridge_pairs(
|
||||
(
|
||||
GPT_5_6_RESPONSES_NATIVE_JSON,
|
||||
GPT_5_6_RESPONSES_CACHE_READ,
|
||||
GPT_5_6_RESPONSES_REASONING,
|
||||
GPT_5_6_RESPONSES_STREAM,
|
||||
GPT_5_6_RESPONSES_STREAM_CACHE_READ,
|
||||
),
|
||||
("/v1/chat/completions",),
|
||||
deployment=Deployment(model="openai/responses/gpt-5.6"),
|
||||
),
|
||||
BridgeCostParityTestCase(native=GPT_6_SOL_RESPONSES_NATIVE_JSON, bridged=GPT_6_SOL_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_NATIVE_JSON, bridged=GPT_6_LUNA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, bridged=GPT_6_ASTRA_MESSAGES_BRIDGED_JSON),
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
CLAUDE_SONNET_5_5_INPUT_TEXT,
|
||||
CLAUDE_SONNET_5_5_STREAM,
|
||||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
ExactExpected,
|
||||
|
|
@ -3157,3 +3158,8 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(plain=VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_OPUS_5_5_STREAM),
|
||||
StreamParityTestCase(plain=VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_FABLE_5_1_STREAM),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = bridge_pairs(
|
||||
(GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -62,6 +62,8 @@ BRIDGE_COST_PARITY_CASES: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
*bedrock_converse.BRIDGE_COST_PARITY,
|
||||
*azure.BRIDGE_COST_PARITY,
|
||||
*anthropic.BRIDGE_COST_PARITY,
|
||||
*gemini.BRIDGE_COST_PARITY,
|
||||
*vertex_ai.BRIDGE_COST_PARITY,
|
||||
)
|
||||
_PARITY_LEGS: Final = frozenset(leg.name for pair in PARITY_CASES for leg in (pair.plain, pair.streamed)) | frozenset(
|
||||
leg.name for pair in BRIDGE_COST_PARITY_CASES for leg in (pair.native, pair.bridged)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
|
|||
from integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario
|
||||
from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase, StoredResponse
|
||||
from integration.cost_calculation.cost_tracking_case import ALIAS_MARKER, CostTrackingTestCase, StoredResponse
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
|
|
@ -289,7 +289,10 @@ def register_scenario_deployment(
|
|||
case.response if response is None else response,
|
||||
)
|
||||
scenario.cleanups.callback(delete_scenario, handle)
|
||||
registered_model_name: Final = f"cost-{marker}{marker_suffix}-{run_marker}"
|
||||
unique_marker: Final = f"{marker}{marker_suffix}-{run_marker}"
|
||||
registered_model_name: Final = (
|
||||
case.alias.replace(ALIAS_MARKER, unique_marker) if case.alias is not None else f"cost-{unique_marker}"
|
||||
)
|
||||
parameters: Final = {
|
||||
"model": case.litellm_model,
|
||||
"api_key": case.api_key,
|
||||
|
|
@ -330,7 +333,14 @@ def register_scenario_deployment(
|
|||
{
|
||||
"model_name": registered_model_name,
|
||||
"litellm_params": parameters,
|
||||
"model_info": ({"base_model": case.base_model} if case.base_model is not None else {}),
|
||||
"model_info": {
|
||||
**({"base_model": case.base_model} if case.base_model is not None else {}),
|
||||
**(
|
||||
{"off_peak_pricing": case.deployment.off_peak_pricing.model_dump()}
|
||||
if case.deployment is not None and case.deployment.off_peak_pricing is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1335,3 +1335,13 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"cache_read_input_token_cost_above_272k_tokens": 4.8e-07,
|
||||
},
|
||||
}
|
||||
|
||||
FALLBACK_GENERALIZATIONS: Final[dict[str, JsonValue]] = {
|
||||
"rules": [
|
||||
{
|
||||
"name": "claude-family-baseline",
|
||||
"pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
|
||||
"model_info": {"mode": "chat", "max_input_tokens": 200000, "max_output_tokens": 64000},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from integration.cost_calculation.cost_map import COST_MAP_ENTRIES
|
|||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator, model_validator
|
||||
|
||||
PRIOR_RESPONSE_ID_MARKER: Final = "$PRIOR_RESPONSE_ID"
|
||||
ALIAS_MARKER: Final = "$MARKER"
|
||||
|
||||
|
||||
class SearchContextCostPerQuery(BaseModel):
|
||||
|
|
@ -132,6 +133,17 @@ class CostMapEntry(BaseModel):
|
|||
provider_specific_entry: ProviderSpecificEntry | None = None
|
||||
|
||||
|
||||
class OffPeakPricing(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
hours_utc: str
|
||||
input_cost_per_token: float
|
||||
output_cost_per_token: float
|
||||
|
||||
|
||||
ALWAYS_OPEN_OFF_PEAK_WINDOW: Final = "00:00-00:00"
|
||||
|
||||
|
||||
class Deployment(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
|
|
@ -140,6 +152,7 @@ class Deployment(BaseModel):
|
|||
input_cost_per_token: float | None = None
|
||||
output_cost_per_token: float | None = None
|
||||
search_cost_per_query: float | None = None
|
||||
off_peak_pricing: OffPeakPricing | None = None
|
||||
|
||||
|
||||
class WavUpload(BaseModel):
|
||||
|
|
@ -286,6 +299,7 @@ class CostTrackingTestCase(BaseModel):
|
|||
covers: str
|
||||
known_bug: str | None = None
|
||||
model: str
|
||||
alias: str | None = None
|
||||
endpoint: (
|
||||
Literal[
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -313,6 +327,13 @@ class CostTrackingTestCase(BaseModel):
|
|||
disconnect_after_frames: int | None = Field(default=None, ge=1)
|
||||
upstream_body: dict[str, JsonValue] | None = None
|
||||
|
||||
@field_validator("alias")
|
||||
@classmethod
|
||||
def validate_alias(cls, value: str | None) -> str | None:
|
||||
if value is not None and ALIAS_MARKER not in value:
|
||||
raise ValueError(f"alias must contain {ALIAS_MARKER} so every run registers a unique model_name")
|
||||
return value
|
||||
|
||||
@property
|
||||
def rates(self) -> CostMapEntry:
|
||||
return COST_MAP[self.model]
|
||||
|
|
@ -664,9 +685,7 @@ def with_response_usage(
|
|||
if not isinstance(base.response, JsonResponse):
|
||||
raise TypeError(f"{base.name}: with_response_usage needs a JsonResponse base")
|
||||
body: Final = {**base.response.body, "usage": dict(usage)}
|
||||
response_body: Final = (
|
||||
_with_choice_updates(body, choice_updates) if choice_updates is not None else body
|
||||
)
|
||||
response_body: Final = _with_choice_updates(body, choice_updates) if choice_updates is not None else body
|
||||
return base.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from __future__ import annotations
|
|||
import io
|
||||
import json
|
||||
import struct
|
||||
import time
|
||||
import uuid
|
||||
import wave
|
||||
import zlib
|
||||
|
|
@ -16,10 +15,11 @@ from typing import Final, cast
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway, string_value
|
||||
from integration._support.client import JSON_OBJECT, Gateway, Scenario, string_value
|
||||
from integration._support.upstream import delete_scenario, register_scenario
|
||||
from integration.cost_calculation.assertions import assert_exact, assert_recount
|
||||
from integration.cost_calculation.conftest import (
|
||||
CostRow,
|
||||
approx_equal,
|
||||
poll_cost_row,
|
||||
poll_failure_row,
|
||||
|
|
@ -29,6 +29,7 @@ from integration.cost_calculation.conftest import (
|
|||
register_scenario_deployment,
|
||||
)
|
||||
from integration.cost_calculation.catalog import STANDALONE_CASES, data_errors
|
||||
from integration.cost_calculation.stream_parity.runner import flush_spend_logs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
BinaryResponse,
|
||||
CostTrackingTestCase,
|
||||
|
|
@ -42,6 +43,14 @@ if _data_errors := data_errors():
|
|||
raise ValueError("\n".join(_data_errors))
|
||||
|
||||
|
||||
def _settled_rows(scenario: Scenario, gateway: Gateway, key: str, count: int) -> tuple[CostRow, ...]:
|
||||
"""Every row the key will ever get, read after a flush request so a late duplicate cannot hide behind `>= count`."""
|
||||
flush_identity: Final = flush_spend_logs(scenario, gateway, key)
|
||||
settled: Final = tuple(row for row in read_rows_now(key) if row.model_id != flush_identity)
|
||||
assert len(settled) == count, f"expected exactly {count} SpendLogs rows for the key, got {len(settled)}: {settled}"
|
||||
return settled
|
||||
|
||||
|
||||
def _marks(case: CostTrackingTestCase) -> tuple[pytest.MarkDecorator, ...]:
|
||||
bug: Final = (
|
||||
(pytest.mark.xfail(strict=True, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else ()
|
||||
|
|
@ -234,6 +243,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
|
|||
assert isinstance(expected, RecountExpected)
|
||||
assert row.status == "success", f"{case.name}: disconnect row status was {row.status}"
|
||||
assert_recount(case.name, expected, row)
|
||||
_settled_rows(scenario, gateway, key, 1)
|
||||
return
|
||||
responses: Final = tuple(
|
||||
(
|
||||
|
|
@ -255,6 +265,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
|
|||
)
|
||||
row: Final = poll_failure_row(key)
|
||||
assert row.spend == 0, f"{case.name}: failure spend was {row.spend}"
|
||||
_settled_rows(scenario, gateway, key, 1)
|
||||
return
|
||||
assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}"
|
||||
if case.upstream_body is not None:
|
||||
|
|
@ -273,19 +284,14 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
|
|||
)
|
||||
if case.response.content_type == "text/event-stream":
|
||||
_assert_stream_has_no_error(response.text)
|
||||
rows: Final = poll_rows(key, len(responses) + (prior_response_id is not None))
|
||||
row_count: Final = len(responses) + (prior_response_id is not None)
|
||||
rows: Final = poll_rows(key, row_count)
|
||||
if isinstance(expected, RecountExpected):
|
||||
row: Final = rows[0]
|
||||
assert_recount(case.name, expected, row)
|
||||
_settled_rows(scenario, gateway, key, row_count)
|
||||
return
|
||||
assert isinstance(expected, ExactExpected)
|
||||
if fallback_deployment is not None:
|
||||
assert deployment is not None
|
||||
time.sleep(3)
|
||||
settled_rows: Final = read_rows_now(key)
|
||||
assert len(settled_rows) == 1
|
||||
assert settled_rows[0].status == "success"
|
||||
assert settled_rows[0].model_id == deployment.identity
|
||||
if isinstance(case.response, BinaryResponse):
|
||||
header: Final = response.headers.get("x-litellm-response-cost")
|
||||
if header is not None:
|
||||
|
|
@ -329,3 +335,8 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
|
|||
assert rollups.daily_team.prompt_tokens == expected.prompt_tokens * 3
|
||||
assert rollups.daily_team.completion_tokens == expected.completion_tokens * 3
|
||||
assert rollups.daily_team.api_requests == 3
|
||||
settled: Final = _settled_rows(scenario, gateway, key, row_count)
|
||||
if fallback_deployment is not None:
|
||||
assert deployment is not None
|
||||
assert settled[0].status == "success"
|
||||
assert settled[0].model_id == deployment.identity
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue