scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

test(integration): use per-run request ids, drop unbillable perplexity search case

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-19 23:01:25 +00:00
parent 72b007a4ae
commit c7113f043a
4 changed files with 33 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
tests/integration/contracts.json
View file

@ -1477,9 +1477,6 @@
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-search_queries_and_citations]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],

9
tests/integration/cost_calculation/cost_tracking_case.py
View file

@ -54,7 +54,6 @@ class CostMapEntry(BaseModel):
cache_creation_input_token_cost_above_200k_tokens: float | None = None
input_cost_per_token_above_200k_tokens: float | None = None
output_cost_per_token_above_200k_tokens: float | None = None
citation_cost_per_token: float | None = None
tiered_pricing: tuple[TieredPrice, ...] | None = None
output_cost_per_reasoning_token: float | None = None
input_cost_per_audio_token: float | None = None
@ -263,6 +262,13 @@ class CostTrackingTestCase(BaseModel):
return "bedrock"
return None
@property
def reports_provider_cost(self) -> bool:
if not isinstance(self.response, JsonResponse):
return False
usage: Final = self.response.body.get("usage")
return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float))
class _CasesFile(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
@ -409,6 +415,7 @@ def data_errors() -> tuple[str, ...]:
not case.expected.breakdown_persisted
and case.passthrough_provider is None
and case.rates.mode != "image_generation"
and not case.reports_provider_cost
)
or (not case.expected.cost_header and case.passthrough_provider is None)
)

72
tests/integration/cost_calculation/cost_tracking_cases.json
View file

@ -42,7 +42,6 @@
"mode": "chat",
"input_cost_per_token": 1.05e-06,
"output_cost_per_token": 1.05e-06,
"citation_cost_per_token": 2e-06,
"search_context_cost_per_query": {
"search_context_size_low": 0.005,
"search_context_size_medium": 0.008,
@ -27918,7 +27917,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "dashscope-tiered-input",
"id": "$REQUEST_ID",
"object": "chat.completion",
"model": "qwen4-max",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -27940,7 +27939,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "dashscope-tiered-boundary",
"id": "$REQUEST_ID",
"object": "chat.completion",
"model": "qwen4-max",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -27962,7 +27961,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "dashscope-tiered-second",
"id": "$REQUEST_ID",
"object": "chat.completion",
"model": "qwen4-max",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -27984,7 +27983,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "dashscope-tiered-top",
"id": "$REQUEST_ID",
"object": "chat.completion",
"model": "qwen4-max",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28046,7 +28045,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "msg-cache-1h",
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
@ -28083,14 +28082,21 @@
"response": {
"content_type": "application/json",
"body": {
"id": "or-reported",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "anthropic/claude-sonnet-5",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252, "cost": 0.0421}
}
},
"expected": {"spend": 0.0421, "input_cost": 0.0, "output_cost": 0.0421, "prompt_tokens": 1840, "completion_tokens": 412}
"expected": {
"spend": 0.0421,
"input_cost": 0.0,
"output_cost": 0.0421,
"prompt_tokens": 1840,
"completion_tokens": 412,
"breakdown_persisted": false
}
},
{
"name": "openrouter-anthropic-claude-sonnet-5-token_priced",
@ -28105,7 +28111,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "or-token",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "anthropic/claude-sonnet-5",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28114,42 +28120,6 @@
},
"expected": {"spend": 0.01248, "input_cost": 0.005888, "output_cost": 0.006592, "prompt_tokens": 1840, "completion_tokens": 412}
},
{
"name": "perplexity-sonar-next-search_queries_and_citations",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "perplexity/sonar-next",
"request": {
"model": "$MODEL",
"messages": [{"role": "user", "content": "search"}],
"web_search_options": {"search_context_size": "high"},
"stream": false,
"allowed_openai_params": ["web_search_options"]
},
"response": {
"content_type": "application/json",
"body": {
"id": "pplx-search",
"object": "chat.completion",
"model": "sonar-next",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"citations": [
"https://e.co/aaaaaaaaaaaaaaaaaaaaaaaaaaa",
"https://e.co/bbbbbbbbbbbbbbbbbbbbbbbbbbb",
"https://e.co/ccccccccccccccccccccccccccc",
"https://e.co/ddddddddddddddddddddddddddd"
],
"usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252, "num_search_queries": 3}
}
},
"expected": {
"spend": 0.0174446,
"input_cost": 0.002012,
"output_cost": 0.0004326,
"tool_usage_cost": 0.015,
"prompt_tokens": 1840,
"completion_tokens": 412
}
},
{
"name": "perplexity-sonar-next-no_search",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
@ -28163,7 +28133,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "pplx-no-search",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "sonar-next",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28185,7 +28155,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "deepseek-cache",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "deepseek-v4-chat",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28221,7 +28191,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "deepseek-no-cache",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "deepseek-v4-chat",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28250,7 +28220,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "xai-reasoning",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "grok-5",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28277,7 +28247,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "xai-search",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "grok-5",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
@ -28311,7 +28281,7 @@
"response": {
"content_type": "application/json",
"body": {
"id": "xai-reported",
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "grok-5",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],

6
tests/integration/cost_calculation/test_cost_tracking.py
View file

@ -148,12 +148,14 @@ def _assert_breakdown(
):
if expected_component is None:
continue
assert actual_component is not None and approx_equal(actual_component, expected_component), (
actual_value: Final = actual_component or 0.0
assert approx_equal(actual_value, expected_component), (
f"{case.name}: {field} {actual_component} != expected {expected_component}"
)
if expected.cost_header and case.response.content_type == "application/json":
header: Final = response.headers.get(header_name)
assert header is not None and approx_equal(float(header), expected_component), (
header_value: Final = float(header) if header is not None else 0.0
assert approx_equal(header_value, expected_component), (
f"{case.name}: {header_name} {header} != expected {expected_component}"
)
if expected.cost_header and case.response.content_type == "application/json" and any(