scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-13 03:59:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 66

fix(cost): bill custom priced deployments at their own rates over the upstream x-litellm-response-cost header (#46120)

Browse source 
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-10-13 00:58:03 +00:00 • committed by GitHub
parent 5338edb7e3
commit f97d3df6de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 250 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/cost_calculator.py
View file

@ -2092,7 +2092,7 @@ def response_cost_calculator(
getattr(response_object, HIDDEN_PARAMS_ATTR),
)
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None:
if provider_response_cost is not None and not custom_pricing:
return provider_response_cost
response_cost = completion_cost(

5
tests/integration/_support/upstream.py
View file

@ -496,6 +496,7 @@ class Provider:
.encode(),
media_type=response.content_type,
status_code=response.status,
headers=response.headers,
)
case BinaryResponse():
return Response(
@ -518,13 +519,13 @@ class Provider:
).encode()
await asyncio.sleep(response.frame_delay_ms / 1000)
return StreamingResponse(stream(), media_type=response.content_type)
return StreamingResponse(stream(), media_type=response.content_type, headers=response.headers)
stream_body: Final = (
("\n\n".join(response.frames) + "\n\n")
.replace("$REQUEST_ID", scenario_id)
.replace("$UNIQUE_ID", unique_id)
)
return Response(content=stream_body.encode(), media_type=response.content_type)
return Response(content=stream_body.encode(), media_type=response.content_type, headers=response.headers)
case EventStreamResponse():
events: Final = (
tuple(

201
tests/integration/cost_calculation/cases/openai.py
View file

@ -7162,6 +7162,201 @@ OPENAI_DEPLOYMENT_PRICING_OVERRIDE: Final = CostTrackingTestCase(
),
)
_CC_CUSTOM_MODEL_COST_HEADER_FRAMES: Final = sse_frames(
{
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "cc-custom-model",
"choices": [
{"index": 0, "delta": {"role": "assistant", "content": "scripted response"}, "finish_reason": None}
],
},
{
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "cc-custom-model",
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
},
{
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "cc-custom-model",
"choices": [],
"usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263},
},
done=True,
)
_CC_CUSTOM_MODEL_RESPONSES_BODY: Final[dict[str, JsonValue]] = {
"id": "resp_$REQUEST_ID",
"object": "response",
"status": "completed",
"created_at": 1700000000,
"model": "cc-custom-model",
"output": [
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
}
],
"usage": {
"input_tokens": 10,
"output_tokens": 253,
"total_tokens": 263,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0},
},
}
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER: Final = CostTrackingTestCase(
name="openai-deployment-zero-pricing-ignores-upstream-cost-header",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0),
request={
"model": "$MODEL",
"messages": [{"role": "user", "content": "openai-deployment-zero-pricing-ignores-upstream-cost-header"}],
"stream": False,
},
response=JsonResponse(
content_type="application/json",
body={
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "cc-custom-model",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "scripted response"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263},
},
headers={"x-litellm-response-cost": "0.000413"},
),
expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253),
)
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_STREAM: Final = CostTrackingTestCase(
name="openai-deployment-zero-pricing-ignores-upstream-cost-header-stream",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0),
request={
"model": "$MODEL",
"messages": [{"role": "user", "content": "openai-deployment-zero-pricing-ignores-upstream-cost-header"}],
"stream": True,
"stream_options": {"include_usage": True},
},
response=SseResponse(
content_type="text/event-stream",
frames=_CC_CUSTOM_MODEL_COST_HEADER_FRAMES,
headers={"x-litellm-response-cost": "0.000413"},
),
expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253),
)
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_MESSAGES: Final = CostTrackingTestCase(
name="openai-deployment-zero-pricing-ignores-upstream-cost-header-messages",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
endpoint="/v1/messages",
deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0),
request={
"model": "$MODEL",
"max_tokens": 300,
"messages": [{"role": "user", "content": "openai-deployment-zero-pricing-ignores-upstream-cost-header"}],
},
response=JsonResponse(
content_type="application/json",
body=_CC_CUSTOM_MODEL_RESPONSES_BODY,
headers={"x-litellm-response-cost": "0.000413"},
),
expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253),
)
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_RESPONSES: Final = CostTrackingTestCase(
name="openai-deployment-zero-pricing-ignores-upstream-cost-header-responses",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
endpoint="/v1/responses",
deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0),
request={
"model": "$MODEL",
"input": "openai-deployment-zero-pricing-ignores-upstream-cost-header",
"stream": False,
},
response=JsonResponse(
content_type="application/json",
body=_CC_CUSTOM_MODEL_RESPONSES_BODY,
headers={"x-litellm-response-cost": "0.000413"},
),
expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253),
)
OPENAI_DEPLOYMENT_PRICING_IGNORES_UPSTREAM_COST_HEADER: Final = CostTrackingTestCase(
name="openai-deployment-pricing-ignores-upstream-cost-header",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.001, output_cost_per_token=0.002),
request={
"model": "$MODEL",
"messages": [{"role": "user", "content": "openai-deployment-pricing-ignores-upstream-cost-header"}],
"stream": False,
},
response=JsonResponse(
content_type="application/json",
body={
"id": "chatcmpl-$UNIQUE_ID",
"object": "chat.completion",
"model": "cc-custom-model",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "scripted response"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263},
},
headers={"x-litellm-response-cost": "0.000413"},
),
expected=ExactExpected(
spend=0.516, input_cost=0.01, output_cost=0.506, prompt_tokens=10, completion_tokens=253, rollups=True
),
)
GPT_5_6_UPSTREAM_COST_HEADER_BILLS_HEADER_VALUE: Final = CostTrackingTestCase(
name="gpt-5.6-upstream-cost-header-bills-header-value",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
request={
"model": "$MODEL",
"messages": [{"role": "user", "content": "gpt-5.6-upstream-cost-header-bills-header-value"}],
"stream": False,
},
response=JsonResponse(
content_type="application/json",
body={
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": "gpt-5.6",
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "scripted response"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263},
},
headers={"x-litellm-response-cost": "0.000413"},
),
expected=ExactExpected(
spend=0.000413,
input_cost=0.0,
output_cost=0.000413,
prompt_tokens=10,
completion_tokens=253,
breakdown_persisted=False,
),
)
GPT_5_6_UPSTREAM_400_ZERO_SPEND: Final = CostTrackingTestCase(
name="gpt-5.6-upstream_400_zero_spend",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -8134,6 +8329,12 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GPT_5_6_RESPONSES_SERVICE_TIER_FLEX,
GPT_5_6_RESPONSES_SERVICE_TIER_PRIORITY,
OPENAI_DEPLOYMENT_PRICING_OVERRIDE,
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER,
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_STREAM,
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_MESSAGES,
OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_RESPONSES,
OPENAI_DEPLOYMENT_PRICING_IGNORES_UPSTREAM_COST_HEADER,
GPT_5_6_UPSTREAM_COST_HEADER_BILLS_HEADER_VALUE,
GPT_5_6_UPSTREAM_400_ZERO_SPEND,
GPT_5_6_UPSTREAM_401_ZERO_SPEND,
GPT_5_6_UPSTREAM_500_STREAM_REQUEST_ZERO_SPEND,

4
tests/integration/cost_calculation/cost_tracking_case.py
View file

@ -129,6 +129,7 @@ class JsonResponse(BaseModel):
content_type: Literal["application/json"]
body: dict[str, JsonValue]
status: int = 200
headers: dict[str, str] = Field(default_factory=dict)
class SseResponse(BaseModel):
@ -137,6 +138,7 @@ class SseResponse(BaseModel):
content_type: Literal["text/event-stream"]
frames: tuple[str, ...]
frame_delay_ms: int = Field(default=0, ge=0)
headers: dict[str, str] = Field(default_factory=dict)
class EventStreamEvent(BaseModel):
@ -329,6 +331,8 @@ class CostTrackingTestCase(BaseModel):
def reports_provider_cost(self) -> bool:
if not isinstance(self.response, JsonResponse):
return False
if "x-litellm-response-cost" in self.response.headers:
return True
usage: Final = self.response.body.get("usage")
return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float))

41
tests/unit/test_cost_calculator.py
View file

@ -186,6 +186,47 @@ def test_cost_calculator_with_response_cost_in_additional_headers():
assert result == 1000
@pytest.mark.parametrize(
("input_cost_per_token", "output_cost_per_token", "expected_cost"),
((0.0, 0.0, 0.0), (0.001, 0.002, 10 * 0.001 + 253 * 0.002)),
)
def test_response_cost_calculator_custom_pricing_ignores_upstream_response_cost_header(
monkeypatch: pytest.MonkeyPatch,
input_cost_per_token: float,
output_cost_per_token: float,
expected_cost: float,
) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.register_model(
model_cost={
"cc-custom-priced-chat": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": input_cost_per_token,
"output_cost_per_token": output_cost_per_token,
}
}
)
response: Final = ModelResponse(
model="cc-custom-priced-chat",
choices=[Choices(message=Message(content="ok"))],
usage=Usage(prompt_tokens=10, completion_tokens=253, total_tokens=263),
)
response._hidden_params = {"additional_headers": {"llm_provider-x-litellm-response-cost": 0.000413}}
cost: Final = response_cost_calculator(
response_object=response,
model="cc-custom-priced-chat",
custom_llm_provider="openai",
call_type="completion",
optional_params={},
custom_pricing=True,
)
assert cost == pytest.approx(expected_cost)
def test_response_cost_calculator_keeps_optional_params_out_of_hidden_params():
class MockResponse(BaseModel):
pass