diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 533c6a9e00e..263d0bfcb81 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2092,7 +2092,7 @@ def response_cost_calculator( getattr(response_object, HIDDEN_PARAMS_ATTR), ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) - if provider_response_cost is not None: + if provider_response_cost is not None and not custom_pricing: return provider_response_cost response_cost = completion_cost( diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index 8c3c88990e1..89a28cffb1d 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -496,6 +496,7 @@ class Provider: .encode(), media_type=response.content_type, status_code=response.status, + headers=response.headers, ) case BinaryResponse(): return Response( @@ -518,13 +519,13 @@ class Provider: ).encode() await asyncio.sleep(response.frame_delay_ms / 1000) - return StreamingResponse(stream(), media_type=response.content_type) + return StreamingResponse(stream(), media_type=response.content_type, headers=response.headers) stream_body: Final = ( ("\n\n".join(response.frames) + "\n\n") .replace("$REQUEST_ID", scenario_id) .replace("$UNIQUE_ID", unique_id) ) - return Response(content=stream_body.encode(), media_type=response.content_type) + return Response(content=stream_body.encode(), media_type=response.content_type, headers=response.headers) case EventStreamResponse(): events: Final = ( tuple( diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 1bfdec9fc2c..ea004937ccd 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -7162,6 +7162,201 @@ OPENAI_DEPLOYMENT_PRICING_OVERRIDE: Final = CostTrackingTestCase( ), ) +_CC_CUSTOM_MODEL_COST_HEADER_FRAMES: Final = sse_frames( + { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "cc-custom-model", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": "scripted response"}, "finish_reason": None} + ], + }, + { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "cc-custom-model", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "cc-custom-model", + "choices": [], + "usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263}, + }, + done=True, +) + +_CC_CUSTOM_MODEL_RESPONSES_BODY: Final[dict[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "status": "completed", + "created_at": 1700000000, + "model": "cc-custom-model", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted response", "annotations": []}], + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 253, + "total_tokens": 263, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, +} + +OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER: Final = CostTrackingTestCase( + name="openai-deployment-zero-pricing-ignores-upstream-cost-header", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0), + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "openai-deployment-zero-pricing-ignores-upstream-cost-header"}], + "stream": False, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "model": "cc-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "scripted response"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263}, + }, + headers={"x-litellm-response-cost": "0.000413"}, + ), + expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253), +) + +OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_STREAM: Final = CostTrackingTestCase( + name="openai-deployment-zero-pricing-ignores-upstream-cost-header-stream", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0), + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "openai-deployment-zero-pricing-ignores-upstream-cost-header"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=_CC_CUSTOM_MODEL_COST_HEADER_FRAMES, + headers={"x-litellm-response-cost": "0.000413"}, + ), + expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253), +) + +OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_MESSAGES: Final = CostTrackingTestCase( + name="openai-deployment-zero-pricing-ignores-upstream-cost-header-messages", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + endpoint="/v1/messages", + deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0), + request={ + "model": "$MODEL", + "max_tokens": 300, + "messages": [{"role": "user", "content": "openai-deployment-zero-pricing-ignores-upstream-cost-header"}], + }, + response=JsonResponse( + content_type="application/json", + body=_CC_CUSTOM_MODEL_RESPONSES_BODY, + headers={"x-litellm-response-cost": "0.000413"}, + ), + expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253), +) + +OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_RESPONSES: Final = CostTrackingTestCase( + name="openai-deployment-zero-pricing-ignores-upstream-cost-header-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + endpoint="/v1/responses", + deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.0, output_cost_per_token=0.0), + request={ + "model": "$MODEL", + "input": "openai-deployment-zero-pricing-ignores-upstream-cost-header", + "stream": False, + }, + response=JsonResponse( + content_type="application/json", + body=_CC_CUSTOM_MODEL_RESPONSES_BODY, + headers={"x-litellm-response-cost": "0.000413"}, + ), + expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=10, completion_tokens=253), +) + +OPENAI_DEPLOYMENT_PRICING_IGNORES_UPSTREAM_COST_HEADER: Final = CostTrackingTestCase( + name="openai-deployment-pricing-ignores-upstream-cost-header", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + deployment=Deployment(model="openai/cc-custom-model", input_cost_per_token=0.001, output_cost_per_token=0.002), + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "openai-deployment-pricing-ignores-upstream-cost-header"}], + "stream": False, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$UNIQUE_ID", + "object": "chat.completion", + "model": "cc-custom-model", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "scripted response"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263}, + }, + headers={"x-litellm-response-cost": "0.000413"}, + ), + expected=ExactExpected( + spend=0.516, input_cost=0.01, output_cost=0.506, prompt_tokens=10, completion_tokens=253, rollups=True + ), +) + +GPT_5_6_UPSTREAM_COST_HEADER_BILLS_HEADER_VALUE: Final = CostTrackingTestCase( + name="gpt-5.6-upstream-cost-header-bills-header-value", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "gpt-5.6-upstream-cost-header-bills-header-value"}], + "stream": False, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "model": "gpt-5.6", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "scripted response"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 253, "total_tokens": 263}, + }, + headers={"x-litellm-response-cost": "0.000413"}, + ), + expected=ExactExpected( + spend=0.000413, + input_cost=0.0, + output_cost=0.000413, + prompt_tokens=10, + completion_tokens=253, + breakdown_persisted=False, + ), +) + GPT_5_6_UPSTREAM_400_ZERO_SPEND: Final = CostTrackingTestCase( name="gpt-5.6-upstream_400_zero_spend", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -8134,6 +8329,12 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_5_6_RESPONSES_SERVICE_TIER_FLEX, GPT_5_6_RESPONSES_SERVICE_TIER_PRIORITY, OPENAI_DEPLOYMENT_PRICING_OVERRIDE, + OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER, + OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_STREAM, + OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_MESSAGES, + OPENAI_DEPLOYMENT_ZERO_PRICING_IGNORES_UPSTREAM_COST_HEADER_RESPONSES, + OPENAI_DEPLOYMENT_PRICING_IGNORES_UPSTREAM_COST_HEADER, + GPT_5_6_UPSTREAM_COST_HEADER_BILLS_HEADER_VALUE, GPT_5_6_UPSTREAM_400_ZERO_SPEND, GPT_5_6_UPSTREAM_401_ZERO_SPEND, GPT_5_6_UPSTREAM_500_STREAM_REQUEST_ZERO_SPEND, diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 7300030ca77..9304100f3a5 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -129,6 +129,7 @@ class JsonResponse(BaseModel): content_type: Literal["application/json"] body: dict[str, JsonValue] status: int = 200 + headers: dict[str, str] = Field(default_factory=dict) class SseResponse(BaseModel): @@ -137,6 +138,7 @@ class SseResponse(BaseModel): content_type: Literal["text/event-stream"] frames: tuple[str, ...] frame_delay_ms: int = Field(default=0, ge=0) + headers: dict[str, str] = Field(default_factory=dict) class EventStreamEvent(BaseModel): @@ -329,6 +331,8 @@ class CostTrackingTestCase(BaseModel): def reports_provider_cost(self) -> bool: if not isinstance(self.response, JsonResponse): return False + if "x-litellm-response-cost" in self.response.headers: + return True usage: Final = self.response.body.get("usage") return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float)) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 01cce6a3e0f..5d5d53d793e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -186,6 +186,47 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 +@pytest.mark.parametrize( + ("input_cost_per_token", "output_cost_per_token", "expected_cost"), + ((0.0, 0.0, 0.0), (0.001, 0.002, 10 * 0.001 + 253 * 0.002)), +) +def test_response_cost_calculator_custom_pricing_ignores_upstream_response_cost_header( + monkeypatch: pytest.MonkeyPatch, + input_cost_per_token: float, + output_cost_per_token: float, + expected_cost: float, +) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.register_model( + model_cost={ + "cc-custom-priced-chat": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + } + } + ) + response: Final = ModelResponse( + model="cc-custom-priced-chat", + choices=[Choices(message=Message(content="ok"))], + usage=Usage(prompt_tokens=10, completion_tokens=253, total_tokens=263), + ) + response._hidden_params = {"additional_headers": {"llm_provider-x-litellm-response-cost": 0.000413}} + + cost: Final = response_cost_calculator( + response_object=response, + model="cc-custom-priced-chat", + custom_llm_provider="openai", + call_type="completion", + optional_params={}, + custom_pricing=True, + ) + + assert cost == pytest.approx(expected_cost) + + def test_response_cost_calculator_keeps_optional_params_out_of_hidden_params(): class MockResponse(BaseModel): pass