fix(cost): pass only per-second rates to native /v1/messages cost tracking
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Other deployment overrides keep billing as on main, matching chat's per-second gate
This commit is contained in:
mateo-berri 2026-10-28 22:36:25 -07:00
parent e4d6586b87
commit d009dc184a
3 changed files with 29 additions and 14 deletions

View file

@ -2551,7 +2551,7 @@ class BaseLLMHTTPHandler:
"preset_cache_key": None,
"stream_response": {},
"model_info": kwargs.get("model_info"),
**litellm_params.custom_pricing_overrides(),
**litellm_params.per_second_pricing(),
**vertex_location_params,
**anthropic_messages_optional_request_params,
},

View file

@ -4130,11 +4130,18 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
"""
return {k: v for k, v in model_info.items() if k not in cls.model_fields}
def custom_pricing_overrides(self) -> Mapping[str, float]:
def custom_pricing_overrides(self) -> Mapping[str, object]:
return MappingProxyType(
self.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
)
def per_second_pricing(self) -> Mapping[str, float]:
return MappingProxyType(
self.model_dump(
include={"cost_per_second", "input_cost_per_second", "output_cost_per_second"}, exclude_none=True
)
)
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})

View file

@ -451,27 +451,26 @@ _PER_SECOND_RATE: Final = 0.5
_UPSTREAM_PAUSE_SECONDS: Final = 0.3
def _per_second_router() -> Router:
def _priced_router(**pricing: float) -> Router:
return Router(
model_list=[
{
"model_name": _ALIAS,
"litellm_params": {
"model": _ANTHROPIC_MODEL,
"api_key": "fake-key",
"input_cost_per_second": _PER_SECOND_RATE,
"output_cost_per_second": _PER_SECOND_RATE,
},
"litellm_params": {"model": _ANTHROPIC_MODEL, "api_key": "fake-key", **pricing},
}
]
)
def _per_second_router() -> Router:
return _priced_router(input_cost_per_second=_PER_SECOND_RATE, output_cost_per_second=_PER_SECOND_RATE)
def _anthropic_sse_event(event: Mapping[str, object]) -> bytes:
return f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
async def _route_like_the_proxy(**request: object) -> object:
async def _route_like_the_proxy(router: Router, **request: object) -> object:
logging_obj, routed_request = litellm.utils.function_setup(
original_function="aanthropic_messages",
rules_obj=litellm.utils.Rules(),
@ -481,7 +480,7 @@ async def _route_like_the_proxy(**request: object) -> object:
max_tokens=100,
**request,
)
return await _per_second_router().aanthropic_messages(**routed_request, litellm_logging_obj=logging_obj)
return await router.aanthropic_messages(**routed_request, litellm_logging_obj=logging_obj)
async def _sse_pausing_after_the_first_event() -> AsyncIterator[bytes]:
@ -492,7 +491,16 @@ async def _sse_pausing_after_the_first_event() -> AsyncIterator[bytes]:
@pytest.mark.asyncio
async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_call_duration(respx_mock, monkeypatch):
@pytest.mark.parametrize(
"pricing",
[
{"input_cost_per_second": _PER_SECOND_RATE, "output_cost_per_second": _PER_SECOND_RATE},
{"cost_per_second": _PER_SECOND_RATE},
],
)
async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_call_duration(
respx_mock, monkeypatch, pricing: Mapping[str, float]
):
messages: Final = _unique_messages()
recorder: Final = _RecordingLogger(messages)
monkeypatch.setattr(litellm, "callbacks", [recorder])
@ -503,7 +511,7 @@ async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_
respx_mock.post(url__regex=_ANTHROPIC_URL).mock(side_effect=slow_upstream)
started: Final = time.monotonic()
await _route_like_the_proxy(messages=messages)
await _route_like_the_proxy(_priced_router(**pricing), messages=messages)
await _wait_for_payload(recorder)
elapsed: Final = time.monotonic() - started
@ -525,7 +533,7 @@ async def test_router_aanthropic_messages_bills_a_per_second_deployment_for_the_
)
)
started: Final = time.monotonic()
stream: Final = await _route_like_the_proxy(messages=messages, stream=True)
stream: Final = await _route_like_the_proxy(_per_second_router(), messages=messages, stream=True)
await _stream_events(stream)
await _wait_for_payload(recorder)
elapsed: Final = time.monotonic() - started