scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-14 04:09:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 97

fix(cost_calculator): honour base_model when the deployment provider row has no rates, with a live-proxy integration test (#46318)

Browse source 
* fix(cost_calculator): price an explicit base_model from its own row when the deployment provider row has no rates

A chatgpt/ deployment with model_info.base_model recorded zero spend because the
base model was looked up under the chatgpt provider, which resolves to the
deliberately unpriced chatgpt/ row. When the deployment provider's row for the
base model declares no rates, price it under the provider of the base model's
own row instead. Deployments whose provider row declares rates are unchanged.

(cherry picked from commit 42d9a8693e)

* test(integration): bill a chatgpt base_model deployment from the priced row on a live proxy

Co-authored-by: sla55er <stiliyanmarkov@gmail.com>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: sla55er <stiliyanmarkov@gmail.com>
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-10-13 17:10:10 -07:00 • committed by GitHub
parent bdc0893c28
commit ccdca3d39c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 286 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
litellm/cost_calculator.py
View file

@ -839,6 +839,22 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool:
)
def _declares_rates(model_info: ModelInfo | None) -> bool:
entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
return entry is not None and _cost_map_entry_prices_anything(entry)
def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None:
"""The provider to price an explicit base_model under, when the deployment's own provider has
no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does"""
if custom_llm_provider is None or _declares_rates(_cost_map_model_info(base_model, custom_llm_provider)):
return None
own_model_info: Final = _cost_map_model_info(base_model, None)
if own_model_info is None or not _declares_rates(own_model_info):
return None
return own_model_info["litellm_provider"]
def select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -1500,10 +1516,21 @@ def completion_cost(
explicit_pricing: Final = custom_pricing is True or base_model is not None
custom_priced_request_model: Final = model if custom_pricing is True else None
base_model_provider: Final = (
_base_model_pricing_provider(
base_model,
_get_provider_for_cost_calc(
model=model,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
),
)
if base_model is not None and custom_pricing is not True
else None
)
selected_model: Final = select_model_name_for_cost_calc(
model=model,
completion_response=completion_response,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=base_model_provider or custom_llm_provider,
custom_pricing=custom_pricing,
base_model=base_model,
router_model_id=router_model_id,
@ -1850,6 +1877,13 @@ def completion_cost(
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
pricing_provider: str | None = (
base_model_provider
if base_model_provider is not None and idx == 0
else custom_llm_provider
if isinstance(custom_llm_provider, str)
else None
)
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1857,7 +1891,7 @@ def completion_cost(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=pricing_provider,
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
@ -1954,12 +1988,9 @@ def completion_cost(
_cache_creation_cost: float | None = None
_billed_token_rates: BilledTokenRates | None = None
if cost_per_token_usage_object is not None and model:
_breakdown_provider: str | None = (
custom_llm_provider if isinstance(custom_llm_provider, str) else None
)
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=_breakdown_provider,
custom_llm_provider=pricing_provider,
usage=cost_per_token_usage_object,
service_tier=service_tier,
data_residency=data_residency,

146
tests/integration/cost_calculation/test_base_model_cross_provider_spend.py Normal file
View file

@ -0,0 +1,146 @@
"""A chatgpt/ subscription deployment has a cost-map row without rates. With model_info.base_model pointing
at a priced row of another provider, the spend row is billed from that row (#46233)"""
import json
import uuid
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from typing import Final
import pytest
from integration._support import codex_vendor as cv
from integration._support import responses_vendor as rv
from integration._support.client import Gateway, eventually, gateway_from_environment
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
pytestmark: Final = pytest.mark.timeout(240)
_MODEL: Final = "cc-subscription-sol"
_INPUT_TOKENS: Final = 1000
_OUTPUT_TOKENS: Final = 100
_COST_MAP: Final[Mapping[str, Mapping[str, JsonValue]]] = {
f"chatgpt/{_MODEL}": {"litellm_provider": "chatgpt", "mode": "responses"},
_MODEL: {
"litellm_provider": "openai",
"mode": "responses",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 8e-6,
},
f"azure/{_MODEL}": {
"litellm_provider": "azure",
"mode": "responses",
"input_cost_per_token": 3e-6,
"output_cost_per_token": 5e-6,
},
}
@dataclass(frozen=True, slots=True)
class _Rig:
wire: Wire
proxy: OwnedProxy
@property
def gateway(self) -> Gateway:
return self.proxy.gateway
def _completed_response(model: str) -> Mapping[str, JsonValue]:
tag: Final = uuid.uuid4().hex
message: Final[dict[str, JsonValue]] = {
"id": f"msg_{tag}",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "priced", "annotations": []}],
}
return {
"id": f"resp_{tag}",
"object": "response",
"created_at": 1,
"status": "completed",
"model": model,
"output": [message],
"usage": {
"input_tokens": _INPUT_TOKENS,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens": _OUTPUT_TOKENS,
"output_tokens_details": {"reasoning_tokens": 0},
"total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS,
},
}
def _respond(request: Request) -> Reply:
if request.method == "GET":
assert request.target.endswith("/_cost_map"), request.target
return Reply(body=json.dumps(_COST_MAP).encode())
assert request.target.endswith("/responses"), request.target
assert request.headers.get("authorization") == f"Bearer {cv.TOKEN}", request.headers
body: Final = rv.JSON_OBJECT.validate_json(request.body)
assert body["model"] == _MODEL, body
response: Final = _completed_response(_MODEL)
events: Final[tuple[Mapping[str, JsonValue], ...]] = (
{
"type": "response.created",
"sequence_number": 0,
"response": {**response, "status": "in_progress", "output": []},
},
{"type": "response.completed", "sequence_number": 1, "response": response},
)
return Reply(content_type="text/event-stream", chunks=tuple(rv.sse(event) for event in events))
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
directory: Final = tmp_path_factory.mktemp("base-model-cross-provider-rig")
with gateway_from_environment() as gateway, wire_server(_respond) as wire:
overrides: Final = {
"CHATGPT_TOKEN_DIR": str(cv.login(directory)),
"CHATGPT_API_BASE": wire.url,
"LITELLM_MODEL_COST_MAP_URL": f"{wire.url}/_cost_map",
"MODEL_COST_MAP_MIN_MODEL_COUNT": "1",
"MODEL_COST_MAP_MAX_SHRINK_RATIO": "0",
}
config: Final = cv.proxy_config(directory, probe=False)
with owned_proxy_process(
gateway, directory, overrides, config=config, remove_environment=("LITELLM_LOCAL_MODEL_COST_MAP",)
) as owned:
yield _Rig(wire, owned)
@pytest.mark.parametrize(
("base_model", "expected_spend"),
[
(_MODEL, _INPUT_TOKENS * 1e-6 + _OUTPUT_TOKENS * 8e-6),
(f"openai/{_MODEL}", _INPUT_TOKENS * 1e-6 + _OUTPUT_TOKENS * 8e-6),
(f"azure/{_MODEL}", _INPUT_TOKENS * 3e-6 + _OUTPUT_TOKENS * 5e-6),
],
)
def test_base_model_bills_the_priced_row_when_the_deployment_provider_row_has_no_rates(
rig: _Rig, base_model: str, expected_spend: float
) -> None:
with rig.gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"chatgpt/{_MODEL}",
api_base=rig.wire.url,
api_key=None,
model_info={"base_model": base_model},
)
response: Final = rig.gateway.request(
"POST", "/v1/responses", {"model": model, "input": [{"role": "user", "content": "price control"}]}
)
assert response.status_code == 200, response.text
assert float(response.headers.get("x-litellm-response-cost", "0")) == pytest.approx(expected_spend), (
response.headers
)
rows: Final = eventually(
lambda: read_rows('SELECT spend, status FROM "LiteLLM_SpendLogs" WHERE model_group = %s', (model,)),
lambda values: len(values) == 1,
seconds=70,
)
assert rows[0]["status"] == "success", rows
assert float(rows[0]["spend"]) == pytest.approx(expected_spend), rows

103
tests/unit/test_cost_calculator.py
View file

@ -4411,6 +4411,109 @@ def test_completion_cost_base_model_ignores_regional_row(_local_model_cost_map):
) == pytest.approx(1000 * flat["input_cost_per_token"])
def _chatgpt_responses_result(model: str) -> ResponsesAPIResponse:
response: Final = ResponsesAPIResponse(
id="resp-chatgpt",
created_at=1700000000,
model=model,
output=[],
usage=ResponseAPIUsage(input_tokens=1000, output_tokens=100, total_tokens=1100),
)
response._hidden_params = {"custom_llm_provider": "chatgpt"}
return response
@pytest.mark.parametrize(
("base_model", "priced_row"),
[
("base-model-cost-sol", "base-model-cost-sol"),
("openai/base-model-cost-sol", "base-model-cost-sol"),
("azure/base-model-cost-sol", "azure/base-model-cost-sol"),
],
)
def test_base_model_prices_deployment_whose_provider_row_has_no_rates(
monkeypatch: pytest.MonkeyPatch, base_model: str, priced_row: str
) -> None:
"""A chatgpt/ deployment's own cost-map row carries no rates, so an explicit base_model must be
priced from the base model's row instead of resolving back to the unpriced chatgpt/ row"""
monkeypatch.setitem(
litellm.model_cost,
"chatgpt/base-model-cost-sol",
{"litellm_provider": "chatgpt", "mode": "responses"},
)
monkeypatch.setitem(
litellm.model_cost,
"base-model-cost-sol",
{
"input_cost_per_token": 1e-6,
"output_cost_per_token": 8e-6,
"litellm_provider": "openai",
"mode": "responses",
},
)
monkeypatch.setitem(
litellm.model_cost,
"azure/base-model-cost-sol",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 5e-6,
"litellm_provider": "azure",
"mode": "responses",
},
)
row: Final = litellm.model_cost[priced_row]
cost: Final = response_cost_calculator(
response_object=_chatgpt_responses_result("base-model-cost-sol"),
model="chatgpt/base-model-cost-sol",
custom_llm_provider="chatgpt",
call_type="aresponses",
optional_params={},
base_model=base_model,
)
assert cost == pytest.approx(1000 * row["input_cost_per_token"] + 100 * row["output_cost_per_token"])
def test_base_model_keeps_deployment_provider_row_that_declares_rates(monkeypatch: pytest.MonkeyPatch) -> None:
"""When the deployment's provider row for the base model declares its own rates, those rates
still win over the base model's row from another provider"""
monkeypatch.setitem(
litellm.model_cost,
"chatgpt/base-model-rated-sol",
{
"input_cost_per_token": 2e-6,
"output_cost_per_token": 4e-6,
"litellm_provider": "chatgpt",
"mode": "responses",
},
)
monkeypatch.setitem(
litellm.model_cost,
"base-model-rated-sol",
{
"input_cost_per_token": 1e-6,
"output_cost_per_token": 8e-6,
"litellm_provider": "openai",
"mode": "responses",
},
)
deployment_row: Final = litellm.model_cost["chatgpt/base-model-rated-sol"]
cost: Final = response_cost_calculator(
response_object=_chatgpt_responses_result("base-model-rated-sol"),
model="chatgpt/base-model-rated-sol",
custom_llm_provider="chatgpt",
call_type="aresponses",
optional_params={},
base_model="base-model-rated-sol",
)
assert cost == pytest.approx(
1000 * deployment_row["input_cost_per_token"] + 100 * deployment_row["output_cost_per_token"]
)
def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map):
"""An alias that resolves to no known cost key keeps the legacy double-prefixed name."""