diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 96d687b44fb..9915d76433d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -839,6 +839,22 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool: ) +def _declares_rates(model_info: ModelInfo | None) -> bool: + entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None + return entry is not None and _cost_map_entry_prices_anything(entry) + + +def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None: + """The provider to price an explicit base_model under, when the deployment's own provider has + no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does""" + if custom_llm_provider is None or _declares_rates(_cost_map_model_info(base_model, custom_llm_provider)): + return None + own_model_info: Final = _cost_map_model_info(base_model, None) + if own_model_info is None or not _declares_rates(own_model_info): + return None + return own_model_info["litellm_provider"] + + def select_model_name_for_cost_calc( model: str | None, completion_response: object | None, @@ -1500,10 +1516,21 @@ def completion_cost( explicit_pricing: Final = custom_pricing is True or base_model is not None custom_priced_request_model: Final = model if custom_pricing is True else None + base_model_provider: Final = ( + _base_model_pricing_provider( + base_model, + _get_provider_for_cost_calc( + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + ), + ) + if base_model is not None and custom_pricing is not True + else None + ) selected_model: Final = select_model_name_for_cost_calc( model=model, completion_response=completion_response, - custom_llm_provider=custom_llm_provider, + custom_llm_provider=base_model_provider or custom_llm_provider, custom_pricing=custom_pricing, base_model=base_model, router_model_id=router_model_id, @@ -1850,6 +1877,13 @@ def completion_cost( if litellm_logging_obj is not None: request_model_for_cost = litellm_logging_obj.model + pricing_provider: str | None = ( + base_model_provider + if base_model_provider is not None and idx == 0 + else custom_llm_provider + if isinstance(custom_llm_provider, str) + else None + ) ( prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar, @@ -1857,7 +1891,7 @@ def completion_cost( model=model, prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0, - custom_llm_provider=custom_llm_provider, + custom_llm_provider=pricing_provider, response_time_ms=response_time_ms, region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, @@ -1954,12 +1988,9 @@ def completion_cost( _cache_creation_cost: float | None = None _billed_token_rates: BilledTokenRates | None = None if cost_per_token_usage_object is not None and model: - _breakdown_provider: str | None = ( - custom_llm_provider if isinstance(custom_llm_provider, str) else None - ) _token_type_breakdown = get_token_type_cost_breakdown( model=model, - custom_llm_provider=_breakdown_provider, + custom_llm_provider=pricing_provider, usage=cost_per_token_usage_object, service_tier=service_tier, data_residency=data_residency, diff --git a/tests/integration/cost_calculation/test_base_model_cross_provider_spend.py b/tests/integration/cost_calculation/test_base_model_cross_provider_spend.py new file mode 100644 index 00000000000..02d8ca9cc73 --- /dev/null +++ b/tests/integration/cost_calculation/test_base_model_cross_provider_spend.py @@ -0,0 +1,146 @@ +"""A chatgpt/ subscription deployment has a cost-map row without rates. With model_info.base_model pointing +at a priced row of another provider, the spend row is billed from that row (#46233)""" + +import json +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support import codex_vendor as cv +from integration._support import responses_vendor as rv +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(240) + +_MODEL: Final = "cc-subscription-sol" +_INPUT_TOKENS: Final = 1000 +_OUTPUT_TOKENS: Final = 100 +_COST_MAP: Final[Mapping[str, Mapping[str, JsonValue]]] = { + f"chatgpt/{_MODEL}": {"litellm_provider": "chatgpt", "mode": "responses"}, + _MODEL: { + "litellm_provider": "openai", + "mode": "responses", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 8e-6, + }, + f"azure/{_MODEL}": { + "litellm_provider": "azure", + "mode": "responses", + "input_cost_per_token": 3e-6, + "output_cost_per_token": 5e-6, + }, +} + + +@dataclass(frozen=True, slots=True) +class _Rig: + wire: Wire + proxy: OwnedProxy + + @property + def gateway(self) -> Gateway: + return self.proxy.gateway + + +def _completed_response(model: str) -> Mapping[str, JsonValue]: + tag: Final = uuid.uuid4().hex + message: Final[dict[str, JsonValue]] = { + "id": f"msg_{tag}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "priced", "annotations": []}], + } + return { + "id": f"resp_{tag}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": model, + "output": [message], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, + } + + +def _respond(request: Request) -> Reply: + if request.method == "GET": + assert request.target.endswith("/_cost_map"), request.target + return Reply(body=json.dumps(_COST_MAP).encode()) + assert request.target.endswith("/responses"), request.target + assert request.headers.get("authorization") == f"Bearer {cv.TOKEN}", request.headers + body: Final = rv.JSON_OBJECT.validate_json(request.body) + assert body["model"] == _MODEL, body + response: Final = _completed_response(_MODEL) + events: Final[tuple[Mapping[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + {"type": "response.completed", "sequence_number": 1, "response": response}, + ) + return Reply(content_type="text/event-stream", chunks=tuple(rv.sse(event) for event in events)) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("base-model-cross-provider-rig") + with gateway_from_environment() as gateway, wire_server(_respond) as wire: + overrides: Final = { + "CHATGPT_TOKEN_DIR": str(cv.login(directory)), + "CHATGPT_API_BASE": wire.url, + "LITELLM_MODEL_COST_MAP_URL": f"{wire.url}/_cost_map", + "MODEL_COST_MAP_MIN_MODEL_COUNT": "1", + "MODEL_COST_MAP_MAX_SHRINK_RATIO": "0", + } + config: Final = cv.proxy_config(directory, probe=False) + with owned_proxy_process( + gateway, directory, overrides, config=config, remove_environment=("LITELLM_LOCAL_MODEL_COST_MAP",) + ) as owned: + yield _Rig(wire, owned) + + +@pytest.mark.parametrize( + ("base_model", "expected_spend"), + [ + (_MODEL, _INPUT_TOKENS * 1e-6 + _OUTPUT_TOKENS * 8e-6), + (f"openai/{_MODEL}", _INPUT_TOKENS * 1e-6 + _OUTPUT_TOKENS * 8e-6), + (f"azure/{_MODEL}", _INPUT_TOKENS * 3e-6 + _OUTPUT_TOKENS * 5e-6), + ], +) +def test_base_model_bills_the_priced_row_when_the_deployment_provider_row_has_no_rates( + rig: _Rig, base_model: str, expected_spend: float +) -> None: + with rig.gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"chatgpt/{_MODEL}", + api_base=rig.wire.url, + api_key=None, + model_info={"base_model": base_model}, + ) + response: Final = rig.gateway.request( + "POST", "/v1/responses", {"model": model, "input": [{"role": "user", "content": "price control"}]} + ) + assert response.status_code == 200, response.text + assert float(response.headers.get("x-litellm-response-cost", "0")) == pytest.approx(expected_spend), ( + response.headers + ) + rows: Final = eventually( + lambda: read_rows('SELECT spend, status FROM "LiteLLM_SpendLogs" WHERE model_group = %s', (model,)), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "success", rows + assert float(rows[0]["spend"]) == pytest.approx(expected_spend), rows diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 93f2352db60..b6fa2054514 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -4411,6 +4411,109 @@ def test_completion_cost_base_model_ignores_regional_row(_local_model_cost_map): ) == pytest.approx(1000 * flat["input_cost_per_token"]) +def _chatgpt_responses_result(model: str) -> ResponsesAPIResponse: + response: Final = ResponsesAPIResponse( + id="resp-chatgpt", + created_at=1700000000, + model=model, + output=[], + usage=ResponseAPIUsage(input_tokens=1000, output_tokens=100, total_tokens=1100), + ) + response._hidden_params = {"custom_llm_provider": "chatgpt"} + return response + + +@pytest.mark.parametrize( + ("base_model", "priced_row"), + [ + ("base-model-cost-sol", "base-model-cost-sol"), + ("openai/base-model-cost-sol", "base-model-cost-sol"), + ("azure/base-model-cost-sol", "azure/base-model-cost-sol"), + ], +) +def test_base_model_prices_deployment_whose_provider_row_has_no_rates( + monkeypatch: pytest.MonkeyPatch, base_model: str, priced_row: str +) -> None: + """A chatgpt/ deployment's own cost-map row carries no rates, so an explicit base_model must be + priced from the base model's row instead of resolving back to the unpriced chatgpt/ row""" + monkeypatch.setitem( + litellm.model_cost, + "chatgpt/base-model-cost-sol", + {"litellm_provider": "chatgpt", "mode": "responses"}, + ) + monkeypatch.setitem( + litellm.model_cost, + "base-model-cost-sol", + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 8e-6, + "litellm_provider": "openai", + "mode": "responses", + }, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/base-model-cost-sol", + { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 5e-6, + "litellm_provider": "azure", + "mode": "responses", + }, + ) + row: Final = litellm.model_cost[priced_row] + + cost: Final = response_cost_calculator( + response_object=_chatgpt_responses_result("base-model-cost-sol"), + model="chatgpt/base-model-cost-sol", + custom_llm_provider="chatgpt", + call_type="aresponses", + optional_params={}, + base_model=base_model, + ) + + assert cost == pytest.approx(1000 * row["input_cost_per_token"] + 100 * row["output_cost_per_token"]) + + +def test_base_model_keeps_deployment_provider_row_that_declares_rates(monkeypatch: pytest.MonkeyPatch) -> None: + """When the deployment's provider row for the base model declares its own rates, those rates + still win over the base model's row from another provider""" + monkeypatch.setitem( + litellm.model_cost, + "chatgpt/base-model-rated-sol", + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 4e-6, + "litellm_provider": "chatgpt", + "mode": "responses", + }, + ) + monkeypatch.setitem( + litellm.model_cost, + "base-model-rated-sol", + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 8e-6, + "litellm_provider": "openai", + "mode": "responses", + }, + ) + deployment_row: Final = litellm.model_cost["chatgpt/base-model-rated-sol"] + + cost: Final = response_cost_calculator( + response_object=_chatgpt_responses_result("base-model-rated-sol"), + model="chatgpt/base-model-rated-sol", + custom_llm_provider="chatgpt", + call_type="aresponses", + optional_params={}, + base_model="base-model-rated-sol", + ) + + assert cost == pytest.approx( + 1000 * deployment_row["input_cost_per_token"] + 100 * deployment_row["output_cost_per_token"] + ) + + def test_select_model_name_unresolvable_alias_unchanged(_local_model_cost_map): """An alias that resolves to no known cost key keeps the legacy double-prefixed name."""