From ecdc0f9ab9beacc13240078fa95c94a68c3248c4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:23:27 -0700 Subject: [PATCH 1/5] fix(cost-map): declare Bedrock gpt-oss rows text-only so modality routing skips them (#48149) Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 16 +++++++++++++ model_prices_and_context_window.json | 16 +++++++++++++ .../test_model_prices_and_context_window.py | 23 +++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3edb30b4218..14fe355a805 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45847,6 +45847,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -45872,6 +45873,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -49587,6 +49589,7 @@ "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49606,6 +49609,7 @@ "supports_tool_choice": true }, "us-gov.openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -60156,6 +60160,7 @@ } }, "bedrock_mantle/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -60181,6 +60186,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -60206,6 +60212,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7e-08, "input_cost_per_token_flex": 7e-08, @@ -60229,6 +60236,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -68080,6 +68088,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68099,6 +68108,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68377,6 +68387,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68396,6 +68407,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68784,6 +68796,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68803,6 +68816,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -68943,6 +68957,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68962,6 +68977,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3edb30b4218..14fe355a805 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45847,6 +45847,7 @@ "output_cost_per_token": 0.0 }, "openai.gpt-oss-120b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -45872,6 +45873,7 @@ "supports_tool_choice": true }, "openai.gpt-oss-20b-1:0": { + "supports_vision": false, "supported_endpoints": [ "/v1/chat/completions" ], @@ -49587,6 +49589,7 @@ "supports_vision": false }, "us-gov.openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -49606,6 +49609,7 @@ "supports_tool_choice": true }, "us-gov.openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -60156,6 +60160,7 @@ } }, "bedrock_mantle/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_flex": 7.5e-08, @@ -60181,6 +60186,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3.5e-08, "input_cost_per_token_flex": 3.5e-08, @@ -60206,6 +60212,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "supports_vision": false, "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7e-08, "input_cost_per_token_flex": 7e-08, @@ -60229,6 +60236,7 @@ "supports_tool_choice": true }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "supports_vision": false, "input_cost_per_token": 7e-08, "input_cost_per_token_batches": 3e-08, "input_cost_per_token_flex": 3e-08, @@ -68080,6 +68088,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68099,6 +68108,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68377,6 +68387,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68396,6 +68407,7 @@ "supports_tool_choice": true }, "bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -68784,6 +68796,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68803,6 +68816,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", @@ -68943,6 +68957,7 @@ "supports_vision": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": { + "supports_vision": false, "input_cost_per_token": 8.4e-08, "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", @@ -68962,6 +68977,7 @@ "supports_tool_choice": true }, "bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": { + "supports_vision": false, "input_cost_per_token": 1.8e-07, "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 701b79d0ba9..9b9fec4c9a1 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -791,3 +791,26 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa sibling = rows["volcengine/doubao-seed-2-1-pro-260628"] assert isinstance(sibling, dict) assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")) + + +def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]: + return tuple( + key + for key, row in prices.items() + if "gpt-oss" in key and isinstance(row, dict) and str(row.get("litellm_provider", "")).startswith("bedrock") + ) + + +@pytest.mark.parametrize("path", [PRICES_PATH, BACKUP_PRICES_PATH], ids=["main", "backup"]) +def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblings(path: Path) -> None: + prices: Final = json.loads(path.read_text()) + rows: Final = _bedrock_gpt_oss_rows(prices) + assert rows, "no Bedrock gpt-oss rows found" + reference: Final = prices["openai.gpt-oss-safeguard-120b"]["supports_vision"] + undeclared: Final = tuple(key for key in rows if prices[key].get("supports_vision") is not reference) + assert undeclared == (), f"modality routing only skips rows that declare supports_vision: {undeclared}" + + +def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None: + assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True + assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True From f29e10c0ef641da97d9f1746ad9d5aa382ab9797 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:52:16 -0700 Subject: [PATCH 2/5] fix(router): keep image requests off deployments declared supports_vision false (#48152) * fix(router): keep image requests off deployments declared supports_vision false in pre-call checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): type the vision pre-call helpers so the basedpyright gate holds Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): keep the pre-call index set typed without a mutable annotation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): cover the vision pre-call helpers directly for the router coverage check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): type the vision pre-call test helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): cover the no-params and unresolvable-provider vision branches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 55 +++++++++++++- tests/unit/test_router/test_router.py | 102 ++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index da04e59f18a..705accbcf8d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14186,6 +14186,43 @@ class Router: deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params) ) + @staticmethod + def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool: + plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts + model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict + declared: Final = model_info.get("supports_vision") + if declared is not None: + return declared is False + raw_params: Final = plain_deployment.get("litellm_params") or {} + params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts + if not params: + return False + base_model: Final = model_info.get("base_model") or params.get("base_model") + try: + model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group) + return litellm.utils.is_vision_explicitly_disabled( + base_model if isinstance(base_model, str) else model, custom_llm_provider + ) + except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + verbose_router_logger.debug( + "litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e + ) + return False + + @staticmethod + def _request_carries_image( + messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is + request_kwargs: Mapping[str, object], + ) -> bool: + from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages + + structured: Final = resolve_structured_messages( + messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts + request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping + ) + return bool(structured) and request_contains_image_content(structured) + @staticmethod def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]: requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs)) @@ -14212,6 +14249,7 @@ class Router: - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models - filter models above rpm limits - if region given, filter out models not in that region / unknown region + - request carries an image and the deployment declares supports_vision false, unless every candidate does - [TODO] function call and model doesn't support function calling """ @@ -14221,7 +14259,7 @@ class Router: # We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request) _returned_deployments = list(healthy_deployments) - invalid_model_indices: Final = set() # Use set for O(1) membership checks + invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks _context_window_error = False _potential_error_str = "" @@ -14371,9 +14409,20 @@ class Router: model=model, llm_provider="", ) - if len(excluded_indices) > 0: + remaining: Final = tuple( + (idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices + ) + text_only_indices: Final[frozenset[int]] = ( + frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model)) + if remaining and self._request_carries_image(messages, request_settings) + else frozenset() + ) + dropped_indices: Final[frozenset[int]] = excluded_indices | ( + text_only_indices if len(text_only_indices) < len(remaining) else frozenset() + ) + if len(dropped_indices) > 0: # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops) - _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices] + _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices] return _returned_deployments diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9cc1bf6ce79..f06f9e08f7e 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -29871,3 +29871,105 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s wrapped.adopt_fallback_headers(fallback) assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens + + +_IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} +_IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}] +_IMAGE_RESPONSES_INPUT = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo="}, + ], + } +] +_VISION_LEG: Final[dict[str, object]] = {"litellm_params": {"model": "openai/gpt-5.6-terra"}, "model_info": {"id": "vision"}} +_DECLARED_TEXT_LEG: Final[dict[str, object]] = { + "litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, + "model_info": {"id": "declared-text", "supports_vision": False}, +} + + +def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router: + return litellm.Router( + model_list=[{"model_name": "mixed", **deployment} for deployment in deployments], + routing_strategy=routing_strategy, + enable_pre_call_checks=True, + ) + + +def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]: + return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)] + + +def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"] + + +def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [ + "vision", + "declared-text", + ] + + +def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image(): + router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG) + assert _pre_call_ids( + router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT} + ) == ["vision"] + + +def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only(): + other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}} + router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg) + assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [ + "declared-text", + "declared-text-2", + ] + + +def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests(): + custom_leg = { + "litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"}, + "model_info": {"id": "custom"}, + } + router = _pre_call_router(_VISION_LEG, custom_leg) + assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"] + + +def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model(): + catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}} + azure_text_leg = { + "litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"}, + "model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"}, + } + router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg) + assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"] + + +@pytest.mark.parametrize( + ("deployment", "expected"), + [ + (_DECLARED_TEXT_LEG, True), + ({"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, "model_info": {"supports_vision": True}}, False), + ({"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {}}, True), + ({"litellm_params": {"model": "azure/my-deploy"}, "model_info": {"base_model": "azure/gpt-audio"}}, True), + ({"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct"}, "model_info": {}}, False), + ({"litellm_params": {"model": "no-such-model-anywhere"}, "model_info": {}}, False), + ({"litellm_params": {}, "model_info": {}}, False), + ], + ids=["override-false", "override-true", "cost-map-false", "base-model-false", "undeclared", "unresolvable", "no-params"], +) +def test_deployment_declares_no_vision_reads_the_override_then_the_cost_map(deployment: dict[str, object], expected: bool): + assert litellm.Router._deployment_declares_no_vision(deployment, "mixed") is expected + + +def test_request_carries_image_sees_chat_parts_and_responses_input_items(): + assert litellm.Router._request_carries_image(_IMAGE_CHAT, {}) is True + assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True + assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False + assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False From e04e89c7010809cc051b14f0e43d71dd018cdaee Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Fri, 23 Oct 2026 22:53:04 -0700 Subject: [PATCH 3/5] fix(responses): honor per-call custom pricing on the native Responses API path (#48119) * fix(responses): honor per-call custom pricing on the native Responses API path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): narrow the custom pricing regression test to the type gates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(main): expose register_custom_pricing_for_request for the responses path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): mark custom pricing test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): price per-call custom rates request-locally instead of registering them globally Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): bind per-call rates on the request logging object instead of inferring them in Logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): keep Router deployments on the cost-map pricing path Router inherits cache rates onto the deployment's cost-map entry, so a routed Responses call binds no request-local price and keeps billing cached tokens at the inherited rate. Moves the deployment-id lookup to router_utils so the Responses and completion paths share it. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): gate request-local pricing on the typed deployment model_info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 4 + litellm/litellm_core_utils/litellm_logging.py | 5 + litellm/responses/main.py | 57 ++++++- .../test_responses_api_request_body.py | 146 ++++++++++++++++-- 4 files changed, 199 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..99944662bf8 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2264,6 +2264,8 @@ def response_cost_calculator( ### VERTEX LOCATION ### vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") region_name: str | None = None, + custom_cost_per_token: CostPerToken | None = None, + custom_cost_per_second: float | None = None, ) -> float: """ Returns @@ -2292,6 +2294,8 @@ def response_cost_calculator( optional_params=optional_params, custom_pricing=custom_pricing, base_model=base_model, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, litellm_model_name=litellm_model_name, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d75d35e4531..41f13817d75 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -161,6 +161,7 @@ from litellm.types.utils import ( CachingDetails, CallTypes, CostBreakdown, + CostPerToken, CustomPricingLiteLLMParams, DynamicPromptManagementParamLiteral, EmbeddingResponse, @@ -621,6 +622,8 @@ class Logging(LiteLLMLoggingBaseClass): prometheusLogger, \ slack_app custom_pricing: bool = False + custom_cost_per_token: CostPerToken | None = None + custom_cost_per_second: float | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -2096,6 +2099,8 @@ class Logging(LiteLLMLoggingBaseClass): "call_type": self.call_type, "optional_params": self.optional_params, "custom_pricing": custom_pricing, + "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, + "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 0b526adf4fa..e7380c26529 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -6,10 +6,10 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast import httpx -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from typing_extensions import ( TypeIs, # noqa: TID251 # narrows a client input item before its mark is read assert_never, @@ -87,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency from litellm.types.responses.main import * from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params +from litellm.types.utils import ( + ADDRESSED_RESPONSE_ID_FIELD, + CallTypes, + CostPerToken, + CustomPricingLiteLLMParams, + all_litellm_params, +) from litellm.utils import ( ProviderConfigManager, client, @@ -546,6 +552,47 @@ async def aresponses_api_with_mcp( return response +def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]: + return MappingProxyType( + litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True) + ) + + +class _DeploymentModelInfo(BaseModel): + model_config = ConfigDict(frozen=True) + id: object = None + + +_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter( + Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")] +) + + +def _is_router_deployment(deployment_model_info: object) -> bool: + """A Router stamps the deployment's model_info (with its id) onto every call it routes.""" + parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info) + return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None + + +_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) + + +def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None: + if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None: + return None + return _COST_PER_TOKEN.validate_python( + litellm_params.model_dump( + include={ + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + }, + exclude_none=True, + ) + ) + + def _bridges_to_chat_completions( responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool ) -> bool: @@ -1673,6 +1720,7 @@ def responses( optional_params=dict(responses_api_request_params), litellm_params={ **responses_api_request_params, + **_per_call_custom_pricing(litellm_params), "aresponses": _is_async, "litellm_call_id": litellm_call_id, "model_info": kwargs.get("model_info"), @@ -1683,6 +1731,9 @@ def responses( custom_llm_provider=custom_llm_provider, ) + if not _is_router_deployment(deployment_model_info): + litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params) + litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second # Decode any litellm-encoded encrypted-content item IDs back to their original IDs input = with_compaction_items_as_summaries( with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 36b5d8f135b..bf571c74e22 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -8,7 +8,8 @@ import asyncio import copy import json import logging -from collections.abc import Awaitable, Callable +import math +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path from importlib import import_module from typing import Final @@ -17,10 +18,12 @@ from unittest.mock import AsyncMock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.llms.openai import ResponsesAPIResponse from tests.unit.llms.bedrock.slow_upstream import ( STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, @@ -333,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up( def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]: - return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False) + return tuple( + level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False + ) def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]: @@ -623,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_ from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -645,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs( from litellm.responses.main import _aresponses_websocket with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -671,7 +678,9 @@ _ORIGINAL_WS_INPUT = [ @pytest.mark.asyncio @pytest.mark.parametrize("nested", [False, True]) -async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket +async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame( + nested: bool, +): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket from unittest.mock import MagicMock from litellm.responses.main import _aresponses_websocket @@ -682,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame ) with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -711,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}' with patch.object( - import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket", + import_module("litellm.responses.main").base_llm_http_handler, + "async_responses_websocket", new_callable=AsyncMock, ) as mock_ws: await _aresponses_websocket( @@ -968,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped.""" monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com") injected_client = AsyncHTTPHandler() - mock_post = AsyncMock( - return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200) - ) + mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)) injected_client.post = mock_post await litellm.aresponses( @@ -1234,6 +1243,77 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) -> ] +async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse: + response: Final = ( + litellm.responses(model=model, input="hi", **pricing) + if sync_mode + else await litellm.aresponses(model=model, input="hi", **pricing) + ) + assert isinstance(response, ResponsesAPIResponse) + return response + + +def _response_cost(response: ResponsesAPIResponse) -> float: + hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params") + return hidden_params["response_cost"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map")) +async def test_responses_bills_per_call_custom_pricing_from_usage( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1])) + ) + pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + + response: Final = await _invoke_responses(sync_mode, model, pricing) + + assert response.usage is not None + expected: Final = ( + response.usage.input_tokens * pricing["input_cost_per_token"] + + response.usage.output_tokens * pricing["output_cost_per_token"] + ) + assert math.isclose(_response_cost(response), expected) + + +@pytest.mark.asyncio +async def test_concurrent_responses_calls_keep_their_own_per_call_prices( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + model: Final = "openai/gpt-5.4-nano" + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + cost_map_row_before: Final = copy.deepcopy(model_cost.get(model)) + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano")) + ) + rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5)) + + responses: Final = await asyncio.gather( + *( + _invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate}) + for in_rate, out_rate in rates + ) + ) + + for response, (in_rate, out_rate) in zip(responses, rates, strict=True): + assert response.usage is not None + assert math.isclose( + _response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate + ) + assert model_cost.get(model) == cost_map_row_before + + @pytest.mark.parametrize("asynchronous", [False, True]) @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize( @@ -1339,3 +1419,49 @@ async def test_responses_opencode_session_uses_effective_destination( assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") assert sent.headers["authorization"] == "Bearer test-key" assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"} + + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +@pytest.mark.asyncio +async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost") + monkeypatch.setattr(litellm, "model_cost", dict(model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "nano", + "litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates}, + "model_info": {"id": "nano-deployment"}, + } + ] + ) + payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano") + usage: Final = _JSON_OBJECT.validate_python(payload["usage"]) + cached_tokens: Final = 8 + respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}}, + ) + ) + + response: Final = await router.aresponses(model="nano", input="hi") + + assert isinstance(response, ResponsesAPIResponse) + assert response.usage is not None + uncached_input: Final = response.usage.input_tokens - cached_tokens + full_price: Final = ( + response.usage.input_tokens * rates["input_cost_per_token"] + + response.usage.output_tokens * rates["output_cost_per_token"] + ) + floor: Final = ( + uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"] + ) + assert floor < _response_cost(response) < full_price From 455923014a6ef95ac5d98a3f8845969fd44d1450 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 09:07:02 -0700 Subject: [PATCH 4/5] feat(cost): bill openai 12-hour prompt cache writes at their own rate (#48088) * feat: add OpenAI 12-hour cache write pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix: omit empty cache-write response fields * style: format cache-write fields * fix(cost): bill openai 30m and 12h cache writes from the usage split, add gpt-5.6 rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost): apply the 12h long-context write rate only above the token threshold Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(cost): key cache write rates by TTL instead of growing the rate tuple _get_token_base_cost returned a positional 6-tuple and every cache lifetime added a slot to it, to the off-peak helper and to every caller. It now returns a TokenBaseCosts dataclass whose cache write rates live in one mapping keyed by TTL. The TTLs come from the ephemeral_*_input_tokens fields on CacheCreationTokenDetails, so supporting a new lifetime only means adding that usage field and its rate key A lifetime with no rate of its own now always falls back to the cache creation rate after off-peak pricing is applied. Before this the non-tiered 12h rate and the tiered 1h rate fell back to the rate from before off-peak, so the same request billed differently depending on the pricing path * refactor(cost): hardcode the cache write TTLs as a Literal instead of parsing usage field names Parsing ephemeral_*_input_tokens names saved nothing, since a new TTL still needs a usage field and provider parsing code. The TTLs are now a CacheWriteTTL Literal with an explicit rate key per TTL * refactor(cost): resolve cache write lifetime rates to floats at lookup Decide the 1h and 12h cost map key in the threshold loop, then resolve each rate once after off-peak with the base write rate as the default, so no None flows through the cost code. Add an integration case for a 12h write on a row without a 12h rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(cost): name every cache write TTL, marking 5m and 30m as base rate Co-Authored-By: Claude Opus 5.5 * refactor(cost): list every cache write TTL in one rate key table, None meaning the base write rate Co-Authored-By: Claude Opus 5.5 * refactor(cost): drop base rate entries from the cache write rate table and rely on fallback Co-Authored-By: Claude Opus 5.5 * refactor(cost): key cache write rates by plain string TTL Co-Authored-By: Claude Opus 5.5 * refactor(cost): fill missing cache write TTL rates with the base write rate after off-peak Co-Authored-By: Claude Opus 5.5 * refactor(cost): default missing cache write TTL rates to the base write rate inline Co-Authored-By: Claude Opus 5.5 * refactor(cost): rename cache write rate key table to make clear it only covers long TTLs Co-Authored-By: Claude Opus 5.5 * fix(cost): list every cache write TTL and bill writes of an unknown TTL at the base write rate Co-Authored-By: Claude Opus 5.5 * refactor(cost): resolve tiered cache write TTL rates after off-peak instead of passing None Co-Authored-By: Claude Opus 5.5 * refactor(cost): carry cache write TTL rates on TokenRates and drop TokenBaseCosts Co-Authored-By: Claude Opus 5.5 * refactor(cost): define TokenRates before its first use to drop the string annotation Co-Authored-By: Claude Opus 5.5 * fix(cost): build the TokenRates TTL default with default_factory and type tiers as Mapping MappingProxyType({}) as a dataclass default is rejected on Python 3.11, where mappingproxy is unhashable, so import litellm failed there. The tier type moves from dict[str, object] to Mapping[str, object] (LIT001), with tier_rate accepting a Mapping to match. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost): type select_tier_for_input tiers as Mapping so tier_rate callers stay typed The tier returned as a bare dict made every tier_rate call site a partially unknown argument once tier_rate took Mapping[str, object]. The range bounds are now validated with a TypeAdapter instead of indexing an unknown list. --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../llm_cost_calc/tiered_pricing.py | 42 ++- .../litellm_core_utils/llm_cost_calc/utils.py | 278 ++++++++++-------- litellm/llms/dashscope/cost_calculator.py | 3 +- ...odel_prices_and_context_window_backup.json | 48 +++ litellm/proxy/_lazy_openapi_snapshot.json | 40 +++ .../transformation.py | 26 +- litellm/responses/utils.py | 22 +- litellm/types/llms/openai.py | 17 +- litellm/types/utils.py | 14 + litellm/utils.py | 18 ++ model_prices_and_context_window.json | 48 +++ .../cost_calculation/cases/openai.py | 102 +++++++ .../integration/cost_calculation/cost_map.py | 2 + .../cost_calculation/cost_tracking_case.py | 6 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 269 ++++++++++++----- .../test_litellm_completion_responses.py | 4 +- tests/unit/responses/test_responses_utils.py | 41 ++- tests/unit/test_utils.py | 12 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 52 +++- 19 files changed, 811 insertions(+), 233 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py b/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py index 9bcc2b1743c..3040e36ec5f 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tiered_pricing.py @@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget reservation logic so neither has to depend on the other. """ -from typing import Final +from collections.abc import Mapping, Sequence +from typing import Final, SupportsFloat + +from pydantic import TypeAdapter, ValidationError + +_TIER_RANGE: Final = TypeAdapter(tuple[float, float]) -def _coerce_cost_per_token(value: float | str | None) -> float: +def _coerce_cost_per_token(value: object) -> float: """ Coerce a per-token cost into a float. Model cost values loaded from YAML config may arrive as strings (e.g. scientific notation like "4e-07"), which would break arithmetic. """ - if value is None: - return 0.0 if isinstance(value, str): try: return float(value) except ValueError: return 0.0 - return float(value) + if isinstance(value, SupportsFloat): + return float(value) + return 0.0 + + +def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None: + try: + return _TIER_RANGE.validate_python(tier.get("range")) + except ValidationError: + return None def select_tier_for_input( - tiered_pricing: list[dict], + tiered_pricing: Sequence[Mapping[str, object]], input_tokens: int, -) -> dict | None: +) -> Mapping[str, object] | None: """ Select the pricing tier for a request based on its total input token count. @@ -44,19 +56,21 @@ def select_tier_for_input( if not tiered_pricing or input_tokens <= 0: return None - sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0]) - valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2] - if not valid_tiers: + ranged_tiers: Final = sorted( + ((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None), + key=lambda ranged: ranged[0][0], + ) + if not ranged_tiers: return None - matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]] + matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end) if matching: return matching[0] - return valid_tiers[-1] + return ranged_tiers[-1][1] def tier_rate( - tier: dict, + tier: Mapping[str, object], cost_key: str, fallback_cost_key: str | None = None, ) -> float: @@ -68,4 +82,6 @@ def tier_rate( primary: Final = tier.get(cost_key) if primary is not None: return _coerce_cost_per_token(primary) + if fallback_cost_key is None: + return 0.0 return _coerce_cost_per_token(tier.get(fallback_cost_key, 0)) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a874459798d..5d4b1a82759 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -3,7 +3,7 @@ import re from collections.abc import Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field, replace from datetime import datetime, timezone, tzinfo from types import MappingProxyType from typing import Final, Literal, TypedDict, cast @@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) _DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields) +_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost" + +_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType( + { + "5m": _BASE_CACHE_WRITE_RATE_KEY, + "30m": _BASE_CACHE_WRITE_RATE_KEY, + "1hr": "cache_creation_input_token_cost_above_1hr", + "12hr": "cache_creation_input_token_cost_above_12hr", + } +) + +_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType( + {ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY} +) + + _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( "input_cost_per_token", "output_cost_per_token", @@ -356,7 +372,21 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide ) -def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None: +@dataclass(frozen=True, slots=True) +class TokenRates: + input_rate: float + output_rate: float + cache_read_rate: float + cache_creation_rate: float + reasoning_rate: float | None + cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) + + @property + def billed_reasoning_rate(self) -> float: + return self.output_rate if self.reasoning_rate is None else self.reasoning_rate + + +def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None: tiered_pricing: Final = model_info.get("tiered_pricing") if not isinstance(tiered_pricing, list) or not tiered_pricing: return None @@ -376,7 +406,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token") -def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None: +def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None: """ Resolve the base rates from a model's ``tiered_pricing`` table, if it has one. @@ -391,22 +421,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, if tier is None: return None - cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") completion_cost: Final = ( tier_rate(tier, "output_cost_per_token") if "output_cost_per_token" in tier else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0 ) - return ( - tier_rate(tier, "input_cost_per_token"), - completion_cost, - cache_creation_cost, - tier_rate(tier, "cache_creation_input_token_cost_above_1hr") - if "cache_creation_input_token_cost_above_1hr" in tier - and tier["cache_creation_input_token_cost_above_1hr"] is not None - else cache_creation_cost, - tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), + cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") + rates: Final = TokenRates( + input_rate=tier_rate(tier, "input_cost_per_token"), + output_rate=completion_cost, + cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), + cache_creation_rate=cache_creation_rate, + reasoning_rate=None, ) + return rates, tier def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool: @@ -532,19 +560,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None = return False -@dataclass(frozen=True, slots=True) -class TokenRates: - input_rate: float - output_rate: float - cache_read_rate: float - cache_creation_rate: float - reasoning_rate: float | None - - @property - def billed_reasoning_rate(self) -> float: - return self.output_rate if self.reasoning_rate is None else self.reasoning_rate - - def resolve_explicit_cache_read_cost_rate( model_info: ModelInfo, usage: Usage, @@ -649,7 +664,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, if off_peak is None: return rates off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token")) - return TokenRates( + return replace( + rates, input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate), output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate), cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate), @@ -661,32 +677,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, def _apply_off_peak_to_base_costs( model_info: ModelInfo, current_time: datetime | None, - base_costs: tuple[float, float, float, float | None, float], -) -> tuple[float, float, float, float, float]: - """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path - produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a - present one passes through untouched and an absent one resolves to the applied - cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate. + base_rates: TokenRates, + tier: Mapping[str, object], +) -> TokenRates: + """Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no + field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier + that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to + _resolve_billed_reasoning_rate. """ - prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs - rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=prompt, - output_rate=completion, - cache_read_rate=cache_read, - cache_creation_rate=cache_creation, - reasoning_rate=None, + rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates) + return replace( + rates, + cache_creation_rates_by_ttl=MappingProxyType( + { + ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key) + for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() + } ), ) - return ( - rates.input_rate, - rates.output_rate, - rates.cache_creation_rate, - rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr, - rates.cache_read_rate, - ) + + +def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str: + return preferred_key if model_info.get(preferred_key) is not None else fallback_key def _get_token_base_cost( @@ -696,7 +708,7 @@ def _get_token_base_cost( current_time: datetime | None = None, *, threshold_is_inclusive: bool = False, -) -> tuple[float, float, float, float, float]: +) -> TokenRates: """ Return prompt cost, completion cost, and cache costs for a given model and usage. @@ -709,15 +721,12 @@ def _get_token_base_cost( An absent cache-creation or cache-read rate always resolves to the resolved input rate, the way the tiered table and custom deployment pricing already do, since a provider that publishes no cache price bills cached tokens as ordinary input. An - absent 1h write rate resolves to the cache-creation rate, off-peak included. An + absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for all of them. - - Returns: - Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage) if tiered_base_costs is not None: - return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs) + return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs) # Get service tier aware cost keys input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) @@ -734,13 +743,13 @@ def _get_token_base_cost( # For image generation models that don't have output_cost_per_token, # use output_cost_per_image_token as the base cost (all output tokens are image tokens) - if completion_base_cost == 0.0 or completion_base_cost is None: + if completion_base_cost == 0.0: output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None) if output_image_cost is not None: completion_base_cost = output_image_cost cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None) - cache_creation_cost_above_1hr = get_cost_per_unit( - model_info, "cache_creation_input_token_cost_above_1hr", default_value=None + cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType( + {ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()} ) cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None) @@ -811,14 +820,6 @@ def _get_token_base_cost( if service_tier else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" ) - cache_creation_1hr_tiered_key = ( - get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens" - ) cache_read_tiered_key = ( get_service_tier_cost_key( f"cache_read_input_token_cost_above_{threshold_str}_tokens", @@ -830,10 +831,19 @@ def _get_token_base_cost( cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost) - cache_creation_cost_above_1hr = get_cost_per_unit( - model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr + cache_creation_keys_by_ttl = MappingProxyType( + { + ttl: _first_present_key( + model_info, + get_service_tier_cost_key( + f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens", + service_tier, + ), + key, + ) + for ttl, key in cache_creation_keys_by_ttl.items() + } ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) except (IndexError, ValueError): continue @@ -849,16 +859,26 @@ def _get_token_base_cost( resolved_cache_creation_cost: Final = ( input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost ) - - return _apply_off_peak_to_base_costs( + base_rates: Final = apply_off_peak_pricing( model_info, current_time, - ( - prompt_base_cost, - completion_base_cost, - resolved_cache_creation_cost, - cache_creation_cost_above_1hr, - resolved_cache_read_cost, + TokenRates( + input_rate=prompt_base_cost, + output_rate=completion_base_cost, + cache_read_rate=resolved_cache_read_cost, + cache_creation_rate=resolved_cache_creation_cost, + reasoning_rate=None, + ), + ) + return replace( + base_rates, + cache_creation_rates_by_ttl=MappingProxyType( + { + ttl: base_rates.cache_creation_rate + if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None + else rate + for ttl, key in cache_creation_keys_by_ttl.items() + } ), ) @@ -975,25 +995,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M def calculate_cache_writing_cost( cache_creation_tokens: int, cache_creation_token_details: CacheCreationTokenDetails | None, - cache_creation_cost_above_1hr: float, cache_creation_cost: float, + cache_creation_cost_by_ttl: Mapping[str, float], ) -> float: """ - Adjust cost of cache creation tokens based on the cache creation token details. + Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts. + Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate. """ - total_cost: float = 0.0 - if cache_creation_token_details is not None: - # get the number of 5m and 1h cache creation tokens - cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens - cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens - # add the number of 5m and 1h cache creation tokens to the cache creation tokens - total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0 - total_cost += ( - cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0 - ) - else: - total_cost += cache_creation_tokens * cache_creation_cost - return total_cost + if cache_creation_token_details is None: + return cache_creation_tokens * cache_creation_cost + tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType( + { + "5m": cache_creation_token_details.ephemeral_5m_input_tokens, + "30m": cache_creation_token_details.ephemeral_30m_input_tokens, + "1hr": cache_creation_token_details.ephemeral_1h_input_tokens, + "12hr": cache_creation_token_details.ephemeral_12h_input_tokens, + } + ) + unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0) + return unattributed_tokens * cache_creation_cost + sum( + (tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost) + for ttl, tokens in tokens_by_ttl.items() + ) class PromptTokensDetailsResult(TypedDict): @@ -1178,7 +1201,7 @@ def _calculate_input_cost( prompt_base_cost: float, cache_read_cost: float, cache_creation_cost: float, - cache_creation_cost_above_1hr: float, + cache_creation_cost_by_ttl: Mapping[str, float], service_tier: str | None = None, ) -> float: """ @@ -1248,8 +1271,8 @@ def _calculate_input_cost( prompt_cost += calculate_cache_writing_cost( cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"], - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, cache_creation_cost=cache_creation_cost, + cache_creation_cost_by_ttl=cache_creation_cost_by_ttl, ) ### CHARACTER COST @@ -1510,34 +1533,31 @@ def generic_cost_per_token( ) billing_time: Final = current_time if current_time is not None else current_billing_time() - ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, - ) = _get_token_base_cost( + base_costs: Final = _get_token_base_cost( model_info=resolved_model_info, usage=usage, service_tier=service_tier, current_time=billing_time, threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), ) + completion_base_cost: Final = base_costs.output_rate resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( model_info=resolved_model_info, usage=usage, current_time=billing_time, service_tier=service_tier, ) - effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost + effective_cache_read_cost: Final = ( + base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost + ) prompt_cost = _calculate_input_cost( prompt_tokens_details=prompt_tokens_details, model_info=resolved_model_info, - prompt_base_cost=prompt_base_cost, + prompt_base_cost=base_costs.input_rate, cache_read_cost=effective_cache_read_cost, - cache_creation_cost=cache_creation_cost, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + cache_creation_cost=base_costs.cache_creation_rate, + cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, service_tier=service_tier, ) @@ -1640,7 +1660,7 @@ class BilledTokenRates: cache_read_input_token_cost: float cache_read_input_audio_token_cost: float cache_creation_input_token_cost: float - cache_creation_input_token_cost_above_1hr: float + cache_creation_input_token_cost_by_ttl: Mapping[str, float] output_cost_per_reasoning_token: float def scaled(self, multiplier: float) -> "BilledTokenRates": @@ -1652,7 +1672,9 @@ class BilledTokenRates: cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier, cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier, cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, - cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, + cache_creation_input_token_cost_by_ttl=MappingProxyType( + {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()} + ), output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, ) @@ -1702,7 +1724,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat cache_read_input_token_cost=cache_read_rate, cache_read_input_audio_token_cost=cache_read_rate, cache_creation_input_token_cost=cache_creation_rate, - cache_creation_input_token_cost_above_1hr=cache_creation_rate, + cache_creation_input_token_cost_by_ttl=MappingProxyType({}), output_cost_per_reasoning_token=output_rate, ) @@ -1717,13 +1739,7 @@ def _cost_map_billed_rates( current_time: datetime | None, ) -> BilledTokenRates: billing_time: Final = current_time if current_time is not None else current_billing_time() - ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost_rate, - cache_creation_cost_above_1hr_rate, - cache_read_cost_rate, - ) = _get_token_base_cost( + base_costs: Final = _get_token_base_cost( model_info=model_info, usage=usage, service_tier=service_tier, @@ -1737,14 +1753,14 @@ def _cost_map_billed_rates( service_tier=service_tier, ) effective_cache_read_cost_rate: Final = ( - cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate + base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate ) reasoning_rate: Final = _resolve_billed_reasoning_rate( model_info=model_info, usage=usage, service_tier=service_tier, - completion_base_cost=completion_base_cost, + completion_base_cost=base_costs.output_rate, current_time=billing_time, ) audio_cache_read_rate: Final = get_cost_per_unit( @@ -1759,14 +1775,14 @@ def _cost_map_billed_rates( * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) ) return BilledTokenRates( - input_cost_per_token=prompt_base_cost, - output_cost_per_token=completion_base_cost, + input_cost_per_token=base_costs.input_rate, + output_cost_per_token=base_costs.output_rate, cache_read_input_token_cost=effective_cache_read_cost_rate, cache_read_input_audio_token_cost=( audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate ), - cache_creation_input_token_cost=cache_creation_cost_rate, - cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, + cache_creation_input_token_cost=base_costs.cache_creation_rate, + cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, output_cost_per_reasoning_token=reasoning_rate, ).scaled(multiplier) @@ -1854,8 +1870,8 @@ def get_token_type_cost_breakdown( else calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, - cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr, cache_creation_cost=rates.cache_creation_input_token_cost, + cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl, ) ) return TokenTypeCostBreakdown( @@ -1886,7 +1902,7 @@ def calculate_prompt_caching_savings( biller saw them rather than at the later spend write. """ model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider) - prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost( + base_costs: Final = _get_token_base_cost( model_info=model_info_with_cache_read_default, usage=usage, service_tier=service_tier, @@ -1899,9 +1915,11 @@ def calculate_prompt_caching_savings( current_time=billed_at, service_tier=service_tier, ) - effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost - write_rate: Final = cache_creation_cost or prompt_base_cost - write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate + prompt_base_cost: Final = base_costs.input_rate + effective_cache_read_cost: Final = ( + base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost + ) + write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost prompt_tokens_details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0) cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0) @@ -1909,7 +1927,9 @@ def calculate_prompt_caching_savings( cache_creation_details: Final = ( CacheCreationTokenDetails( ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0), + ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0), ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0), + ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0), ) if details is not None else None @@ -1918,8 +1938,10 @@ def calculate_prompt_caching_savings( write_premium: Final = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_details, - cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost, cache_creation_cost=write_rate - prompt_base_cost, + cache_creation_cost_by_ttl=MappingProxyType( + {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()} + ), ) uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( model_info, vertex_location diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 98504f54533..3f388637a83 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -7,6 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate. See https://help.aliyun.com/zh/model-studio/billing-for-model-studio """ +from collections.abc import Mapping from dataclasses import dataclass, replace from datetime import datetime from typing import Final @@ -72,7 +73,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates: ) -def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates: +def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates: # A tier that declares output rates keeps the request on them, all-or-nothing. A tier table # spelling out only input rates would serve every completion for free, so there the model's # own output rates stand in diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 14fe355a805..8c0c6dffdc7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -36795,6 +36795,12 @@ }, "gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_12hr": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, + "cache_creation_input_token_cost_above_12hr_priority": 5e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36880,6 +36886,12 @@ }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -36957,6 +36969,12 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_12hr": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, + "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, @@ -37043,6 +37061,12 @@ }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37117,6 +37141,12 @@ }, "gpt-5.6-sol": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37193,6 +37223,12 @@ }, "gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -37268,6 +37304,12 @@ }, "gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_12hr": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, @@ -74933,6 +74975,12 @@ }, "gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 0dd978ee855..4e183bf9d5c 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -22858,6 +22858,25 @@ "title": "Body_image_edit_api_openai_deployments__model__images_edits_post", "type": "object" }, + "CacheWriteTokensDetails": { + "additionalProperties": true, + "properties": { + "ttl_12h_tokens": { + "title": "Ttl 12H Tokens", + "type": "integer" + }, + "ttl_30m_tokens": { + "title": "Ttl 30M Tokens", + "type": "integer" + } + }, + "required": [ + "ttl_30m_tokens", + "ttl_12h_tokens" + ], + "title": "CacheWriteTokensDetails", + "type": "object" + }, "CachedTokensDetails": { "properties": { "audio_tokens": { @@ -24352,6 +24371,27 @@ ], "title": "Audio Tokens" }, + "cache_write_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Cache Write Tokens" + }, + "cache_write_tokens_details": { + "anyOf": [ + { + "$ref": "#/components/schemas/CacheWriteTokensDetails" + }, + { + "type": "null" + } + ] + }, "cached_tokens": { "default": 0, "title": "Cached Tokens", diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 02d9566d396..a9c7818b577 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3663,18 +3663,20 @@ class LiteLLMCompletionResponsesConfig: if count is not None } ) - response_usage.input_tokens_details = InputTokensDetails( - cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, - text_tokens=prompt_details.text_tokens, - audio_tokens=prompt_details.audio_tokens, - image_tokens=prompt_details.image_tokens, - video_tokens=prompt_details.video_tokens, - cached_tokens_details=( - cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None - ), - **cache_write_extra, - **cache_type_extra, - **grounding_request_counts, + response_usage.input_tokens_details = InputTokensDetails.model_validate( + { + "cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, + "text_tokens": prompt_details.text_tokens, + "audio_tokens": prompt_details.audio_tokens, + "image_tokens": prompt_details.image_tokens, + "video_tokens": prompt_details.video_tokens, + "cached_tokens_details": ( + cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None + ), + **cache_write_extra, + **cache_type_extra, + **grounding_request_counts, + } ) # Translate completion_tokens_details to output_tokens_details diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 503d85aaed9..f256480d952 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, + InputTokensDetails, OutputTokensDetails, ResponseAPIUsage, ResponseInputParam, @@ -23,6 +24,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import DecodedResponseId from litellm.types.utils import ( + CacheCreationTokenDetails, CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, SpecialEnums, @@ -41,6 +43,17 @@ def _output_token_detail(details: object, field: str) -> int | None: return value if isinstance(value, int) else None +def _cache_write_lifetimes(cache_write_tokens_details: object) -> CacheCreationTokenDetails | None: + """OpenAI Responses splits `cache_write_tokens` into `ttl_30m_tokens` and `ttl_12h_tokens`.""" + ttl_30m_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_30m_tokens") + ttl_12h_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_12h_tokens") + if ttl_30m_tokens is None and ttl_12h_tokens is None: + return None + return CacheCreationTokenDetails( + ephemeral_30m_input_tokens=ttl_30m_tokens, ephemeral_12h_input_tokens=ttl_12h_tokens + ) + + def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything return isinstance(value, list) @@ -1247,7 +1260,11 @@ class ResponseAPILoggingUtils: prompt_tokens_details: PromptTokensDetailsWrapper | None = None if response_api_usage.input_tokens_details: if isinstance(response_api_usage.input_tokens_details, dict): - prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details) + input_details: Final = InputTokensDetails.model_validate(response_api_usage.input_tokens_details) + prompt_tokens_details = PromptTokensDetailsWrapper( + **input_details.model_dump(exclude={"cache_write_tokens_details"}), + cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details), + ) else: prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), @@ -1259,6 +1276,9 @@ class ResponseAPILoggingUtils: ), video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), + cache_creation_token_details=_cache_write_lifetimes( + getattr(response_api_usage.input_tokens_details, "cache_write_tokens_details", None) + ), cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), google_maps_grounding_requests=getattr( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 62dc152ca84..9a4ecf37127 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -683,12 +683,12 @@ class ChatCompletionCachedContent(TypedDict): class PromptCacheBreakpoint(TypedDict): mode: ReadOnly[Literal["explicit"]] - ttl: NotRequired[ReadOnly[Literal["30m"]]] + ttl: NotRequired[ReadOnly[Literal["30m", "12h"]]] class PromptCacheOptions(TypedDict, total=False): mode: ReadOnly[Literal["implicit", "explicit"]] - ttl: ReadOnly[Literal["30m"]] + ttl: ReadOnly[Literal["30m", "12h"]] class ChatCompletionThinkingBlock(TypedDict, total=False): @@ -1382,10 +1382,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): model_config = {"extra": "allow"} +class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject): + ttl_30m_tokens: int + ttl_12h_tokens: int + + model_config = ConfigDict(extra="allow", frozen=True) + + +def _is_none(value: object) -> bool: + return value is None + + class InputTokensDetails(BaseLiteLLMOpenAIResponseObject): audio_tokens: int | None = None cached_tokens: int = 0 cached_tokens_details: CachedTokensDetails | None = None + cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none) + cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none) image_tokens: int | None = None text_tokens: int | None = None video_tokens: int | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..ed66cbd8cf0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -322,6 +322,12 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_272k_tokens_flex: float | None cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_creation_input_token_cost_above_1hr: float | None + cache_creation_input_token_cost_above_12hr: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_flex: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_priority: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None] + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None] cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing @@ -1804,7 +1810,9 @@ class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for ol class CacheCreationTokenDetails(LiteLLMBaseModel): ephemeral_5m_input_tokens: int | None = None + ephemeral_30m_input_tokens: int | None = None ephemeral_1h_input_tokens: int | None = None + ephemeral_12h_input_tokens: int | None = None class PromptTokensDetailsWrapper( @@ -3954,6 +3962,12 @@ class MirroredPricingParams(LiteLLMBaseModel): explicit_cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None + cache_creation_input_token_cost_above_12hr: float | None = None + cache_creation_input_token_cost_above_12hr_flex: float | None = None + cache_creation_input_token_cost_above_12hr_priority: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None tiered_pricing: list[dict[str, Any]] | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 809a94cd93e..58eaa43e687 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6817,6 +6817,24 @@ def get_model_info_helper( cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), + cache_creation_input_token_cost_above_12hr=_model_info.get( + "cache_creation_input_token_cost_above_12hr", None + ), + cache_creation_input_token_cost_above_12hr_flex=_model_info.get( + "cache_creation_input_token_cost_above_12hr_flex", None + ), + cache_creation_input_token_cost_above_12hr_priority=_model_info.get( + "cache_creation_input_token_cost_above_12hr_priority", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None + ), + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get( + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None + ), off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 14fe355a805..8c0c6dffdc7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -36795,6 +36795,12 @@ }, "gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_12hr": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-05, + "cache_creation_input_token_cost_above_12hr_priority": 5e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36880,6 +36886,12 @@ }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -36957,6 +36969,12 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_12hr": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, + "cache_creation_input_token_cost_above_12hr_priority": 5e-07, "cache_creation_input_token_cost_decisions": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07, @@ -37043,6 +37061,12 @@ }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37117,6 +37141,12 @@ }, "gpt-5.6-sol": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_12hr": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05, + "cache_creation_input_token_cost_above_12hr_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 2e-05, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37193,6 +37223,12 @@ }, "gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -37268,6 +37304,12 @@ }, "gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_12hr": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_priority": 1e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, @@ -74933,6 +74975,12 @@ }, "gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_12hr": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05, + "cache_creation_input_token_cost_above_12hr_flex": 2.5e-06, + "cache_creation_input_token_cost_above_12hr_priority": 1e-05, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index acba66223e5..53fd5c1cec2 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -6985,6 +6985,106 @@ GPT_5_6_RESPONSES_CACHE_READ: Final = CostTrackingTestCase( ), ) +GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase( + name="gpt-5.6-responses_cache_write_12h", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-5.6", + endpoint="/v1/responses", + request={ + "model": "$MODEL", + "input": "summarize this text", + "prompt_cache_options": {"mode": "implicit", "ttl": "12h"}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted response", "annotations": []}], + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 5000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000}, + }, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + ), + expected=ExactExpected( + spend=0.0344865, + input_cost=0.0291865, + output_cost=0.00532, + prompt_tokens=12928, + completion_tokens=380, + cache_creation_cost=0.0153125, + ), +) + +GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase( + name="gpt-6-luna-responses_cache_write_12h_no_12h_rate", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-6-luna", + endpoint="/v1/responses", + request={ + "model": "$MODEL", + "input": "summarize this text", + "prompt_cache_options": {"mode": "implicit", "ttl": "12h"}, + }, + response=JsonResponse( + content_type="application/json", + body={ + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6-luna", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted response", "annotations": []}], + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 0, + "cache_write_tokens": 5000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000}, + }, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + ), + expected=ExactExpected( + spend=0.0016078, + input_cost=0.0014178, + output_cost=0.00019, + prompt_tokens=12928, + completion_tokens=380, + cache_creation_cost=0.000625, + ), +) + GPT_5_6_RESPONSES_REASONING: Final = CostTrackingTestCase( name="gpt-5.6-responses_reasoning", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -11913,6 +12013,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( OMNI_MODERATIONS_NEXT_SINGLE, OMNI_MODERATIONS_NEXT_LIST, GPT_5_6_RESPONSES_CACHE_READ, + GPT_5_6_RESPONSES_CACHE_WRITE_12H, + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, GPT_5_6_RESPONSES_REASONING, GPT_5_6_RESPONSES_STREAM, GPT_5_6_RESPONSES_STREAM_CACHE_READ, diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 012d847e676..cd979528fe6 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -174,6 +174,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { }, }, "gpt-5.6": { + "cache_creation_input_token_cost": 2.1875e-06, + "cache_creation_input_token_cost_above_12hr": 4.375e-06, "cache_read_input_token_cost": 1.75e-07, "input_cost_per_audio_token": 4e-05, "input_cost_per_token": 1.75e-06, diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index e98f6a44222..e978a9a316a 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -85,6 +85,12 @@ class CostMapEntry(BaseModel): cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None + cache_creation_input_token_cost_above_12hr: float | None = None + cache_creation_input_token_cost_above_12hr_flex: float | None = None + cache_creation_input_token_cost_above_12hr_priority: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index c99147d2e63..1729db94ed8 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -65,8 +65,8 @@ def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate prompt_cost, _ = generic_cost_per_token( "policy-fixture", usage, "openai", service_tier=service_tier, model_info=info ) - assert billed[4] == pytest.approx(read_rate if read_rate is not None else billed[0]) - assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4]) + assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate) + assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate) def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: @@ -248,7 +248,7 @@ def test_missing_cache_read_uses_off_peak_input_rate(): } when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) - assert billed[0] == billed[4] == 5e-6 + assert billed.input_rate == billed.cache_read_rate == 5e-6 def test_reasoning_tokens_no_price_set(_local_model_cost_map): @@ -324,8 +324,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) - - def test_image_tokens_with_custom_pricing(): """Test that image_tokens in completion are properly costed with output_cost_per_image_token.""" from unittest.mock import patch @@ -578,7 +576,7 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): } usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010) - prompt_base_cost = _get_token_base_cost(model_info, usage)[0] + prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate assert prompt_base_cost == 9e-6 @@ -779,12 +777,12 @@ def test_get_token_base_cost_weekend_only_off_peak_rate(): saturday_peak_hours = _get_token_base_cost( model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc) ) - assert saturday_peak_hours[:2] == (5e-7, 1e-6) + assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6) monday_same_hours = _get_token_base_cost( model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc) ) - assert monday_same_hours[:2] == (1e-6, 2e-6) + assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6) def test_get_token_base_cost_applies_off_peak_pricing(): @@ -810,14 +808,14 @@ def test_get_token_base_cost_applies_off_peak_pricing(): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak[0] == 5e-7 - assert off_peak[1] == 1e-6 - assert off_peak[4] == 5e-8 + assert off_peak.input_rate == 5e-7 + assert off_peak.output_rate == 1e-6 + assert off_peak.cache_read_rate == 5e-8 peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak[0] == 1e-6 - assert peak[1] == 2e-6 - assert peak[4] == 1e-7 + assert peak.input_rate == 1e-6 + assert peak.output_rate == 2e-6 + assert peak.cache_read_rate == 1e-7 def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): @@ -843,8 +841,8 @@ def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): }, ) result = _get_token_base_cost(model_info, usage, current_time=when) - assert result[0] == 1e-6 - assert result[1] == 2e-6 + assert result.input_rate == 1e-6 + assert result.output_rate == 2e-6 def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): @@ -864,8 +862,8 @@ def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert result[0] == 5e-7 - assert result[1] == 2e-6 + assert result.input_rate == 5e-7 + assert result.output_rate == 2e-6 def test_get_token_base_cost_off_peak_wins_over_threshold(): @@ -891,12 +889,12 @@ def test_get_token_base_cost_off_peak_wins_over_threshold(): usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000) off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak[0] == 5e-7 - assert off_peak[1] == 1e-6 + assert off_peak.input_rate == 5e-7 + assert off_peak.output_rate == 1e-6 peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak[0] == 3e-6 - assert peak[1] == 4e-6 + assert peak.input_rate == 3e-6 + assert peak.output_rate == 4e-6 def test_get_model_info_propagates_off_peak_fields(): @@ -948,10 +946,10 @@ def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert inside[:2] == (5e-7, 1e-6) + assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6) outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside[:2] == (3e-6, 6e-6) + assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6) def _register_off_peak_reasoning_model( @@ -1153,17 +1151,17 @@ def test_get_token_base_cost_off_peak_cache_creation_rate(): inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) inside = _get_token_base_cost(model_info, usage, current_time=inside_window) - assert inside[2] == 5e-7 - assert inside[3] == 2e-6 + assert inside.cache_creation_rate == 5e-7 + assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6 outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside[2] == 1.25e-6 + assert outside.cache_creation_rate == 1.25e-6 without_key = cast( ModelInfo, {**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}}, ) - assert _get_token_base_cost(without_key, usage, current_time=inside_window)[2] == 1.25e-6 + assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6 def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates(): @@ -1583,19 +1581,14 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback( ), ) costs: Final = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openrouter") - breakdown: Final = get_token_type_cost_breakdown( - model=model, usage=usage, custom_llm_provider="openrouter" - ) + breakdown: Final = get_token_type_cost_breakdown(model=model, usage=usage, custom_llm_provider="openrouter") ordinary_rate: Final = cache_rates.get("cache_creation_input_token_cost") - expected_write_cost: Final = ( - (20 - one_hour_tokens) * (ordinary_rate if ordinary_rate is not None else 1.0) - + one_hour_tokens * expected_write_rate - ) + expected_write_cost: Final = (20 - one_hour_tokens) * ( + ordinary_rate if ordinary_rate is not None else 1.0 + ) + one_hour_tokens * expected_write_rate assert costs == pytest.approx((40 * 1.0 + 40 * 0.1 + expected_write_cost, 10 * 2.0)) assert breakdown.cache_creation_cost == pytest.approx(expected_write_cost) - assert breakdown.rates is not None - assert breakdown.rates.cache_creation_input_token_cost_above_1hr == expected_write_rate finally: litellm.model_cost.pop(model, None) @@ -2020,8 +2013,8 @@ def test_calculate_cache_writing_cost(): result = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, cache_creation_cost=cache_creation_cost, + cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) # Expected calculation: @@ -2041,8 +2034,8 @@ def test_calculate_cache_writing_cost(): result_no_details = calculate_cache_writing_cost( cache_creation_tokens=cache_creation_tokens_no_details, cache_creation_token_details=cache_creation_token_details_none, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, cache_creation_cost=cache_creation_cost_fallback, + cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) # Expected calculation when no details: 1000 * 5e-06 = 0.005 @@ -2059,15 +2052,13 @@ def test_calculate_cache_writing_cost(): result_partial = calculate_cache_writing_cost( cache_creation_tokens=500, cache_creation_token_details=cache_creation_token_details_partial, - cache_creation_cost_above_1hr=6e-06, cache_creation_cost=3e-06, + cache_creation_cost_by_ttl={"1hr": 6e-06, "12hr": 6e-06}, ) - # Expected calculation: 0 (for None 5m tokens) + (100 * 6e-06) = 0.0006 - expected_cost_partial = (0.0) + (100 * 6e-06) + expected_cost_partial = (400 * 3e-06) + (100 * 6e-06) assert round(result_partial, 6) == round(expected_cost_partial, 6) - assert round(result_partial, 6) == 0.0006 # Test case 4: Zero costs result_zero = calculate_cache_writing_cost( @@ -2075,13 +2066,119 @@ def test_calculate_cache_writing_cost(): cache_creation_token_details=CacheCreationTokenDetails( ephemeral_5m_input_tokens=50, ephemeral_1h_input_tokens=950 ), - cache_creation_cost_above_1hr=0.0, cache_creation_cost=0.0, + cache_creation_cost_by_ttl={"1hr": 0.0, "12hr": 0.0}, ) assert result_zero == 0.0 +@pytest.mark.parametrize("field", CacheCreationTokenDetails.model_fields) +def test_calculate_cache_writing_cost_bills_every_reported_lifetime(field: str) -> None: + cost = calculate_cache_writing_cost( + cache_creation_tokens=10, + cache_creation_token_details=CacheCreationTokenDetails(**{field: 10}), + cache_creation_cost=2.0, + cache_creation_cost_by_ttl={"1hr": 2.0, "12hr": 2.0}, + ) + assert cost == 20.0 + + +def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the_base_rate() -> None: + cost = calculate_cache_writing_cost( + cache_creation_tokens=10, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=2, ephemeral_12h_input_tokens=3 + ), + cache_creation_cost=1.0, + cache_creation_cost_by_ttl={"1hr": 4.0, "12hr": 4.0}, + ) + assert cost == 2 * 1.0 + 3 * 4.0 + 5 * 1.0 + + +@pytest.mark.parametrize( + ("prompt_tokens", "service_tier", "write_rate", "twelve_hour_rate"), + [ + (10_000, "default", 2e-6, 4e-6), + (10_000, "flex", 1e-6, 2e-6), + (10_000, "priority", 3e-6, 6e-6), + (300_000, "default", 4e-6, 8e-6), + (300_000, "flex", 2e-6, 4e-6), + (300_000, "priority", 6e-6, 12e-6), + ], +) +def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold( + prompt_tokens: int, service_tier: str, write_rate: float, twelve_hour_rate: float +) -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 2e-6, + "cache_creation_input_token_cost_flex": 1e-6, + "cache_creation_input_token_cost_priority": 3e-6, + "cache_creation_input_token_cost_above_272k_tokens": 4e-6, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2e-6, + "cache_creation_input_token_cost_above_272k_tokens_priority": 6e-6, + "cache_creation_input_token_cost_above_12hr": 4e-6, + "cache_creation_input_token_cost_above_12hr_flex": 2e-6, + "cache_creation_input_token_cost_above_12hr_priority": 6e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 8e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 4e-6, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 12e-6, + } + usage: Final = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=0, + total_tokens=prompt_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=5_000, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_30m_input_tokens=3_000, ephemeral_12h_input_tokens=2_000 + ), + ), + ) + + result: Final = generic_cost_per_token( + model="test-12h-cache-write", + usage=usage, + custom_llm_provider="openai", + service_tier=None if service_tier == "default" else service_tier, + model_info=model_info, + ) + + assert result[0] == pytest.approx( + (usage.prompt_tokens - usage.prompt_tokens_details.cache_creation_tokens) * model_info["input_cost_per_token"] + + 3_000 * write_rate + + 2_000 * twelve_hour_rate + ) + + +def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 3e-6, + } + usage: Final = Usage( + prompt_tokens=100, + completion_tokens=0, + total_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=100, + cache_creation_token_details=CacheCreationTokenDetails(ephemeral_12h_input_tokens=100), + ), + ) + + result: Final = generic_cost_per_token( + model="test-12h-cache-write-fallback", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + ) + + assert result[0] == pytest.approx(100 * model_info["cache_creation_input_token_cost"]) + + def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): """ Regression test: when cache_creation_tokens is 0 but cache_creation_token_details @@ -2119,7 +2216,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_base_cost=0.0, cache_read_cost=0.0, cache_creation_cost=cache_creation_cost, - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, ) # Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575 @@ -2330,10 +2427,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) - - - - def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): usage = Usage( prompt_tokens=0, @@ -2512,8 +2605,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located - - def test_vertex_uplift_invalid_multiplier_defaults_to_one(): """A malformed multiplier in the cost map degrades to base pricing, never raises.""" from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -2569,10 +2660,10 @@ def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: model_info=info, usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004), ) - assert rates[0] == 0.001 - assert rates[1] == (0.005 if field == "output_cost_per_token" else 0.002) - assert rates[2] == (0.005 if field == "cache_creation_input_token_cost" else 0.004) - assert rates[4] == (0.005 if field == "cache_read_input_token_cost" else 0.003) + assert rates.input_rate == 0.001 + assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002) + assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004) + assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003) @pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001)) @@ -2599,10 +2690,12 @@ def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, h model_info=info, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3), ) - assert rates[0] == (0.005 if prompt_tokens > 200_000 else 0.001) - assert rates[1] == (higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002) - assert rates[2] == (0.007 if prompt_tokens > 200_000 else 0.004) - assert rates[4] == (higher_rate if prompt_tokens > 512_000 else 0.003) + assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001) + assert rates.output_rate == ( + higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002 + ) + assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004) + assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003) def test_threshold_keys_exclude_service_tier_variants(): @@ -2626,8 +2719,7 @@ def test_threshold_keys_exclude_service_tier_variants(): }, ) usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000) - prompt_base, *_ = _get_token_base_cost(model_info=model_info, usage=usage) - assert prompt_base == 3e-6 + assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6 @pytest.mark.parametrize( @@ -2897,7 +2989,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp cache_read_input_token_cost=6e-7, cache_read_input_audio_token_cost=6e-7, cache_creation_input_token_cost=7.5e-6, - cache_creation_input_token_cost_above_1hr=7.5e-6, + cache_creation_input_token_cost_by_ttl={"1hr": 7.5e-6, "12hr": 7.5e-6}, output_cost_per_reasoning_token=3e-5, ) assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost) @@ -3713,6 +3805,23 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a 3e-7, id="no-1h-price-uses-the-off-peak-write-price", ), + pytest.param( + { + "tiered_pricing": [ + { + "range": [0, 128000], + "input_cost_per_token": 2e-7, + "output_cost_per_token": 1.25e-6, + "cache_creation_input_token_cost": 2.5e-7, + } + ], + "off_peak_pricing": {"hours_utc": "00:00-23:59", "cache_creation_input_token_cost": 3e-7}, + }, + datetime(2026, 9, 14, 12, tzinfo=timezone.utc), + 3e-7, + 3e-7, + id="tiered-no-1h-price-uses-the-off-peak-write-price", + ), ), ) def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path( @@ -3724,10 +3833,17 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates} usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11) - _, _, creation, creation_1h, _ = _get_token_base_cost(model_info, usage, current_time=current_time) + rates = _get_token_base_cost(model_info, usage, current_time=current_time) - assert creation == pytest.approx(expected_creation) - assert creation_1h == pytest.approx(expected_creation_1h) + assert rates.cache_creation_rate == pytest.approx(expected_creation) + assert calculate_cache_writing_cost( + cache_creation_tokens=3, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2 + ), + cache_creation_cost=rates.cache_creation_rate, + cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl, + ) == pytest.approx(3 * expected_creation_1h) def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse: @@ -3787,8 +3903,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima assert cost == pytest.approx(0.07) - - @pytest.mark.parametrize( ("custom_llm_provider", "model"), [ @@ -4206,17 +4320,28 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa "cache_creation_input_token_cost": rate, } usage: Final = Usage( - prompt_tokens=20, completion_tokens=3, total_tokens=23, - prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5, + prompt_tokens=20, + completion_tokens=3, + total_tokens=23, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), + cache_creation_input_tokens=5, ) prompt_cost, output_cost = generic_cost_per_token( - model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info, + model="supplied-pricing-test", + usage=usage, + custom_llm_provider="bedrock", + model_info=model_info, ) breakdown: Final = get_token_type_cost_breakdown( - model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info, + model="supplied-pricing-test", + usage=usage, + custom_llm_provider="bedrock", + model_info=model_info, ) assert breakdown.cache_read_cost == pytest.approx(5 * rate) assert breakdown.cache_creation_cost == pytest.approx(5 * rate) assert breakdown.rates is not None - assert prompt_cost == pytest.approx(10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost) + assert prompt_cost == pytest.approx( + 10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost + ) assert output_cost == pytest.approx(3 * breakdown.rates.output_cost_per_token) diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 30e13442c92..023d0ea26fb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -3398,7 +3398,9 @@ class TestUsageTransformation: assert response_usage.input_tokens_details is not None assert response_usage.input_tokens_details.cached_tokens == 5 assert response_usage.input_tokens_details.text_tokens == 8 - assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump() + assert {"cache_write_tokens", "cache_write_tokens_details"}.isdisjoint( + response_usage.input_tokens_details.model_dump() + ) def test_transform_usage_with_cached_tokens_gemini(self): """Test that cached_tokens from Gemini are properly transformed to input_tokens_details""" diff --git a/tests/unit/responses/test_responses_utils.py b/tests/unit/responses/test_responses_utils.py index 6ee2c495705..97a39f64001 100644 --- a/tests/unit/responses/test_responses_utils.py +++ b/tests/unit/responses/test_responses_utils.py @@ -1,5 +1,6 @@ from importlib import import_module import base64 +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -7,10 +8,11 @@ import pytest import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse -from litellm.types.utils import Usage +from litellm.types.utils import ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map @@ -413,6 +415,43 @@ class TestResponseAPILoggingUtils: assert result.prompt_tokens_details.cache_creation_tokens == 10059 assert result.prompt_tokens_details.cached_tokens == 0 + @pytest.mark.parametrize("as_model", [False, True]) + def test_transform_response_api_usage_prices_12h_cache_writes(self, as_model: bool) -> None: + usage_data: Final = { + "input_tokens": 5_000, + "output_tokens": 0, + "total_tokens": 5_000, + "input_tokens_details": { + "cache_write_tokens": 5_000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3_000, "ttl_12h_tokens": 2_000}, + }, + } + usage_input: Final = ResponseAPIUsage(**usage_data) if as_model else usage_data + usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_input) + model_info: Final[ModelInfo] = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_creation_input_token_cost": 3e-6, + "cache_creation_input_token_cost_above_12hr": 5e-6, + } + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cache_write_tokens == 5_000 + assert usage.prompt_tokens_details.cache_creation_token_details is not None + assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000 + assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000 + cost: Final = generic_cost_per_token( + model="test-12h-cache-write", + usage=usage, + custom_llm_provider="openai", + model_info=model_info, + )[0] + + assert cost == pytest.approx( + 3_000 * model_info["cache_creation_input_token_cost"] + + 2_000 * model_info["cache_creation_input_token_cost_above_12hr"] + ) + def test_transform_response_api_usage_mixed_details(self): """Test transformation handles mixed token details (cached + image + audio).""" # Setup - hypothetical usage with mixed token types diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ad069aa987b..fcd6b8aa402 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -939,6 +939,18 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { + "type": "number" + }, + "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1d36f5fd9a2..da9e77e9b58 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28439,6 +28439,15 @@ export interface components { */ uncached_input_tokens: number; }; + /** CacheWriteTokensDetails */ + CacheWriteTokensDetails: { + /** Ttl 12H Tokens */ + ttl_12h_tokens: number; + /** Ttl 30M Tokens */ + ttl_30m_tokens: number; + } & { + [key: string]: unknown; + }; /** CachedTokensDetails */ CachedTokensDetails: { /** Audio Tokens */ @@ -33589,6 +33598,9 @@ export interface components { InputTokensDetails: { /** Audio Tokens */ audio_tokens?: number | null; + /** Cache Write Tokens */ + cache_write_tokens?: number | null; + cache_write_tokens_details?: components["schemas"]["CacheWriteTokensDetails"] | null; /** * Cached Tokens * @default 0 @@ -35207,6 +35219,18 @@ export interface components { cache_creation_input_token_cost_above_100k_tokens?: number | null; /** Cache Creation Input Token Cost Above 100K Tokens Batches */ cache_creation_input_token_cost_above_100k_tokens_batches?: number | null; + /** Cache Creation Input Token Cost Above 12Hr */ + cache_creation_input_token_cost_above_12hr?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Flex */ + cache_creation_input_token_cost_above_12hr_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Priority */ + cache_creation_input_token_cost_above_12hr_priority?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ @@ -41506,9 +41530,9 @@ export interface components { mode?: "implicit" | "explicit"; /** * Ttl - * @constant + * @enum {string} */ - ttl?: "30m"; + ttl?: "30m" | "12h"; }; /** PromptCachingRequest */ PromptCachingRequest: { @@ -50199,6 +50223,18 @@ export interface components { blocked?: boolean | null; /** Cache Creation Input Token Cost */ cache_creation_input_token_cost?: number | null; + /** Cache Creation Input Token Cost Above 12Hr */ + cache_creation_input_token_cost_above_12hr?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Flex */ + cache_creation_input_token_cost_above_12hr_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Priority */ + cache_creation_input_token_cost_above_12hr_priority?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ @@ -50492,6 +50528,18 @@ export interface components { cache_creation_input_token_cost_above_100k_tokens?: number | null; /** Cache Creation Input Token Cost Above 100K Tokens Batches */ cache_creation_input_token_cost_above_100k_tokens_batches?: number | null; + /** Cache Creation Input Token Cost Above 12Hr */ + cache_creation_input_token_cost_above_12hr?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */ + cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Flex */ + cache_creation_input_token_cost_above_12hr_flex?: number | null; + /** Cache Creation Input Token Cost Above 12Hr Priority */ + cache_creation_input_token_cost_above_12hr_priority?: number | null; /** Cache Creation Input Token Cost Above 1Hr */ cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */ From dc0237ccd722c7e4d010b9a29ff4ba22af217f5b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 24 Oct 2026 09:12:31 -0700 Subject: [PATCH 5/5] fix(cost): bill gemini audio input and cached audio at flex, priority and batch rates (#48086) * fix(gemini): price tiered audio token rates * fix: correct Gemini audio pricing catalog * fix(gemini): preserve cached modality pricing --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 6 +- .../llm_cost_calc/zero_cost_diagnostic.py | 29 +++- .../vertex_and_google_ai_studio_gemini.py | 31 ++-- ...odel_prices_and_context_window_backup.json | 16 ++ litellm/types/utils.py | 10 ++ litellm/utils.py | 9 ++ model_prices_and_context_window.json | 16 ++ .../llm_cost_calc/test_utils.py | 62 ++++++++ .../test_zero_cost_diagnostic.py | 31 ++++ ...test_vertex_and_google_ai_studio_gemini.py | 147 +++++++++++++++++- tests/unit/test_cost_calculator.py | 77 +++++---- tests/unit/test_utils.py | 4 + tests/unit/test_utils_get_model_info.py | 24 +++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++ 14 files changed, 429 insertions(+), 53 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 99944662bf8..98f6f1409f2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2879,6 +2879,7 @@ def _batch_rate( model_info: ModelInfo, key: Literal[ "input_cost_per_audio_token_batches", + "cache_read_input_audio_token_cost_batches", "input_cost_per_image_token_batches", "input_cost_per_video_token_batches", ], @@ -3012,6 +3013,7 @@ def _batch_prompt_cost( ) -> float: details: Final = parse_prompt_tokens_details(usage) cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 audio_tokens, image_tokens, video_tokens = ( details["audio_tokens"], @@ -3026,7 +3028,9 @@ def _batch_prompt_cost( + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) - + cached_tokens * (cache_read_rate or 0.0) + + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) + + cached_audio_tokens + * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) + written_tokens * (cache_creation_rate or 0.0) ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py index 9688b511ea8..3ed598b5e97 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py +++ b/litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py @@ -5,6 +5,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.types.utils import ( CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, @@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total" _TEXT_INPUT_RATE: Final = "input_cost_per_token" _AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token" +_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost" +_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost" _TEXT_OUTPUT_RATE: Final = "output_cost_per_token" _AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token" _RATE_KEY_MARKERS: Final = ("cost", "pricing") @@ -33,19 +36,38 @@ def _tokens(value: object) -> int: def used_pricing_keys(usage: Usage) -> tuple[str, ...]: - prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details) + prompt_details: Final = parse_prompt_tokens_details(usage) + prompt_audio: Final = prompt_details["audio_tokens"] completion_audio: Final = _audio_tokens(usage.completion_tokens_details) - prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio + prompt_text: Final = max( + _tokens(usage.prompt_tokens) + - prompt_details["cache_hit_tokens"] + - prompt_audio + - prompt_details["image_tokens"] + - prompt_details["video_tokens"], + 0, + ) + cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0) completion_text: Final = _tokens(usage.completion_tokens) - completion_audio components: Final = ( (_TEXT_INPUT_RATE, prompt_text), (_AUDIO_INPUT_RATE, prompt_audio), + (_CACHE_READ_INPUT_RATE, cached_text), + (_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]), (_TEXT_OUTPUT_RATE, completion_text), (_AUDIO_OUTPUT_RATE, completion_audio), ) return tuple(key for key, count in components if count > 0) +def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str: + if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None: + return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE + if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None: + return _TEXT_INPUT_RATE + return key + + def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None: try: return _NESTED_PRICING.validate_python(value) @@ -89,7 +111,8 @@ def diagnose_zero_cost( pricing_entry: Mapping[str, object], calculation_failed: bool, ) -> StandardLoggingZeroCostDiagnostic | None: - used_keys: Final = used_pricing_keys(usage) + effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage)) + used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index]) if not used_keys: return None missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ce632bc0eef..d788ccb0f1d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import ( get_httpx_client, ) from litellm.types.llms.anthropic import AnthropicThinkingParam +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.gemini import BidiGenerateContentServerMessage from litellm.types.llms.openai import ( AllMessageValues, @@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif modality == "DOCUMENT": cached_text_tokens = (cached_text_tokens or 0) + token_count - ## Calculate non-cached tokens by subtracting cached from total (per modality) - ## This is necessary because promptTokensDetails includes both cached and non-cached tokens - ## See: https://github.com/BerriAI/litellm/issues/18750 - if cached_text_tokens is not None and prompt_text_tokens is not None: - # Explicit caching: subtract cached tokens per modality from cacheTokensDetails - prompt_text_tokens = prompt_text_tokens - cached_text_tokens - elif ( + if ( cached_tokens is not None and prompt_text_tokens is not None and cached_text_tokens is None and "cacheTokensDetails" not in usage_metadata ): - # Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails) - # Subtract from text tokens since implicit caching is primarily for text content - # See: https://github.com/BerriAI/litellm/issues/16341 prompt_text_tokens = prompt_text_tokens - cached_tokens - if cached_audio_tokens is not None and prompt_audio_tokens is not None: - prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens - if cached_image_tokens is not None and prompt_image_tokens is not None: - prompt_image_tokens = prompt_image_tokens - cached_image_tokens if cached_video_tokens is not None and prompt_video_tokens is not None: prompt_video_tokens = prompt_video_tokens - cached_video_tokens @@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): billable_text_tokens: Final = ( prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens ) + cached_tokens_details_fields: Final = MappingProxyType( + { + key: token_count + for key, token_count in ( + ("text_tokens", cached_text_tokens), + ("audio_tokens", cached_audio_tokens), + ("image_tokens", cached_image_tokens), + ) + if token_count is not None + } + ) + cached_tokens_details: Final = ( + CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None + ) prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=cached_tokens, + cached_tokens_details=cached_tokens_details, audio_tokens=prompt_audio_tokens, text_tokens=billable_text_tokens, image_tokens=prompt_image_tokens, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8c0c6dffdc7..15dfd196c6f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31137,11 +31137,15 @@ }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31466,11 +31470,15 @@ }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, + "cache_read_input_audio_token_cost_batches": 3e-08, + "cache_read_input_audio_token_cost_flex": 3e-08, + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_flex": 1.5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31760,12 +31768,16 @@ }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -31886,10 +31898,14 @@ }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ed66cbd8cf0..270d9b43dea 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -333,6 +333,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None cache_read_input_audio_token_cost: ReadOnly[float | None] + cache_read_input_audio_token_cost_flex: ReadOnly[float | None] + cache_read_input_audio_token_cost_priority: ReadOnly[float | None] cache_read_input_image_token_cost: ReadOnly[float | None] explicit_cache_read_input_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing @@ -348,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None] cache_read_input_token_cost_above_512k_tokens: float | None cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_audio_token_cost_batches: ReadOnly[float | None] cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] @@ -361,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None + input_cost_per_audio_token_flex: ReadOnly[float | None] + input_cost_per_audio_token_priority: ReadOnly[float | None] input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models input_cost_per_token_above_100k_tokens: ReadOnly[float | None] @@ -4020,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_100k_tokens_batches: float | None = None cache_read_input_token_cost_above_200k_tokens_batches: float | None = None cache_read_input_token_cost_above_272k_tokens_batches: float | None = None + cache_read_input_audio_token_cost_batches: float | None = None cache_creation_input_token_cost_batches: float | None = None cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost: float | None = None + cache_read_input_audio_token_cost_flex: float | None = None + cache_read_input_audio_token_cost_priority: float | None = None cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None input_cost_per_audio_token: float | None = None + input_cost_per_audio_token_flex: float | None = None + input_cost_per_audio_token_priority: float | None = None input_cost_per_token_cache_hit: float | None = None input_cost_per_token_above_128k_tokens: float | None = None input_cost_per_token_above_100k_tokens: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 58eaa43e687..bb94ca6d40a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6773,6 +6773,13 @@ def get_model_info_helper( ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), + cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None), + cache_read_input_audio_token_cost_priority=_model_info.get( + "cache_read_input_audio_token_cost_priority", None + ), + cache_read_input_audio_token_cost_batches=_model_info.get( + "cache_read_input_audio_token_cost_batches", None + ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( @@ -6858,6 +6865,8 @@ def get_model_info_helper( input_cost_per_second=_model_info.get("input_cost_per_second", None), input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), + input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None), + input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None), input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8c0c6dffdc7..15dfd196c6f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31137,11 +31137,15 @@ }, "gemini/gemini-2.5-flash": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31466,11 +31470,15 @@ }, "gemini/gemini-2.5-flash-lite": { "cache_read_input_audio_token_cost": 3e-08, + "cache_read_input_audio_token_cost_batches": 3e-08, + "cache_read_input_audio_token_cost_flex": 3e-08, + "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, + "input_cost_per_audio_token_flex": 1.5e-07, "input_cost_per_token": 1e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, @@ -31760,12 +31768,16 @@ }, "gemini/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, + "cache_read_input_audio_token_cost_batches": 2.5e-08, + "cache_read_input_audio_token_cost_flex": 2.5e-08, + "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-05-07", "input_cost_per_audio_token": 5e-07, + "input_cost_per_audio_token_flex": 2.5e-07, "input_cost_per_token": 2.5e-07, "input_cost_per_token_batches": 1.25e-07, "input_cost_per_token_flex": 1.25e-07, @@ -31886,10 +31898,14 @@ }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, + "cache_read_input_audio_token_cost_batches": 1e-07, + "cache_read_input_audio_token_cost_flex": 1e-07, + "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, + "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", "max_input_tokens": 1048576, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py index 142d49bfff1..b6c18e0da86 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -11,6 +11,7 @@ import respx import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage TIER_MODEL: Final = "tier-priced-test-model" @@ -31,6 +32,20 @@ PROMPT_TOKENS: Final = 1000 CACHED_TOKENS: Final = 200 COMPLETION_TOKENS: Final = 500 TIER_API_BASE: Final = "https://tier-pricing.invalid/v1" +AUDIO_TIER_MODEL: Final = "tier-priced-gemini-audio-test-model" +AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( + { + "input_cost_per_token": 4e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_audio_token": 3e-06, + "cache_read_input_audio_token_cost": 2e-06, + "input_cost_per_audio_token_flex": 1e-06, + "input_cost_per_audio_token_priority": 2e-06, + "cache_read_input_audio_token_cost_flex": 4e-07, + "cache_read_input_audio_token_cost_priority": 8e-07, + } +) def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float: @@ -78,6 +93,53 @@ def test_completion_cost_bills_the_price_columns_of_the_service_tier( assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix)) +@pytest.mark.parametrize( + ("service_tier", "audio_rate_key", "cache_audio_rate_key"), + ( + pytest.param("flex", "input_cost_per_audio_token_flex", "cache_read_input_audio_token_cost_flex"), + pytest.param("priority", "input_cost_per_audio_token_priority", "cache_read_input_audio_token_cost_priority"), + ), +) +def test_gemini_audio_and_cached_audio_use_service_tier_rates( + local_model_cost_map: None, service_tier: str, audio_rate_key: str, cache_audio_rate_key: str +) -> None: + litellm.register_model( + { + AUDIO_TIER_MODEL: { + "litellm_provider": "gemini", + "mode": "chat", + **AUDIO_TIER_ROW, + } + } + ) + usage: Final = Usage( + prompt_tokens=40, + completion_tokens=0, + total_tokens=40, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=20, + audio_tokens=20, + cached_tokens=10, + cached_tokens_details=CachedTokensDetails(audio_tokens=10), + ), + ) + response: Final = ModelResponse(model=AUDIO_TIER_MODEL, usage=usage) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=AUDIO_TIER_MODEL, + custom_llm_provider="gemini", + service_tier=service_tier, + ) + + expected: Final = ( + 20 * AUDIO_TIER_ROW["input_cost_per_token"] + + 10 * AUDIO_TIER_ROW[audio_rate_key] + + 10 * AUDIO_TIER_ROW[cache_audio_rate_key] + ) + assert cost == pytest.approx(expected) + + LONG_CONTEXT_TIER_MODEL: Final = "long-context-tier-priced-test-model" LONG_CONTEXT_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( { diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py index 921dde8bf7b..bdc14c4934b 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( used_pricing_keys, zero_cost_warning, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage PER_SECOND_ENTRY: Final = {"cost_per_second": 0.00042} @@ -135,6 +136,36 @@ def test_audio_tokens_need_the_audio_rates() -> None: assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token") +def test_cached_audio_uses_its_cache_rate_in_zero_cost_diagnostic() -> None: + usage: Final = Usage( + prompt_tokens=20, + completion_tokens=0, + total_tokens=20, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, + audio_tokens=20, + cached_tokens=20, + cached_tokens_details=CachedTokensDetails(audio_tokens=20), + ), + ) + pricing_entry: Final = { + "input_cost_per_token": 1e-6, + "input_cost_per_audio_token": 2e-6, + "cache_read_input_audio_token_cost": 0, + } + + assert used_pricing_keys(usage) == ("cache_read_input_audio_token_cost",) + assert ( + diagnose_zero_cost( + usage=usage, + pricing_model="gemini-audio", + pricing_entry=pricing_entry, + calculation_failed=False, + ) + is None + ) + + def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None: diagnostic = diagnose_zero_cost( usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 4c3ee9e7ca5..3bc864cc047 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -16,8 +16,10 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion, embedding, image_generation from litellm._logging import verbose_logger +from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -30,8 +32,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( normalize_gemini_speech_config, reject_unknown_gemini_tts_voice_keys, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata -from litellm.types.utils import ChoiceLogprobs, Usage +from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.integrations.custom_logger import CustomLogger @@ -1515,6 +1518,7 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): v = VertexGeminiConfig() usage_metadata = { "promptTokenCount": 210, + "cachedContentTokenCount": 54, "candidatesTokenCount": 50, "totalTokenCount": 260, "promptTokensDetails": [ @@ -1536,9 +1540,10 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): usage_metadata = UsageMetadata(**usage_metadata) result = v.calculate_usage(completion_response={"usageMetadata": usage_metadata}) - # prompt details are total - cached per modality - assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4 - assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10) + assert result.prompt_tokens_details.text_tokens == 20 + assert result.prompt_tokens_details.image_tokens == 190 + assert result.prompt_tokens_details.cached_tokens == 54 + assert result.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=4, image_tokens=50) # candidates details accumulate duplicate modalities assert result.completion_tokens_details.text_tokens == 20 # 15 + 5 @@ -1546,6 +1551,118 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities(): assert result.completion_tokens == 50 +def _gemini_usage_with_cached_audio(cached_audio_tokens: int) -> Usage: + cache_tokens_details: Final = ( + ( + {"modality": "TEXT", "tokenCount": 5}, + {"modality": "AUDIO", "tokenCount": cached_audio_tokens}, + ) + if cached_audio_tokens > 0 + else ({"modality": "TEXT", "tokenCount": 5},) + ) + completion_response: Final = { + "usageMetadata": { + "promptTokenCount": 40, + "cachedContentTokenCount": 5 + cached_audio_tokens, + "promptTokensDetails": ( + {"modality": "TEXT", "tokenCount": 20}, + {"modality": "AUDIO", "tokenCount": 20}, + ), + "cacheTokensDetails": cache_tokens_details, + "candidatesTokenCount": 7, + "totalTokenCount": 47, + } + } + return VertexGeminiConfig.calculate_usage(completion_response=completion_response) + + +def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_batch_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_batches": 1e-6, + "input_cost_per_audio_token": 6e-6, + "input_cost_per_audio_token_batches": 3e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_token_cost_batches": 4e-7, + "cache_read_input_audio_token_cost": 5e-7, + "cache_read_input_audio_token_cost_batches": 2e-7, + } + usage: Final = _gemini_usage_with_cached_audio(5) + expected_prompt_cost: Final = ( + 15 * model_info["input_cost_per_token_batches"] + + 15 * model_info["input_cost_per_audio_token_batches"] + + 5 * model_info["cache_read_input_token_cost_batches"] + + 5 * model_info["cache_read_input_audio_token_cost_batches"] + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-audio-batch-test-model", + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_audio_token": 6e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_audio_token_cost": 5e-7, + } + usage: Final = _gemini_usage_with_cached_audio(5) + expected_prompt_cost: Final = ( + 15 * model_info["input_cost_per_token"] + + 15 * model_info["input_cost_per_audio_token"] + + 5 * model_info["cache_read_input_token_cost"] + + 5 * model_info["cache_read_input_audio_token_cost"] + ) + + prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_audio_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_audio_token": 6e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_audio_token_cost": 6e-6, + } + cached_usage: Final = _gemini_usage_with_cached_audio(5) + uncached_usage: Final = _gemini_usage_with_cached_audio(0) + cached_details: Final = parse_prompt_tokens_details(cached_usage) + cached_prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=cached_usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + uncached_prompt_cost, _ = generic_cost_per_token( + model="gemini-audio-standard-test-model", + usage=uncached_usage, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert cached_usage.prompt_tokens_details.text_tokens == 20 + assert cached_usage.prompt_tokens_details.audio_tokens == 20 + assert cached_usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( + text_tokens=5, audio_tokens=5 + ) + assert cached_details["text_tokens"] == 15 + assert cached_details["audio_tokens"] == 15 + assert cached_prompt_cost == pytest.approx(uncached_prompt_cost) + + def test_vertex_ai_map_thinking_param_with_budget_tokens_0(): """ If budget_tokens is 0, do not set includeThoughts to True @@ -6150,7 +6267,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt(): def test_vertex_ai_usage_metadata_with_document_tokens_cached(): - """Test that cached DOCUMENT tokens are correctly subtracted from prompt text tokens.""" + """Test cached DOCUMENT token details and unchanged prompt cost.""" v = VertexGeminiConfig() usage_metadata_dict = { @@ -6174,10 +6291,24 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached(): completion_response = {"usageMetadata": usage_metadata_dict} result = v.calculate_usage(completion_response=completion_response) - # DOCUMENT cached tokens map to cached_text_tokens, so: - # text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382 - assert result.prompt_tokens_details.text_tokens == 382, "text_tokens should be (8 + 774) - 400 cached = 382" + assert result.prompt_tokens_details.text_tokens == 782 assert result.prompt_tokens_details.cached_tokens == 400 + assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 400 + + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + } + prompt_cost, _ = generic_cost_per_token( + model="gemini-document-cache-test-model", + usage=result, + custom_llm_provider="gemini", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx( + 382 * model_info["input_cost_per_token"] + 400 * model_info["cache_read_input_token_cost"] + ) def test_async_streaming_uses_custom_client(): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 5d4eb5fae12..550fe5f0d1a 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -16,6 +16,7 @@ import litellm from litellm import TranscriptionResponse, model_cost from litellm.constants import REALTIME_METERED_USAGE_KEY from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, @@ -3337,14 +3338,7 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca def test_gemini_cache_tokens_details_no_negative_values(): - """ - Test for Issue #18750: Negative text_tokens with Gemini caching - - When using Gemini with explicit caching, the response includes cacheTokensDetails - which breaks down cached tokens by modality. This test ensures that: - 1. text_tokens is never negative - 2. We correctly subtract cached tokens per modality (not total) - """ + """Ensure issue #18750 cache details remain non-negative after cost parsing.""" from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -3372,27 +3366,17 @@ def test_gemini_cache_tokens_details_no_negative_values(): usage = VertexGeminiConfig.calculate_usage(completion_response) - # Text tokens should be non-cached text only: 9402 - 9393 = 9 - assert usage.prompt_tokens_details.text_tokens == 9, ( - f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}" - ) + prompt_details: Final = parse_prompt_tokens_details(usage) - # Image tokens should be non-cached image only: 258 - 258 = 0 - assert usage.prompt_tokens_details.image_tokens == 0, ( - f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}" + assert usage.prompt_tokens_details.text_tokens == 9402 + assert usage.prompt_tokens_details.image_tokens == 258 + assert usage.prompt_tokens_details.cached_tokens == 9651 + assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( + text_tokens=9393, image_tokens=258 ) - - # Total cached should match - assert usage.prompt_tokens_details.cached_tokens == 9651, ( - f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}" - ) - - # MOST IMPORTANT: text_tokens should NEVER be negative - assert usage.prompt_tokens_details.text_tokens >= 0, ( - f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750" - ) - - print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative") + assert prompt_details["text_tokens"] == 9 + assert prompt_details["image_tokens"] == 0 + assert prompt_details["text_tokens"] >= 0 def test_gemini_without_cache_tokens_details(): @@ -5990,6 +5974,45 @@ def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_t assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7) +def test_batch_cost_calculator_prices_cached_audio_at_its_batch_rate() -> None: + model_info: Final[ModelInfo] = { + "input_cost_per_token": 2e-6, + "input_cost_per_token_batches": 1e-6, + "input_cost_per_audio_token": 6e-6, + "input_cost_per_audio_token_batches": 3e-6, + "cache_read_input_token_cost": 8e-7, + "cache_read_input_token_cost_batches": 4e-7, + "cache_read_input_audio_token_cost": 5e-7, + "cache_read_input_audio_token_cost_batches": 2e-7, + } + usage: Final = Usage( + prompt_tokens=40, + completion_tokens=0, + total_tokens=40, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=20, + audio_tokens=20, + cached_tokens=10, + cached_tokens_details=CachedTokensDetails(text_tokens=5, audio_tokens=5), + ), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-audio-batch-test-model", + custom_llm_provider="gemini", + model_info=model_info, + ) + + expected: Final = ( + 15 * model_info["input_cost_per_token_batches"] + + 15 * model_info["input_cost_per_audio_token_batches"] + + 5 * model_info["cache_read_input_token_cost_batches"] + + 5 * model_info["cache_read_input_audio_token_cost_batches"] + ) + assert prompt_cost == pytest.approx(expected) + + QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fcd6b8aa402..338faae188f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -994,6 +994,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, + "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_audio_token_cost_flex": {"type": "number"}, + "cache_read_input_audio_token_cost_priority": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, @@ -1001,6 +1004,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, "google_maps_grounding_cost_per_query": {"type": "number"}, "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_audio_token_flex": {"type": "number"}, "input_cost_per_image_token": {"type": "number"}, "input_cost_per_character": {"type": "number"}, "input_cost_per_character_above_128k_tokens": {"type": "number"}, diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index bfaac44a483..4bd16a2997f 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -61,6 +61,30 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch): assert model_info["input_cost_per_token"] == 0.0 +def test_get_model_info_passes_through_gemini_tiered_audio_rates() -> None: + model: Final = "gemini-tiered-audio-rates-test" + prices: Final = { + "input_cost_per_audio_token_flex": 1e-6, + "input_cost_per_audio_token_priority": 2e-6, + "cache_read_input_audio_token_cost_flex": 3e-7, + "cache_read_input_audio_token_cost_priority": 4e-7, + "cache_read_input_audio_token_cost_batches": 5e-7, + } + litellm.register_model( + { + model: { + "litellm_provider": "gemini", + "mode": "chat", + **prices, + } + } + ) + + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + assert {key: model_info[key] for key in prices} == prices + + def test_get_model_info_ollama_chat(): from litellm.llms.ollama.completion.transformation import OllamaConfig diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index da9e77e9b58..b5a86dd2187 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35259,6 +35259,12 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Audio Token Cost Batches */ + cache_read_input_audio_token_cost_batches?: number | null; + /** Cache Read Input Audio Token Cost Flex */ + cache_read_input_audio_token_cost_flex?: number | null; + /** Cache Read Input Audio Token Cost Priority */ + cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -35338,6 +35344,10 @@ export interface components { input_cost_per_audio_token?: number | null; /** Input Cost Per Audio Token Batches */ input_cost_per_audio_token_batches?: number | null; + /** Input Cost Per Audio Token Flex */ + input_cost_per_audio_token_flex?: number | null; + /** Input Cost Per Audio Token Priority */ + input_cost_per_audio_token_priority?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */ @@ -50568,6 +50578,12 @@ export interface components { cache_creation_input_token_cost_ultrafast?: number | null; /** Cache Read Input Audio Token Cost */ cache_read_input_audio_token_cost?: number | null; + /** Cache Read Input Audio Token Cost Batches */ + cache_read_input_audio_token_cost_batches?: number | null; + /** Cache Read Input Audio Token Cost Flex */ + cache_read_input_audio_token_cost_flex?: number | null; + /** Cache Read Input Audio Token Cost Priority */ + cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; /** Cache Read Input Token Cost */ @@ -50647,6 +50663,10 @@ export interface components { input_cost_per_audio_token?: number | null; /** Input Cost Per Audio Token Batches */ input_cost_per_audio_token_batches?: number | null; + /** Input Cost Per Audio Token Flex */ + input_cost_per_audio_token_flex?: number | null; + /** Input Cost Per Audio Token Priority */ + input_cost_per_audio_token_priority?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */