diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb9bfa6ffe4..e2c418708c3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -36699,6 +36699,8 @@ }, "gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 0.000025, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00005, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36784,6 +36786,8 @@ }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 0.000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -36861,6 +36865,8 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 0.00000025, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000005, "cache_creation_input_token_cost_decisions": 1e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07, @@ -36947,6 +36953,8 @@ }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 0.00001, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37021,6 +37029,8 @@ }, "gpt-5.6-sol": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 0.00001, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37097,6 +37107,8 @@ }, "gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 0.000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -37172,6 +37184,8 @@ }, "gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_1hr": 0.0000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.000001, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, @@ -37247,6 +37261,8 @@ }, "gpt-5.6-cyber": { "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_above_1hr": 0.00003125, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000625, "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens": 2.5e-06, @@ -74677,6 +74693,8 @@ }, "gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 0.000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 545b317055a..50867489c52 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -22860,6 +22860,35 @@ "title": "Body_image_edit_api_openai_deployments__model__images_edits_post", "type": "object" }, + "CacheWriteTokensDetails": { + "additionalProperties": true, + "properties": { + "ttl_12h_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ttl 12H Tokens" + }, + "ttl_30m_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Ttl 30M Tokens" + } + }, + "title": "CacheWriteTokensDetails", + "type": "object" + }, "CachedTokensDetails": { "properties": { "audio_tokens": { @@ -24354,6 +24383,27 @@ ], "title": "Audio Tokens" }, + "cache_write_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Cache Write Tokens" + }, + "cache_write_tokens_details": { + "anyOf": [ + { + "$ref": "#/components/schemas/CacheWriteTokensDetails" + }, + { + "type": "null" + } + ] + }, "cached_tokens": { "default": 0, "title": "Cached Tokens", diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index d9ebf50cc1a..6a3cbcea8fb 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -3401,11 +3401,6 @@ class LiteLLMCompletionResponsesConfig: cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr( prompt_details, "cache_creation_tokens", None ) - cache_write_extra: Final[Mapping[str, int]] = ( - MappingProxyType({"cache_write_tokens": cache_write_tokens}) - if cache_write_tokens is not None - else MappingProxyType({}) - ) cache_type: Final = getattr(prompt_details, "cache_type", None) cache_type_extra: Final[Mapping[str, str]] = ( MappingProxyType({"cache_type": cache_type}) if cache_type is not None else MappingProxyType({}) @@ -3426,18 +3421,20 @@ class LiteLLMCompletionResponsesConfig: if count is not None } ) - response_usage.input_tokens_details = InputTokensDetails( - cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, - text_tokens=prompt_details.text_tokens, - audio_tokens=prompt_details.audio_tokens, - image_tokens=prompt_details.image_tokens, - video_tokens=prompt_details.video_tokens, - cached_tokens_details=( - cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None - ), - **cache_write_extra, - **cache_type_extra, - **grounding_request_counts, + response_usage.input_tokens_details = InputTokensDetails.model_validate( + { + "cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0, + "text_tokens": prompt_details.text_tokens, + "audio_tokens": prompt_details.audio_tokens, + "image_tokens": prompt_details.image_tokens, + "video_tokens": prompt_details.video_tokens, + "cached_tokens_details": ( + cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None + ), + "cache_write_tokens": cache_write_tokens, + **cache_type_extra, + **grounding_request_counts, + } ) # Translate completion_tokens_details to output_tokens_details diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 503d85aaed9..fd5bc0a067a 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -13,6 +13,8 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, + CacheWriteTokensDetails, + InputTokensDetails, OutputTokensDetails, ResponseAPIUsage, ResponseInputParam, @@ -23,6 +25,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.responses.main import DecodedResponseId from litellm.types.utils import ( + CacheCreationTokenDetails, CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, SpecialEnums, @@ -41,6 +44,23 @@ def _output_token_detail(details: object, field: str) -> int | None: return value if isinstance(value, int) else None +def _normalize_input_tokens_details(details: object) -> InputTokensDetails: + if isinstance(details, InputTokensDetails): + return details + return InputTokensDetails.model_validate(details) + + +def _cache_creation_token_details( + details: CacheWriteTokensDetails | None, +) -> CacheCreationTokenDetails | None: + if details is None: + return None + return CacheCreationTokenDetails( + ephemeral_5m_input_tokens=details.ttl_30m_tokens, + ephemeral_1h_input_tokens=details.ttl_12h_tokens, + ) + + def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything return isinstance(value, list) @@ -1246,25 +1266,25 @@ class ResponseAPILoggingUtils: completion_tokens: Final[int] = response_api_usage.output_tokens or 0 prompt_tokens_details: PromptTokensDetailsWrapper | None = None if response_api_usage.input_tokens_details: - if isinstance(response_api_usage.input_tokens_details, dict): - prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details) - else: - prompt_tokens_details = PromptTokensDetailsWrapper( - cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), - audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), - text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), - image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), - video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), - cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), - cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), - web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), - google_maps_grounding_requests=getattr( - response_api_usage.input_tokens_details, "google_maps_grounding_requests", None - ), - ) + input_tokens_details: Final[InputTokensDetails] = _normalize_input_tokens_details( + response_api_usage.input_tokens_details + ) + cache_write_tokens_details: Final[CacheWriteTokensDetails | None] = ( + input_tokens_details.cache_write_tokens_details + ) + prompt_tokens_details = PromptTokensDetailsWrapper( + cached_tokens=input_tokens_details.cached_tokens, + audio_tokens=input_tokens_details.audio_tokens, + text_tokens=input_tokens_details.text_tokens, + image_tokens=input_tokens_details.image_tokens, + cached_tokens_details=input_tokens_details.cached_tokens_details, + video_tokens=input_tokens_details.video_tokens, + cache_write_tokens=input_tokens_details.cache_write_tokens, + cache_type=getattr(input_tokens_details, "cache_type", None), + web_search_requests=getattr(input_tokens_details, "web_search_requests", None), + google_maps_grounding_requests=getattr(input_tokens_details, "google_maps_grounding_requests", None), + cache_creation_token_details=_cache_creation_token_details(cache_write_tokens_details), + ) completion_tokens_details: CompletionTokensDetailsWrapper | None = None output_tokens_details: Final[OutputTokensDetails | None] = getattr( response_api_usage, "output_tokens_details", None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8dbc0d97d54..5aeb6a724b8 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1381,10 +1381,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject): model_config = {"extra": "allow"} +class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject): + ttl_30m_tokens: int | None = None + ttl_12h_tokens: int | None = None + + model_config = ConfigDict(frozen=True) + + +def _is_none(value: object) -> bool: + return value is None + + class InputTokensDetails(BaseLiteLLMOpenAIResponseObject): audio_tokens: int | None = None cached_tokens: int = 0 cached_tokens_details: CachedTokensDetails | None = None + cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none) + cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none) image_tokens: int | None = None text_tokens: int | None = None video_tokens: int | None = None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb9bfa6ffe4..e2c418708c3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -36699,6 +36699,8 @@ }, "gpt-6-astra": { "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 0.000025, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00005, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05, @@ -36784,6 +36786,8 @@ }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 0.000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -36861,6 +36865,8 @@ }, "gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 0.00000025, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000005, "cache_creation_input_token_cost_decisions": 1e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07, @@ -36947,6 +36953,8 @@ }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 0.00001, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37021,6 +37029,8 @@ }, "gpt-5.6-sol": { "cache_creation_input_token_cost": 5e-06, + "cache_creation_input_token_cost_above_1hr": 0.00001, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002, "cache_creation_input_token_cost_above_272k_tokens": 1e-05, "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05, @@ -37097,6 +37107,8 @@ }, "gpt-5.6-terra": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 0.000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, @@ -37172,6 +37184,8 @@ }, "gpt-5.6-luna": { "cache_creation_input_token_cost": 2.5e-07, + "cache_creation_input_token_cost_above_1hr": 0.0000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.000001, "cache_creation_input_token_cost_above_272k_tokens": 5e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06, @@ -37247,6 +37261,8 @@ }, "gpt-5.6-cyber": { "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_above_1hr": 0.00003125, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000625, "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_above_272k_tokens": 2.5e-06, @@ -74677,6 +74693,8 @@ }, "gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, + "cache_creation_input_token_cost_above_1hr": 0.000005, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py index b32f9f63073..21cbf370dff 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py @@ -13,13 +13,16 @@ break-outs that overlap it, matching the Anthropic path. A request with no cache activity leaves both fields unset, which is the documented behavior. """ +from collections.abc import Mapping from datetime import datetime +from typing import Final, cast import pytest import litellm from litellm.litellm_core_utils.litellm_logging import Logging from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.llms.openai import CacheWriteTokensDetails, InputTokensDetails, ResponseAPIUsage from litellm.types.utils import Choices, Message, ModelResponse MODEL = "gpt-5.6" @@ -93,6 +96,61 @@ def test_responses_api_cache_costs_are_itemized_in_the_breakdown(local_model_cos assert total == pytest.approx(breakdown["total_cost"]) +@pytest.mark.parametrize("details_as_object", [False, True], ids=["dict", "model"]) +def test_responses_api_12h_cache_writes_use_the_12h_rate(local_model_cost_map, details_as_object: bool): + rates: Final[Mapping[str, float]] = cast(Mapping[str, float], litellm.model_cost[MODEL]) + usage_input: Final[dict[str, object] | ResponseAPIUsage] = ( + ResponseAPIUsage( + input_tokens=12000, + output_tokens=200, + total_tokens=12200, + input_tokens_details=InputTokensDetails( + cached_tokens=4000, + cache_write_tokens=7000, + cache_write_tokens_details=CacheWriteTokensDetails(ttl_30m_tokens=3000, ttl_12h_tokens=4000), + ), + ) + if details_as_object + else { + "input_tokens": 12000, + "output_tokens": 200, + "total_tokens": 12200, + "input_tokens_details": { + "cached_tokens": 4000, + "cache_write_tokens": 7000, + "cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 4000}, + }, + } + ) + usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_input) + completion_response: Final = ModelResponse( + id="x", + created=1, + model=MODEL, + object="chat.completion", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=usage, + ) + logging_obj: Final = _logging_obj() + + litellm.completion_cost( + completion_response=completion_response, + model=MODEL, + custom_llm_provider="openai", + litellm_logging_obj=logging_obj, + ) + + breakdown: Final = logging_obj.cost_breakdown + assert breakdown is not None + expected_input: Final = ( + 1000 * rates["input_cost_per_token"] + + 4000 * rates["cache_read_input_token_cost"] + + 3000 * rates["cache_creation_input_token_cost"] + + 4000 * rates["cache_creation_input_token_cost_above_1hr"] + ) + assert breakdown["input_cost"] == pytest.approx(expected_input) + + def test_cache_fields_stay_unset_when_there_was_no_cache_activity(local_model_cost_map): """A request that neither read nor wrote the cache leaves both break-out fields off the breakdown, so they serialize as null rather than a misleading $0.""" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cdff03af3b4..108778a670a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -940,6 +940,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_audio_token_cost": {"type": "number"}, "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, + "cache_creation_input_token_cost_above_1hr_above_272k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_32k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_100k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2b174a1141a..754efbe1c9d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -28452,6 +28452,15 @@ export interface components { */ uncached_input_tokens: number; }; + /** CacheWriteTokensDetails */ + CacheWriteTokensDetails: { + /** Ttl 12H Tokens */ + ttl_12h_tokens?: number | null; + /** Ttl 30M Tokens */ + ttl_30m_tokens?: number | null; + } & { + [key: string]: unknown; + }; /** CachedTokensDetails */ CachedTokensDetails: { /** Audio Tokens */ @@ -33607,6 +33616,9 @@ export interface components { InputTokensDetails: { /** Audio Tokens */ audio_tokens?: number | null; + /** Cache Write Tokens */ + cache_write_tokens?: number | null; + cache_write_tokens_details?: components["schemas"]["CacheWriteTokensDetails"] | null; /** * Cached Tokens * @default 0