diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5dc0ce13de..7e4f5e3f4c1 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2075,6 +2075,17 @@ def completion_cost( ) else: additional_costs = None + if custom_llm_provider in ("gemini", "vertex_ai", "vertex_ai_beta"): + from litellm.llms.vertex_ai.context_caching.storage_pricing import ( + recorded_cache_storage_cost, + ) + + cache_storage_cost = recorded_cache_storage_cost(litellm_logging_obj) + if cache_storage_cost: + additional_costs = { + **(additional_costs or {}), + "gemini_context_cache_storage": cache_storage_cost, + } _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( diff --git a/litellm/llms/vertex_ai/context_caching/storage_pricing.py b/litellm/llms/vertex_ai/context_caching/storage_pricing.py new file mode 100644 index 00000000000..aca90d861c9 --- /dev/null +++ b/litellm/llms/vertex_ai/context_caching/storage_pricing.py @@ -0,0 +1,95 @@ +""" +Storage pricing for Gemini / Vertex AI explicit context caches. + +Google bills a cachedContents object twice: once for the cached tokens when it is created, and then for storage, +per cached token per hour, until it expires. LiteLLM creates the cache on the caller's behalf from a client +``cache_control`` marker, takes the TTL from the client, and never deletes the cache, so the storage Google bills is +``cached tokens x lifetime hours x storage rate`` where the lifetime is the cache's ``expireTime - createTime`` (the +requested TTL, or Google's 1 hour default). + +* ``cache_storage_cost_per_token_per_hour`` in the model cost map is the storage rate. +* ``gemini_context_cache_max_ttl_seconds`` (litellm setting, default 3600) caps the TTL a client may request; a + larger value is lowered to the cap before the cache is created. +* A model without a storage rate gets no explicit cache (the request is sent uncached), so storage is never bought + without a price. +""" + +from datetime import datetime +from typing import Final + +import litellm + +CACHE_STORAGE_PRICE_KEY: Final = "cache_storage_cost_per_token_per_hour" +GOOGLE_DEFAULT_CACHE_TTL_SECONDS: Final = 3600.0 +DEFAULT_MAX_CACHE_TTL_SECONDS: Final = 3600.0 + + +def max_cache_ttl_seconds() -> float: + configured: Final[object] = getattr(litellm, "gemini_context_cache_max_ttl_seconds", None) + if isinstance(configured, (int, float)) and not isinstance(configured, bool) and configured > 0: + return float(configured) + return DEFAULT_MAX_CACHE_TTL_SECONDS + + +def cap_cache_ttl(ttl: str | None) -> str | None: + """Lower a normalized ``"s"`` TTL to the configured maximum. ``None`` keeps Google's 1 hour default + unless the cap is shorter than that.""" + cap: Final = max_cache_ttl_seconds() + seconds: Final = float(ttl[:-1]) if ttl else GOOGLE_DEFAULT_CACHE_TTL_SECONDS + if seconds <= cap: + return ttl + return f"{cap:.9f}".rstrip("0").rstrip(".") + "s" + + +def cache_storage_price(model: str, custom_llm_provider: str) -> float | None: + """Storage rate (USD per cached token per hour) from the model cost map, or None when the model has none.""" + provider_prefixed: Final = f"{custom_llm_provider}/{model}" + candidates: Final = (provider_prefixed, model) if custom_llm_provider == "gemini" else (model, provider_prefixed) + for key in candidates: + entry = litellm.model_cost.get(key) + if isinstance(entry, dict): + price = entry.get(CACHE_STORAGE_PRICE_KEY) + if isinstance(price, (int, float)) and not isinstance(price, bool) and price >= 0: + return float(price) + return None + + +def _parse_time(value: object) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def cache_lifetime_hours(created: dict, requested_ttl: str | None) -> float: + """Lifetime Google will bill storage for: the created cache's expireTime - createTime, else the requested TTL, + else Google's 1 hour default.""" + create_time: Final = _parse_time(created.get("createTime")) + expire_time: Final = _parse_time(created.get("expireTime")) + if create_time is not None and expire_time is not None and expire_time > create_time: + return (expire_time - create_time).total_seconds() / 3600.0 + if requested_ttl: + try: + return float(requested_ttl[:-1]) / 3600.0 + except ValueError: + pass + return GOOGLE_DEFAULT_CACHE_TTL_SECONDS / 3600.0 + + +def record_cache_storage( + logging_obj: object, created: dict, requested_ttl: str | None, model: str, custom_llm_provider: str +) -> None: + """Store the storage cost of a cache this request created on the logging object; the cost calculator adds it.""" + tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0 + price: Final = cache_storage_price(model, custom_llm_provider) or 0.0 + hours: Final = cache_lifetime_hours(created, requested_ttl) + cost: Final = max(tokens, 0) * hours * price + logging_obj.vertex_ai_cache_storage_token_hours = max(tokens, 0) * hours # type: ignore[attr-defined] # rebind-ok + logging_obj.vertex_ai_cache_storage_cost = cost # type: ignore[attr-defined] # rebind-ok + + +def recorded_cache_storage_cost(logging_obj: object | None) -> float: + recorded: Final[object] = getattr(logging_obj, "vertex_ai_cache_storage_cost", 0.0) if logging_obj else 0.0 + return float(recorded) if isinstance(recorded, (int, float)) and recorded > 0 else 0.0 diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index f683ad0e377..481c0c28b54 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -15,6 +15,7 @@ from litellm.types.llms.vertex_ai import CachedContentRequestBody from litellm.utils import is_cached_message from ..common_utils import get_supports_system_message +from .storage_pricing import cap_cache_ttl from ..gemini.transformation import ( gemini_convert_messages_with_history, transform_system_message, @@ -167,7 +168,8 @@ def transform_openai_messages_to_gemini_context_caching( vertex_location: str | None, ) -> CachedContentRequestBody: # Extract TTL from cached messages BEFORE system message transformation - ttl: Final = extract_ttl_from_cached_messages(messages) + # Google bills storage per token-hour for the whole TTL; keep the client's TTL within the operator's cap. + ttl: Final = cap_cache_ttl(extract_ttl_from_cached_messages(messages)) supports_system_message: Final = get_supports_system_message(model=model, custom_llm_provider=custom_llm_provider) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 5bff43828d6..13b059ab560 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -21,6 +21,7 @@ from litellm.utils import is_prompt_caching_valid_prompt from ..common_utils import VertexAIError, get_vertex_base_url from ..vertex_llm_base import VertexBase from .known_cached_contents import KnownCachedContents, known_cached_contents +from .storage_pricing import cache_storage_price, record_cache_storage from .transformation import ( cached_messages_end_on_supported_turn, separate_cached_messages, @@ -53,6 +54,7 @@ class _CachedContentUsageMetadata(TypedDict): @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) class _CreatedCachedContent(TypedDict): name: ReadOnly[NotRequired[str | None]] + createTime: ReadOnly[NotRequired[str | None]] expireTime: ReadOnly[NotRequired[str | None]] usageMetadata: ReadOnly[NotRequired[_CachedContentUsageMetadata | None]] @@ -393,6 +395,14 @@ class ContextCachingEndpoints(VertexBase): ) return messages, optional_params, None + # Storage is billed per token-hour for the cache's lifetime; never buy it without a price. + if cache_storage_price(model, custom_llm_provider) is None: + verbose_logger.debug( + "Vertex AI context caching: no cache_storage_cost_per_token_per_hour for %s. Skipping context caching.", + model, + ) + return messages, optional_params, None + tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -487,6 +497,9 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root _record_cache_creation_tokens(logging_obj, raw_response_cached) + record_cache_storage( + logging_obj, raw_response_cached, cached_content_request_body.get("ttl"), model, custom_llm_provider + ) self._known_caches.remember( KnownCachedContents.scope( url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location @@ -559,6 +572,14 @@ class ContextCachingEndpoints(VertexBase): ) return messages, optional_params, None + # Storage is billed per token-hour for the cache's lifetime; never buy it without a price. + if cache_storage_price(model, custom_llm_provider) is None: + verbose_logger.debug( + "Vertex AI context caching: no cache_storage_cost_per_token_per_hour for %s. Skipping context caching.", + model, + ) + return messages, optional_params, None + tools: Final = optional_params.pop("tools", None) tool_choice: Final = optional_params.pop("tool_choice", None) @@ -654,6 +675,9 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root _record_cache_creation_tokens(logging_obj, raw_response_cached) + record_cache_storage( + logging_obj, raw_response_cached, cached_content_request_body.get("ttl"), model, custom_llm_provider + ) self._known_caches.remember( scope, generated_cache_key, raw_response_cached.get("name"), raw_response_cached.get("expireTime") ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 23c6ccf9c0a..2e36b04dc82 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28793,6 +28793,7 @@ "cache_read_input_audio_token_cost": 1e-07, "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, @@ -29149,6 +29150,7 @@ }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -29200,6 +29202,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -29261,6 +29264,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, @@ -29356,6 +29360,7 @@ "cache_read_input_audio_token_cost": 3e-08, "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, @@ -29412,6 +29417,7 @@ }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -29458,6 +29464,7 @@ }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-language-models", @@ -29638,6 +29645,7 @@ "gemini-2.5-pro": { "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, @@ -29752,11 +29760,13 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_storage_cost_per_token_per_hour": 4.5e-06 }, "gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, @@ -30715,6 +30725,7 @@ "gemini/gemini-3.1-flash-lite-preview": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -30874,6 +30885,7 @@ "cache_read_input_audio_token_cost_flex": 1e-07, "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, @@ -31200,6 +31212,7 @@ "cache_read_input_audio_token_cost_flex": 3e-08, "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, @@ -31398,6 +31411,7 @@ }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_batches": 1.25e-07, @@ -31492,6 +31506,7 @@ "cache_read_input_audio_token_cost_flex": 2.5e-08, "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -31556,6 +31571,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 2e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -31620,6 +31636,7 @@ "cache_read_input_audio_token_cost_flex": 1e-07, "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, @@ -31683,6 +31700,7 @@ "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -31745,6 +31763,7 @@ "gemini/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -31806,6 +31825,7 @@ "gemini/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -31868,6 +31888,7 @@ "gemini/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -31963,6 +31984,7 @@ "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -32026,6 +32048,7 @@ "gemini/gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -32137,7 +32160,8 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, @@ -32173,6 +32197,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -32233,6 +32258,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -32293,6 +32319,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -32405,10 +32432,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-3.8-flash-cyber": { "cache_read_input_token_cost": 1.5e-07, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, @@ -52258,6 +52287,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -52309,6 +52339,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -52371,6 +52402,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 23c6ccf9c0a..2e36b04dc82 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28793,6 +28793,7 @@ "cache_read_input_audio_token_cost": 1e-07, "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_audio_token": 1e-06, @@ -29149,6 +29150,7 @@ }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -29200,6 +29202,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -29261,6 +29264,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, @@ -29356,6 +29360,7 @@ "cache_read_input_audio_token_cost": 3e-08, "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, "input_cost_per_audio_token": 3e-07, @@ -29412,6 +29417,7 @@ }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 3e-07, "input_cost_per_token": 1e-07, "litellm_provider": "vertex_ai-language-models", @@ -29458,6 +29464,7 @@ }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "litellm_provider": "vertex_ai-language-models", @@ -29638,6 +29645,7 @@ "gemini-2.5-pro": { "deprecation_date": "2026-10-20", "cache_read_input_token_cost": 1.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, @@ -29752,11 +29760,13 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_storage_cost_per_token_per_hour": 4.5e-06 }, "gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, "input_cost_per_token": 2e-06, @@ -30715,6 +30725,7 @@ "gemini/gemini-3.1-flash-lite-preview": { "cache_read_input_audio_token_cost": 5e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -30874,6 +30885,7 @@ "cache_read_input_audio_token_cost_flex": 1e-07, "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, @@ -31200,6 +31212,7 @@ "cache_read_input_audio_token_cost_flex": 3e-08, "cache_read_input_audio_token_cost_priority": 5.4e-08, "cache_read_input_token_cost": 1e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, @@ -31398,6 +31411,7 @@ }, "gemini/gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_batches": 1.25e-07, @@ -31492,6 +31506,7 @@ "cache_read_input_audio_token_cost_flex": 2.5e-08, "cache_read_input_audio_token_cost_priority": 9e-08, "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -31556,6 +31571,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 2e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, @@ -31620,6 +31636,7 @@ "cache_read_input_audio_token_cost_flex": 1e-07, "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "input_cost_per_audio_token": 1e-06, @@ -31683,6 +31700,7 @@ "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 1.5e-07, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -31745,6 +31763,7 @@ "gemini/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -31806,6 +31825,7 @@ "gemini/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -31868,6 +31888,7 @@ "gemini/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -31963,6 +31984,7 @@ "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -32026,6 +32048,7 @@ "gemini/gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_200k_tokens": 4e-06, @@ -32137,7 +32160,8 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, @@ -32173,6 +32197,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, @@ -32233,6 +32258,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -32293,6 +32319,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, @@ -32405,10 +32432,12 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, - "web_search_billing_unit": "per_query" + "web_search_billing_unit": "per_query", + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-3.8-flash-cyber": { "cache_read_input_token_cost": 1.5e-07, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, @@ -52258,6 +52287,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -52309,6 +52339,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, @@ -52371,6 +52402,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_storage_cost_per_token_per_hour": 1.0e-06, "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, diff --git a/tests/unit/llms/vertex_ai/context_caching/test_context_cache_storage_pricing.py b/tests/unit/llms/vertex_ai/context_caching/test_context_cache_storage_pricing.py new file mode 100644 index 00000000000..a5bda5a13e9 --- /dev/null +++ b/tests/unit/llms/vertex_ai/context_caching/test_context_cache_storage_pricing.py @@ -0,0 +1,174 @@ +"""Storage pricing for Gemini explicit context caches created from a client ``cache_control`` marker. + +Google bills a cachedContents object for storage per cached token per hour until it expires; LiteLLM picks the TTL +from the client and never deletes the cache. The spend LiteLLM records for the request that created the cache must +include that storage, and the TTL a client can buy must be bounded by the operator. +""" + +import math +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.types.utils import ModelResponse, Usage + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.vertex_ai.context_caching.storage_pricing import ( + cache_lifetime_hours, + cache_storage_price, + cap_cache_ttl, + recorded_cache_storage_cost, +) +from litellm.llms.vertex_ai.context_caching.transformation import ( + transform_openai_messages_to_gemini_context_caching, +) +from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ContextCachingEndpoints + +PRO_RATE = 4.5e-06 # USD per cached token per hour, Gemini 2.5 Pro (Vertex AI pricing, context cache storage) + + +@pytest.fixture(autouse=True) +def local_cost_map(monkeypatch): + original = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + monkeypatch.delattr(litellm, "gemini_context_cache_max_ttl_seconds", raising=False) + yield + litellm.model_cost = original + litellm.get_model_info.cache_clear() + + +def _messages(ttl: str | None) -> list: + control = {"type": "ephemeral"} | ({"ttl": ttl} if ttl else {}) + return [ + {"role": "system", "content": [{"type": "text", "text": "long prefix " * 400, "cache_control": control}]}, + {"role": "user", "content": "question"}, + ] + + +def test_storage_rate_is_in_the_cost_map(): + assert cache_storage_price("gemini-2.5-pro", "gemini") == PRO_RATE + assert cache_storage_price("gemini-2.5-pro", "vertex_ai") == PRO_RATE + assert cache_storage_price("gemini-2.5-flash", "gemini") == 1e-06 + assert cache_storage_price("no-such-model", "gemini") is None + + +@pytest.mark.parametrize( + "requested, cap, expected", + [ + ("31536000s", None, "3600s"), + ("600s", None, "600s"), + (None, None, None), + ("31536000s", 86400, "86400s"), + (None, 1800, "1800s"), + ], +) +def test_client_ttl_is_capped(monkeypatch, requested, cap, expected): + if cap is not None: + monkeypatch.setattr(litellm, "gemini_context_cache_max_ttl_seconds", cap, raising=False) + assert cap_cache_ttl(requested) == expected + + +def test_transformed_cache_request_carries_the_capped_ttl(): + body = transform_openai_messages_to_gemini_context_caching( + model="gemini-2.5-pro", + messages=_messages("8760h")[:1], + custom_llm_provider="gemini", + cache_key="k", + vertex_project=None, + vertex_location=None, + ) + assert body["ttl"] == "3600s" + + +def test_lifetime_comes_from_the_created_cache(): + created = {"createTime": "2026-10-14T00:00:00Z", "expireTime": "2026-10-14T05:00:00Z"} + assert cache_lifetime_hours(created, "60s") == 5.0 + assert cache_lifetime_hours({}, "7200s") == 2.0 + assert cache_lifetime_hours({}, None) == 1.0 + + +def _created_response(tokens: int, hours: float) -> httpx.Response: + created = datetime(2026, 10, 14, tzinfo=timezone.utc) + expires = datetime.fromtimestamp(created.timestamp() + hours * 3600, timezone.utc) + body = { + "name": "cachedContents/abc", + "usageMetadata": {"totalTokenCount": tokens}, + "createTime": created.strftime("%Y-%m-%dT%H:%M:%SZ"), + "expireTime": expires.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + return httpx.Response(200, json=body, request=httpx.Request("POST", "https://x/cachedContents")) + + +def _endpoints_call(model: str, client: MagicMock, logging_obj: Logging, ttl: str | None): + with patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.is_prompt_caching_valid_prompt", + return_value=True, + ): + return ContextCachingEndpoints().check_and_create_cache( + messages=_messages(ttl), + optional_params={}, + api_key="k", + api_base=None, + model=model, + client=client, + timeout=10, + logging_obj=logging_obj, + custom_llm_provider="gemini", + vertex_project=None, + vertex_location=None, + vertex_auth_header=None, + ) + + +def _logging_obj() -> Logging: + return Logging( + model="gemini/gemini-2.5-pro", + messages=[], + stream=False, + call_type="completion", + start_time=datetime.now(timezone.utc), + litellm_call_id="c", + function_id="f", + ) + + +def test_created_cache_storage_is_recorded_and_priced(): + client = MagicMock(spec=HTTPHandler) + client.get.return_value = httpx.Response(200, json={}, request=httpx.Request("GET", "https://x")) + client.post.return_value = _created_response(tokens=2000, hours=1.0) + logging_obj = _logging_obj() + _, _, name = _endpoints_call("gemini-2.5-pro", client, logging_obj, "8760h") + assert name == "cachedContents/abc" + assert client.post.call_args.kwargs["json"]["ttl"] == "3600s" + assert math.isclose(recorded_cache_storage_cost(logging_obj), 2000 * 1.0 * PRO_RATE, rel_tol=1e-9) + + response = ModelResponse( + model="gemini-2.5-pro", + usage=Usage(prompt_tokens=2100, completion_tokens=10, total_tokens=2110, cache_read_input_tokens=2000), + ) + with_storage = litellm.completion_cost( + completion_response=response, + model="gemini/gemini-2.5-pro", + custom_llm_provider="gemini", + litellm_logging_obj=logging_obj, + ) + without = litellm.completion_cost( + completion_response=response, + model="gemini/gemini-2.5-pro", + custom_llm_provider="gemini", + litellm_logging_obj=_logging_obj(), + ) + assert math.isclose(with_storage - without, 2000 * PRO_RATE, rel_tol=1e-6) + + +def test_model_without_storage_price_gets_no_explicit_cache(): + client = MagicMock(spec=HTTPHandler) + messages, _, name = _endpoints_call("gemini-unpriced-test-model", client, _logging_obj(), "8760h") + assert name is None + client.post.assert_not_called() + assert len(messages) == 2 diff --git a/tests/unit/llms/vertex_ai/context_caching/test_context_caching_ttl.py b/tests/unit/llms/vertex_ai/context_caching/test_context_caching_ttl.py index 44ce97b73ac..59b9298e297 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_context_caching_ttl.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_context_caching_ttl.py @@ -339,8 +339,11 @@ class TestTransformationWithTTL: @pytest.mark.parametrize( "custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"] ) - def test_transform_with_system_message_and_ttl(self, custom_llm_provider): - """Test transformation with system message and TTL""" + def test_transform_with_system_message_and_ttl(self, custom_llm_provider, monkeypatch): + """Test transformation with system message and TTL (operator allows TTLs above the 1 hour default cap)""" + import litellm + + monkeypatch.setattr(litellm, "gemini_context_cache_max_ttl_seconds", 86400, raising=False) messages = [ { "role": "system", diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index 19e7e2dee52..553e2b99679 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -52,6 +52,13 @@ class TestContextCachingEndpoints: return_value=True, ) self._token_check_patcher.start() + # Storage pricing is covered by test_context_cache_storage_pricing.py; these tests use + # models without a storage rate in the cost map, so give every model one here. + self._storage_price_patcher = patch( + "litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.cache_storage_price", + return_value=4.5e-06, + ) + self._storage_price_patcher.start() # Sample messages for testing self.sample_messages = [ @@ -83,6 +90,7 @@ class TestContextCachingEndpoints: def teardown_method(self): """Teardown for each test method""" self._token_check_patcher.stop() + self._storage_price_patcher.stop() @pytest.mark.parametrize( "custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]