diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index afbbfec59e6..2e0ca7805d2 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -58,6 +58,7 @@ pub struct ModelInfoFields { pub cache_creation_input_token_cost_above_12hr_ultrafast: Option, pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option, pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option, + pub cache_storage_cost_per_token_per_hour: Option, pub off_peak_pricing: Option, pub input_cost_per_character: Option, pub input_cost_per_token_above_128k_tokens: Option, diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 401634f8c62..195d5bc9c0f 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -53,6 +53,8 @@ attributes! { bedrock_converse_supports_strict_tools: bool, /// Highest reasoning effort the Bedrock output_config accepts for this model. bedrock_output_config_effort_ceiling: EffortLevel, + /// USD per token per hour to keep an explicitly created context cache stored. + cache_storage_cost_per_token_per_hour: UnitPrice, comment: String, /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'. default_reasoning_effort: EffortLevel, @@ -313,6 +315,8 @@ mod tests { #[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)] #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)] #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)] + #[case::cache_storage_rate("cache_storage_cost_per_token_per_hour", json!(4.5e-6), true)] + #[case::negative_cache_storage_rate("cache_storage_cost_per_token_per_hour", json!(-1), false)] #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)] #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)] #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 3910a554905..c30d1655513 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -328,6 +328,45 @@ def _get_additional_costs( return None +def _recorded_cache_storage_token_hours(litellm_logging_obj: LitellmLoggingObject | None) -> object: + return litellm_logging_obj.context_cache_storage_token_hours if litellm_logging_obj else 0.0 + + +def _context_cache_storage_cost( + litellm_logging_obj: LitellmLoggingObject | None, + *, + requested_model: str | None, + completion_response: object | None, + base_model: str | None, + custom_llm_provider: str | None, + router_model_id: str | None, + region_name: str | None, +) -> float: + token_hours: Final = _recorded_cache_storage_token_hours(litellm_logging_obj) + if not isinstance(token_hours, int | float) or token_hours <= 0: + return 0.0 + underlying_model: Final = select_model_name_for_cost_calc( + model=requested_model, + completion_response=completion_response, + base_model=base_model, + custom_llm_provider=custom_llm_provider, + region_name=region_name, + ) + rate: Final = next( + ( + price + for source in ( + _deployment_model_info(litellm_logging_obj, custom_pricing=True, router_model_id=router_model_id), + _cost_map_model_info(underlying_model, custom_llm_provider) if underlying_model else None, + ) + if source is not None + and isinstance(price := source.get("cache_storage_cost_per_token_per_hour"), int | float) + ), + 0.0, + ) + return token_hours * rate + + _TranscriptionRateKey: TypeAlias = Literal[ "input_cost_per_second", "output_cost_per_second", @@ -2350,6 +2389,20 @@ def completion_cost( ) else: additional_costs = None + _storage_response: ModelResponse | None = ( + completion_response if isinstance(completion_response, ModelResponse) else None + ) + _storage_provider: str | None = custom_llm_provider if isinstance(custom_llm_provider, str) else None + _storage_region: str | None = region_name if isinstance(region_name, str) else None + cache_storage_cost: float = _context_cache_storage_cost( + litellm_logging_obj, + requested_model=requested_model, + completion_response=_storage_response, + base_model=base_model, + custom_llm_provider=_storage_provider, + router_model_id=router_model_id, + region_name=_storage_region, + ) _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -2362,6 +2415,12 @@ def completion_cost( _final_cost += cost_for_built_in_tools if additional_costs: _final_cost += sum(additional_costs.values()) + if cache_storage_cost > 0: + _final_cost += cache_storage_cost + if additional_costs is None: + additional_costs = {"cache_storage_cost": cache_storage_cost} + else: + additional_costs["cache_storage_cost"] = cache_storage_cost original_cost = _final_cost if litellm.cost_discount_config: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9b6ec8d28d6..f7487697271 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -691,6 +691,10 @@ class Logging(LiteLLMLoggingBaseClass): baseline_cache_context: "BaselineCacheContext | None" = None baseline_observation: "CapturedBaselineObservation | None" = None _on_failure_payload_built: "Callable[[], None] | None" = None + context_cache_storage_token_hours: float = 0.0 + + def record_context_cache_storage(self, token_hours: float) -> None: + self.context_cache_storage_token_hours = token_hours @property def _defer_async_logging(self) -> bool: diff --git a/litellm/llms/vertex_ai/context_caching/storage_cost.py b/litellm/llms/vertex_ai/context_caching/storage_cost.py new file mode 100644 index 00000000000..7cb74296f94 --- /dev/null +++ b/litellm/llms/vertex_ai/context_caching/storage_cost.py @@ -0,0 +1,56 @@ +from datetime import datetime +from typing import Final + +from pydantic import BaseModel, ConfigDict, ValidationError + +DEFAULT_CONTEXT_CACHE_TTL_SECONDS: Final = 3600.0 +_SECONDS_PER_HOUR: Final = 3600.0 + + +class _CachedContentUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + totalTokenCount: int = 0 + + +class _CreatedCachedContent(BaseModel): + model_config = ConfigDict(frozen=True) + + usageMetadata: _CachedContentUsage = _CachedContentUsage() + createTime: datetime | None = None + expireTime: datetime | None = None + + +def _reported_lifetime_seconds(created: _CreatedCachedContent) -> float | None: + if created.createTime is None or created.expireTime is None: + return None + seconds: Final = (created.expireTime - created.createTime).total_seconds() + return seconds if seconds > 0 else None + + +def _requested_ttl_seconds(requested_ttl: str | None) -> float | None: + if requested_ttl is None or not requested_ttl.endswith("s"): + return None + try: + seconds: Final = float(requested_ttl[:-1]) + except ValueError: + return None + return seconds if seconds > 0 else None + + +def context_cache_storage_token_hours(created_cached_content: object, requested_ttl: str | None) -> float: + """Token-hours a newly created cachedContents resource is billed for over its whole TTL. + + The lifetime comes from the create response (expireTime - createTime), then the TTL that was + requested, then the provider default of one hour. + """ + try: + created: Final = _CreatedCachedContent.model_validate(created_cached_content) + except ValidationError: + return 0.0 + lifetime_seconds: Final = ( + _reported_lifetime_seconds(created) + or _requested_ttl_seconds(requested_ttl) + or DEFAULT_CONTEXT_CACHE_TTL_SECONDS + ) + return created.usageMetadata.totalTokenCount * lifetime_seconds / _SECONDS_PER_HOUR diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 4603f612926..d120471fab3 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -22,6 +22,7 @@ from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_p from ..common_utils import VertexAIError, get_vertex_base_url from ..vertex_llm_base import VertexBase, publisher_model_location from .known_cached_contents import KnownCachedContents, known_cached_contents +from .storage_cost import context_cache_storage_token_hours from .transformation import ( cached_messages_end_on_supported_turn, separate_cached_messages, @@ -544,6 +545,9 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root _record_cache_creation_tokens(logging_obj, raw_response_cached) + logging_obj.record_context_cache_storage( + context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl")) + ) self._known_caches.remember( KnownCachedContents.scope( url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location @@ -696,6 +700,9 @@ class ContextCachingEndpoints(VertexBase): raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root _record_cache_creation_tokens(logging_obj, raw_response_cached) + logging_obj.record_context_cache_storage( + context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl")) + ) self._known_caches.remember( scope, generated_cache_key, raw_response_cached.get("name"), raw_response_cached.get("expireTime") ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 868dcdad74d..2e5ba77ac53 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -168,6 +168,7 @@ "max_output_tokens": 4096 }, "sample_spec": { + "cache_storage_cost_per_token_per_hour": 0.0, "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, "computer_use_output_cost_per_1k_tokens": 0.0, @@ -29582,6 +29583,7 @@ "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-10-20", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 1e-06, @@ -29948,6 +29950,7 @@ }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -30005,6 +30008,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, @@ -30070,6 +30074,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -30166,6 +30171,7 @@ "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-10-20", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 3e-07, @@ -30458,6 +30464,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -30541,6 +30548,7 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "prompt_cache_min_tokens": 4096.0, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -30673,6 +30681,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30686,6 +30695,7 @@ "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_above_200k_tokens": 1e-06, @@ -30801,7 +30811,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -30854,6 +30865,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "vertex_ai/gemini-3.8-flash", @@ -30918,6 +30930,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "vertex_ai/gemini-3.8-flash", @@ -30935,6 +30948,7 @@ "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30992,6 +31006,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -31101,6 +31116,7 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "prompt_cache_min_tokens": 4096.0, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -32989,6 +33005,7 @@ "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_above_200k_tokens": 1e-06, @@ -33137,7 +33154,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -33190,6 +33208,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "gemini-3.8-flash", @@ -33254,6 +33273,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "gemini-3.8-flash", @@ -33271,6 +33291,7 @@ "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -33328,6 +33349,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -53795,6 +53817,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -53852,6 +53875,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, @@ -53918,6 +53942,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, diff --git a/litellm/types/router.py b/litellm/types/router.py index a9f28fbc673..73109d8bb0d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -331,6 +331,7 @@ class ModelInfo(MirroredPricingParams): ptu_effective_from: datetime.datetime | None = None ptu_effective_to: datetime.datetime | None = None ptu_shares: Mapping[str, Annotated[StrictInt, BeforeValidator(_whole_float_as_int)]] | None = None + cache_storage_cost_per_token_per_hour: float | None = None # when tag-based routing's "!" or "&" constraints eliminate every deployment # in this model group, fall back to the default-tagged pool instead of diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 7c137b72c3e..e8f31b064fa 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -377,6 +377,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_creation_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] + cache_storage_cost_per_token_per_hour: ReadOnly[float | None] # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None @@ -4297,6 +4298,8 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) +DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS: Final[frozenset[str]] = frozenset({"cache_storage_cost_per_token_per_hour"}) + DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( { "supports_regex_lookaround", @@ -4327,6 +4330,7 @@ SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) - frozenset(CustomPricingLiteLLMParams.model_fields) - DEPLOYMENT_SCOPED_PRICING_FIELDS + - DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS - DEPLOYMENT_SCOPED_CAPABILITY_FIELDS - DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS ) diff --git a/litellm/utils.py b/litellm/utils.py index 53b02ebddac..3a387af58fc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7438,6 +7438,7 @@ def get_model_info_helper( cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast=_model_info.get( "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast", None ), + cache_storage_cost_per_token_per_hour=_model_info.get("cache_storage_cost_per_token_per_hour"), off_peak_pricing=_model_info.get("off_peak_pricing", None), input_cost_per_character=_model_info.get("input_cost_per_character", None), input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 868dcdad74d..2e5ba77ac53 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -168,6 +168,7 @@ "max_output_tokens": 4096 }, "sample_spec": { + "cache_storage_cost_per_token_per_hour": 0.0, "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, "computer_use_output_cost_per_1k_tokens": 0.0, @@ -29582,6 +29583,7 @@ "cache_read_input_token_cost": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-10-20", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 1e-06, @@ -29948,6 +29950,7 @@ }, "gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -30005,6 +30008,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, @@ -30070,6 +30074,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, @@ -30166,6 +30171,7 @@ "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 1.8e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-10-20", "google_maps_grounding_cost_per_query": 0.025, "input_cost_per_audio_token": 3e-07, @@ -30458,6 +30464,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07, "cache_read_input_token_cost_flex": 1.25e-07, "cache_read_input_token_cost_priority": 2.25e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "input_cost_per_token": 1.25e-06, "input_cost_per_token_above_200k_tokens": 2.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -30541,6 +30548,7 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "prompt_cache_min_tokens": 4096.0, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -30673,6 +30681,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30686,6 +30695,7 @@ "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_above_200k_tokens": 1e-06, @@ -30801,7 +30811,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -30854,6 +30865,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "vertex_ai/gemini-3.8-flash", @@ -30918,6 +30930,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "vertex_ai/gemini-3.8-flash", @@ -30935,6 +30948,7 @@ "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30992,6 +31006,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -31101,6 +31116,7 @@ "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.16e-05, "prompt_cache_min_tokens": 4096.0, + "cache_storage_cost_per_token_per_hour": 4.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -32989,6 +33005,7 @@ "cache_read_input_audio_token_cost_priority": 1.8e-07, "cache_read_input_token_cost": 5e-08, "cache_read_input_token_cost_priority": 9e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_above_200k_tokens": 1e-06, @@ -33137,7 +33154,8 @@ "output_cost_per_token_batches": 4.5e-06, "input_cost_per_token_flex": 7.5e-07, "output_cost_per_token_flex": 4.5e-06, - "cache_read_input_token_cost_flex": 7.5e-08 + "cache_read_input_token_cost_flex": 7.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06 }, "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, @@ -33190,6 +33208,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2026-11-19", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "gemini-3.8-flash", @@ -33254,6 +33273,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-01-28", "deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions", "successor_model": "gemini-3.8-flash", @@ -33271,6 +33291,7 @@ "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -33328,6 +33349,7 @@ "cache_read_input_token_cost_batches": 7.5e-08, "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, "input_cost_per_token_flex": 7.5e-07, @@ -53795,6 +53817,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-preview": { "cache_read_input_token_cost": 2.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 2.5e-07, "litellm_provider": "vertex_ai-language-models", @@ -53852,6 +53875,7 @@ "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "deprecation_date": "2027-05-07", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_audio_token": 5e-07, @@ -53918,6 +53942,7 @@ "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "cache_storage_cost_per_token_per_hour": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, "input_cost_per_token_flex": 1.5e-07, diff --git a/tests/unit/llms/vertex_ai/context_caching/test_storage_cost.py b/tests/unit/llms/vertex_ai/context_caching/test_storage_cost.py new file mode 100644 index 00000000000..5fd3bbad9cc --- /dev/null +++ b/tests/unit/llms/vertex_ai/context_caching/test_storage_cost.py @@ -0,0 +1,51 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.context_caching.storage_cost import ( + context_cache_storage_token_hours, +) + + +@pytest.mark.parametrize( + "created, requested_ttl, expected", + [ + ( + { + "usageMetadata": {"totalTokenCount": 5000}, + "createTime": "2026-01-01T00:00:00.123456789Z", + "expireTime": "2026-01-01T02:00:00.123456789Z", + }, + "60s", + 10000.0, + ), + ({"usageMetadata": {"totalTokenCount": 5000}}, "1800s", 2500.0), + ({"usageMetadata": {"totalTokenCount": 5000}}, "90.5s", 5000 * 90.5 / 3600), + ( + {"usageMetadata": {"totalTokenCount": 4000}, "expireTime": "2026-01-01T02:00:00Z"}, + "900s", + 1000.0, + ), + ({"name": "cachedContents/1"}, "3600s", 0.0), + ({"usageMetadata": {"totalTokenCount": "many"}}, "3600s", 0.0), + ({"usageMetadata": {"totalTokenCount": 5000}, "expireTime": "not-a-time"}, "3600s", 0.0), + ("not-an-object", "3600s", 0.0), + ], +) +def test_token_hours_follow_reported_lifetime_then_requested_ttl( + created: object, requested_ttl: str | None, expected: float +) -> None: + assert context_cache_storage_token_hours(created, requested_ttl) == pytest.approx(expected) + + +def test_token_hours_default_to_one_hour_without_lifetime_or_ttl() -> None: + # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-create + # "The default expiration time of a context cache is 60 minutes after it's created" (read 2026-10-07) + assert context_cache_storage_token_hours({"usageMetadata": {"totalTokenCount": 5000}}, None) == 5000.0 + + +@pytest.mark.parametrize("requested_ttl", ["", "s", "abc", "10m", "-5s"]) +def test_unparseable_requested_ttl_falls_back_to_default_lifetime(requested_ttl: str) -> None: + created: Final = {"usageMetadata": {"totalTokenCount": 3600}} + + assert context_cache_storage_token_hours(created, requested_ttl) == context_cache_storage_token_hours(created, None) diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index bc8421c80f8..b6ccab05883 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,9 +1,13 @@ +import asyncio import json +from collections.abc import Mapping +from datetime import datetime, timezone from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from typing_extensions import ReadOnly, TypedDict import litellm @@ -14,6 +18,7 @@ from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCa from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ( ContextCachingEndpoints, ) +from litellm.types.llms.openai import AllMessageValues @pytest.fixture @@ -2378,6 +2383,212 @@ class TestContextCachingMultiRegionUrls: assert "/locations/global/cachedContents" in url +_STORAGE_TEST_MODEL: Final = "gemini-2.5-pro" + + +_STORAGE_TEST_CACHE_NAME: Final = "cachedContents/storage-cost" + + +def _storage_logging_obj() -> Logging: + logging_obj: Final = Logging( + model=_STORAGE_TEST_MODEL, + messages=[], + stream=False, + call_type="acompletion", + start_time=None, + litellm_call_id="context-cache-storage", + function_id="context-cache-storage", + ) + logging_obj.update_environment_variables( + model=_STORAGE_TEST_MODEL, litellm_params={}, optional_params={}, custom_llm_provider="gemini" + ) + return logging_obj + + +def _storage_messages() -> list[AllMessageValues]: + return [ + { + "role": "system", + "content": [{"type": "text", "text": "policy " * 6000, "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "Hello"}, + ] + + +def _json_response(method: str, payload: Mapping[str, object]) -> httpx.Response: + return httpx.Response(200, json=payload, request=httpx.Request(method, "https://example.invalid")) + + +_CREATED_CACHE: Final = { + "name": _STORAGE_TEST_CACHE_NAME, + "model": f"models/{_STORAGE_TEST_MODEL}", + "usageMetadata": {"totalTokenCount": 7200}, + "createTime": "2026-01-01T00:00:00Z", + "expireTime": "2026-01-01T00:30:00Z", +} + + +class _CreateCacheKwargs(TypedDict): + messages: ReadOnly[list[AllMessageValues]] + optional_params: ReadOnly[dict[str, object]] + api_key: ReadOnly[str] + api_base: ReadOnly[str | None] + model: ReadOnly[str] + client: ReadOnly[MagicMock] + timeout: ReadOnly[float] + logging_obj: ReadOnly[Logging] + custom_llm_provider: ReadOnly[Literal["gemini"]] + vertex_project: ReadOnly[str | None] + vertex_location: ReadOnly[str | None] + vertex_auth_header: ReadOnly[str | None] + + +def _create_kwargs(logging_obj: Logging, client: MagicMock) -> _CreateCacheKwargs: + return { + "messages": _storage_messages(), + "optional_params": {}, + "api_key": "test_key", + "api_base": None, + "model": _STORAGE_TEST_MODEL, + "client": client, + "timeout": 30.0, + "logging_obj": logging_obj, + "custom_llm_provider": "gemini", + "vertex_project": None, + "vertex_location": None, + "vertex_auth_header": None, + } + + +def test_creating_a_cache_records_its_storage_token_hours(local_model_cost_map: None) -> None: + logging_obj: Final = _storage_logging_obj() + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _json_response("GET", {}) + client.post.return_value = _json_response("POST", _CREATED_CACHE) + + _, _, cache_name = ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(logging_obj, client)) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert logging_obj.context_cache_storage_token_hours == pytest.approx(7200 * 0.5) + + +@pytest.mark.asyncio +async def test_async_creating_a_cache_records_its_storage_token_hours(local_model_cost_map: None) -> None: + logging_obj: Final = _storage_logging_obj() + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_json_response("GET", {})) + client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE)) + + _, _, cache_name = await ContextCachingEndpoints().async_check_and_create_cache( + **_create_kwargs(logging_obj, client) + ) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert logging_obj.context_cache_storage_token_hours == pytest.approx(7200 * 0.5) + + +def _listing_with(created_body: Mapping[str, object]) -> Mapping[str, object]: + return {"cachedContents": [{"name": _STORAGE_TEST_CACHE_NAME, "displayName": created_body["displayName"]}]} + + +def test_reusing_an_existing_cache_records_no_storage(local_model_cost_map: None) -> None: + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _json_response("GET", {}) + client.post.return_value = _json_response("POST", _CREATED_CACHE) + ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(_storage_logging_obj(), client)) + client.get.return_value = _json_response("GET", _listing_with(client.post.call_args.kwargs["json"])) + reusing_logging_obj: Final = _storage_logging_obj() + + _, _, cache_name = ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(reusing_logging_obj, client)) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert client.post.call_count == 1 + assert reusing_logging_obj.context_cache_storage_token_hours == 0.0 + + +@pytest.mark.asyncio +async def test_async_reusing_an_existing_cache_records_no_storage(local_model_cost_map: None) -> None: + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_json_response("GET", {})) + client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE)) + await ContextCachingEndpoints().async_check_and_create_cache(**_create_kwargs(_storage_logging_obj(), client)) + client.get.return_value = _json_response("GET", _listing_with(client.post.call_args.kwargs["json"])) + reusing_logging_obj: Final = _storage_logging_obj() + + _, _, cache_name = await ContextCachingEndpoints().async_check_and_create_cache( + **_create_kwargs(reusing_logging_obj, client) + ) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert client.post.call_count == 1 + assert reusing_logging_obj.context_cache_storage_token_hours == 0.0 + + +_STORAGE_TEST_CREATED_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc).timestamp() + + +def _remembering_endpoints() -> ContextCachingEndpoints: + return ContextCachingEndpoints(known_caches=KnownCachedContents(clock=lambda: _STORAGE_TEST_CREATED_AT)) + + +def test_reusing_a_remembered_cache_records_no_storage(local_model_cost_map: None) -> None: + endpoints: Final = _remembering_endpoints() + client: Final = MagicMock(spec=HTTPHandler) + client.get.return_value = _json_response("GET", {}) + client.post.return_value = _json_response("POST", _CREATED_CACHE) + creating_logging_obj: Final = _storage_logging_obj() + endpoints.check_and_create_cache(**_create_kwargs(creating_logging_obj, client)) + reusing_logging_obj: Final = _storage_logging_obj() + + _, _, cache_name = endpoints.check_and_create_cache(**_create_kwargs(reusing_logging_obj, client)) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert (client.get.call_count, client.post.call_count) == (1, 1) + assert creating_logging_obj.context_cache_storage_token_hours == 7200 * 0.5 + assert reusing_logging_obj.context_cache_storage_token_hours == 0.0 + + +@pytest.mark.asyncio +async def test_async_reusing_a_remembered_cache_records_no_storage(local_model_cost_map: None) -> None: + endpoints: Final = _remembering_endpoints() + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.get = AsyncMock(return_value=_json_response("GET", {})) + client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE)) + creating_logging_obj: Final = _storage_logging_obj() + await endpoints.async_check_and_create_cache(**_create_kwargs(creating_logging_obj, client)) + reusing_logging_obj: Final = _storage_logging_obj() + + _, _, cache_name = await endpoints.async_check_and_create_cache(**_create_kwargs(reusing_logging_obj, client)) + + assert cache_name == _STORAGE_TEST_CACHE_NAME + assert (client.get.await_count, client.post.await_count) == (1, 1) + assert creating_logging_obj.context_cache_storage_token_hours == 7200 * 0.5 + assert reusing_logging_obj.context_cache_storage_token_hours == 0.0 + + +@pytest.mark.asyncio +async def test_concurrent_requests_for_one_prefix_record_storage_once(local_model_cost_map: None) -> None: + endpoints: Final = _remembering_endpoints() + client: Final = MagicMock(spec=AsyncHTTPHandler) + + async def slow_list(**_: object) -> httpx.Response: + await asyncio.sleep(0.01) + return _json_response("GET", {}) + + client.get = AsyncMock(side_effect=slow_list) + client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE)) + logging_objs: Final = [_storage_logging_obj() for _ in range(3)] + + results: Final = await asyncio.gather( + *(endpoints.async_check_and_create_cache(**_create_kwargs(logging_obj, client)) for logging_obj in logging_objs) + ) + + assert {cache_name for _, _, cache_name in results} == {_STORAGE_TEST_CACHE_NAME} + assert client.post.await_count == 1 + recorded: Final = sorted(logging_obj.context_cache_storage_token_hours for logging_obj in logging_objs) + assert recorded == [0.0, 0.0, 7200 * 0.5] + + def _created_cache_body(total_token_count: int) -> dict: return { "name": "cachedContents/created", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 49d51e99790..7fc7fe03e77 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1,6 +1,7 @@ import json import asyncio import datetime +import math import time from collections.abc import Iterator, Mapping from types import MappingProxyType, SimpleNamespace @@ -7272,6 +7273,211 @@ def test_completion_cost_bills_base_when_gemini_serves_on_demand( assert cost == pytest.approx(100 * 0.001 + 50 * 0.002) +_STORAGE_RATE_KEY: Final = "cache_storage_cost_per_token_per_hour" + + +_FIXED_START_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + +@pytest.fixture +def _storage_cost_map(_local_model_cost_map: None) -> Iterator[None]: + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _first_chat_model(provider_prefix: str, *, with_storage_rate: bool) -> str: + names: Final = [name for name in litellm.model_cost if isinstance(name, str)] + return next( + name + for name in names + if _is_priced_chat_entry(_cost_map_entry(name), provider_prefix, with_storage_rate=with_storage_rate) + ) + + +def _cost_map_entry(name: str) -> object: + entry: Final[object] = litellm.model_cost.get(name) + return entry + + +def _is_priced_chat_entry(entry: object, provider_prefix: str, *, with_storage_rate: bool) -> bool: + return ( + isinstance(entry, dict) + and entry.get("mode") == "chat" + and isinstance(provider := entry.get("litellm_provider"), str) + and provider.startswith(provider_prefix) + and entry.get("input_cost_per_token") is not None + and (_STORAGE_RATE_KEY in entry) == with_storage_rate + ) + + +def _cache_storage_logging_obj(model: str, token_hours: float) -> Logging: + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=_FIXED_START_TIME, + litellm_call_id="context-cache-storage", + function_id="f", + ) + if token_hours: + logging_obj.record_context_cache_storage(token_hours) + return logging_obj + + +def _chat_cost(model: str, logging_obj: Logging, **kwargs: object) -> float: + usage: Final = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) + return completion_cost( + completion_response=ModelResponse(model=model, usage=usage), + model=model, + litellm_logging_obj=logging_obj, + **kwargs, + ) + + +def test_context_cache_storage_is_billed_on_the_creating_request(_storage_cost_map: None) -> None: + model: Final = _first_chat_model("vertex_ai", with_storage_rate=True) + rate: Final = litellm.get_model_info(model)[_STORAGE_RATE_KEY] + creating_request: Final = _cache_storage_logging_obj(model, token_hours=3_600.0) + + with_storage: Final = _chat_cost(model, creating_request) + without_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0)) + + assert rate is not None and rate > 0 + assert with_storage - without_storage == pytest.approx(3_600.0 * rate) + assert creating_request.cost_breakdown["additional_costs"] == pytest.approx({"cache_storage_cost": 3_600.0 * rate}) + assert creating_request.cost_breakdown["total_cost"] == pytest.approx(with_storage) + + +def test_context_cache_storage_costs_nothing_for_models_without_a_storage_rate(_storage_cost_map: None) -> None: + model: Final = _first_chat_model("gemini", with_storage_rate=False) + creating_request: Final = _cache_storage_logging_obj(model, token_hours=3_600.0) + + with_storage: Final = _chat_cost(model, creating_request) + + assert with_storage == _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0)) + assert creating_request.cost_breakdown.get("additional_costs") is None + + +def test_context_cache_storage_uses_the_deployment_storage_rate(_storage_cost_map: None) -> None: + model: Final = _first_chat_model("vertex_ai", with_storage_rate=True) + deployment_rate: Final = litellm.get_model_info(model)[_STORAGE_RATE_KEY] * 3 + litellm.register_model( + { + "storage-cost-deployment": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + _STORAGE_RATE_KEY: deployment_rate, + } + } + ) + pricing: Final = {"custom_pricing": True, "router_model_id": "storage-cost-deployment"} + + with_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=500.0), **pricing) + without_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0), **pricing) + + assert with_storage - without_storage == pytest.approx(500.0 * deployment_rate) + + +def test_context_cache_storage_joins_the_provider_additional_costs(_storage_cost_map: None) -> None: + litellm.register_model( + {"router-storage-deployment": {"litellm_provider": "azure_ai", "mode": "chat", _STORAGE_RATE_KEY: 1e-6}} + ) + logging_obj: Final = _cache_storage_logging_obj("azure_ai/model-router", token_hours=500.0) + usage: Final = Usage(prompt_tokens=1_000_000, completion_tokens=100, total_tokens=1_000_100) + + cost: Final = completion_cost( + completion_response=ModelResponse(model="azure_ai/gpt-4o-2024-08-06", usage=usage), + model="azure_ai/model-router", + custom_llm_provider="azure_ai", + litellm_logging_obj=logging_obj, + custom_pricing=True, + router_model_id="router-storage-deployment", + ) + + breakdown: Final = logging_obj.cost_breakdown + assert breakdown is not None + additional_costs: Final = breakdown.get("additional_costs") or {} + assert additional_costs["Azure Model Router Flat Cost"] > 0 + assert math.isclose(additional_costs["cache_storage_cost"], 500.0 * 1e-6) + assert math.isclose(cost, sum(additional_costs.values())) + + +_STORAGE_DEPLOYMENT_MODEL: Final = "gemini-2.5-flash" + + +def _routed_request_cost( + token_prices: Mapping[str, float], token_hours: float, *, storage_rate: float | None = None +) -> float | None: + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "storage-cost-deployment", + "litellm_params": {"model": f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}", **token_prices}, + "model_info": {} if storage_rate is None else {_STORAGE_RATE_KEY: storage_rate}, + } + ] + ) + deployment_id: Final = router.model_list[0]["model_info"]["id"] + assert isinstance(deployment_id, str) + return _vertex_request_cost({**token_prices, "metadata": {"model_info": {"id": deployment_id}}}, token_hours) + + +def _vertex_request_cost(litellm_params: Mapping[str, object], token_hours: float) -> float | None: + logging_obj: Final = _cache_storage_logging_obj(_STORAGE_DEPLOYMENT_MODEL, token_hours) + logging_obj.update_environment_variables( + model=_STORAGE_DEPLOYMENT_MODEL, + litellm_params=dict(litellm_params), + optional_params={}, + custom_llm_provider="vertex_ai", + ) + usage: Final = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) + return logging_obj.response_cost_calculator(ModelResponse(model=_STORAGE_DEPLOYMENT_MODEL, usage=usage)) + + +def test_custom_token_prices_without_a_storage_rate_bill_storage_at_the_underlying_model_rate( + _storage_cost_map: None, +) -> None: + cost_map_rate: Final = litellm.get_model_info(f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}")[_STORAGE_RATE_KEY] + token_prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} + + with_storage: Final = _routed_request_cost(token_prices, token_hours=500.0) + without_storage: Final = _routed_request_cost(token_prices, token_hours=0.0) + + assert cost_map_rate is not None and cost_map_rate > 0 + assert without_storage == pytest.approx(1_000 * 1e-6 + 100 * 2e-6) + assert with_storage == pytest.approx(without_storage + 500.0 * cost_map_rate) + + +def test_a_storage_only_override_keeps_the_underlying_token_prices(_storage_cost_map: None) -> None: + cost_map_rate: Final = litellm.get_model_info(f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}")[_STORAGE_RATE_KEY] + assert isinstance(cost_map_rate, float) + deployment_rate: Final = cost_map_rate * 3 + + no_override: Final = _vertex_request_cost({}, token_hours=0.0) + without_storage: Final = _routed_request_cost({}, token_hours=0.0, storage_rate=deployment_rate) + with_storage: Final = _routed_request_cost({}, token_hours=500.0, storage_rate=deployment_rate) + + assert no_override is not None and no_override > 0 + assert without_storage == pytest.approx(no_override) + assert with_storage == pytest.approx(no_override + 500.0 * deployment_rate) + + +def test_a_deployment_storage_rate_stays_off_the_shared_backend_key(_storage_cost_map: None) -> None: + backend_key: Final = f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}" + cost_map_rate: Final = litellm.get_model_info(backend_key)[_STORAGE_RATE_KEY] + assert isinstance(cost_map_rate, float) + + _routed_request_cost({}, token_hours=0.0, storage_rate=cost_map_rate * 3) + + assert litellm.get_model_info(backend_key)[_STORAGE_RATE_KEY] == cost_map_rate + + def test_an_unpriced_provider_slug_does_not_bill_the_turn_at_zero( _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index c0c8a0655fe..08f4fb2dac6 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -1484,13 +1484,17 @@ def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields(): """ import typing - from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfoBase + from litellm.types.utils import ( + DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS, + CustomPricingLiteLLMParams, + ModelInfoBase, + ) pricing_markers = ("cost", "price", "uplift", "vector_size", "tiered_pricing") builtin_pricing_fields = { name for name in typing.get_type_hints(ModelInfoBase) if any(marker in name for marker in pricing_markers) } - denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) + denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) | DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS uncovered = sorted(builtin_pricing_fields - denylisted_fields) assert not uncovered, ( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c41c4db2c02..9f4f52bfffe 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1002,6 +1002,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost_flex": {"type": "number"}, "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_creation_input_token_cost_ultrafast": {"type": "number"}, + "cache_storage_cost_per_token_per_hour": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, "explicit_cache_read_input_token_cost": {"type": "number"}, "cache_read_input_token_cost_above_32k_tokens": {"type": "number"}, @@ -9148,6 +9149,26 @@ async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: assert counting_logger.logged_results[0] is inner_result +def test_get_model_info_surfaces_context_cache_storage_rate(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + local_cost_map: Final[Mapping[str, object]] = litellm.get_model_cost_map(url="") + monkeypatch.setattr(litellm, "model_cost", local_cost_map) + litellm.get_model_info.cache_clear() + priced: Final = { + name: entry["cache_storage_cost_per_token_per_hour"] + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) and "cache_storage_cost_per_token_per_hour" in entry + } + + surfaced: Final = { + name: litellm.get_model_info(name).get("cache_storage_cost_per_token_per_hour") for name in priced + } + litellm.get_model_info.cache_clear() + + assert priced + assert surfaced == priced + + def test_basic_trimming(): litellm.turn_on_debug() messages = [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2272b4bea43..6780266a61e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -52224,6 +52224,8 @@ export interface components { cache_creation_input_token_cost_above_1hr?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; + /** Cache Storage Cost Per Token Per Hour */ + cache_storage_cost_per_token_per_hour?: number | null; /** Cost Per Ptu Per Hour */ cost_per_ptu_per_hour?: number | null; /** Created At */