mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(gemini): account for context cache storage costs
This commit is contained in:
tomatomptt
parent
f2118fc767
commit
1d8f42f71b
9 changed files with 390 additions and 9 deletions
|
|
@ -2075,6 +2075,17 @@ def completion_cost(
|
|||
)
|
||||
else:
|
||||
additional_costs = None
|
||||
if custom_llm_provider in ("gemini", "vertex_ai", "vertex_ai_beta"):
|
||||
from litellm.llms.vertex_ai.context_caching.storage_pricing import (
|
||||
recorded_cache_storage_cost,
|
||||
)
|
||||
|
||||
cache_storage_cost = recorded_cache_storage_cost(litellm_logging_obj)
|
||||
if cache_storage_cost:
|
||||
additional_costs = {
|
||||
**(additional_costs or {}),
|
||||
"gemini_context_cache_storage": cache_storage_cost,
|
||||
}
|
||||
|
||||
_final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
|
||||
cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
|
|
|
|||
95
litellm/llms/vertex_ai/context_caching/storage_pricing.py
Normal file
95
litellm/llms/vertex_ai/context_caching/storage_pricing.py
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
"""
|
||||
Storage pricing for Gemini / Vertex AI explicit context caches.
|
||||
|
||||
Google bills a cachedContents object twice: once for the cached tokens when it is created, and then for storage,
|
||||
per cached token per hour, until it expires. LiteLLM creates the cache on the caller's behalf from a client
|
||||
``cache_control`` marker, takes the TTL from the client, and never deletes the cache, so the storage Google bills is
|
||||
``cached tokens x lifetime hours x storage rate`` where the lifetime is the cache's ``expireTime - createTime`` (the
|
||||
requested TTL, or Google's 1 hour default).
|
||||
|
||||
* ``cache_storage_cost_per_token_per_hour`` in the model cost map is the storage rate.
|
||||
* ``gemini_context_cache_max_ttl_seconds`` (litellm setting, default 3600) caps the TTL a client may request; a
|
||||
larger value is lowered to the cap before the cache is created.
|
||||
* A model without a storage rate gets no explicit cache (the request is sent uncached), so storage is never bought
|
||||
without a price.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
||||
CACHE_STORAGE_PRICE_KEY: Final = "cache_storage_cost_per_token_per_hour"
|
||||
GOOGLE_DEFAULT_CACHE_TTL_SECONDS: Final = 3600.0
|
||||
DEFAULT_MAX_CACHE_TTL_SECONDS: Final = 3600.0
|
||||
|
||||
|
||||
def max_cache_ttl_seconds() -> float:
|
||||
configured: Final[object] = getattr(litellm, "gemini_context_cache_max_ttl_seconds", None)
|
||||
if isinstance(configured, (int, float)) and not isinstance(configured, bool) and configured > 0:
|
||||
return float(configured)
|
||||
return DEFAULT_MAX_CACHE_TTL_SECONDS
|
||||
|
||||
|
||||
def cap_cache_ttl(ttl: str | None) -> str | None:
|
||||
"""Lower a normalized ``"<seconds>s"`` TTL to the configured maximum. ``None`` keeps Google's 1 hour default
|
||||
unless the cap is shorter than that."""
|
||||
cap: Final = max_cache_ttl_seconds()
|
||||
seconds: Final = float(ttl[:-1]) if ttl else GOOGLE_DEFAULT_CACHE_TTL_SECONDS
|
||||
if seconds <= cap:
|
||||
return ttl
|
||||
return f"{cap:.9f}".rstrip("0").rstrip(".") + "s"
|
||||
|
||||
|
||||
def cache_storage_price(model: str, custom_llm_provider: str) -> float | None:
|
||||
"""Storage rate (USD per cached token per hour) from the model cost map, or None when the model has none."""
|
||||
provider_prefixed: Final = f"{custom_llm_provider}/{model}"
|
||||
candidates: Final = (provider_prefixed, model) if custom_llm_provider == "gemini" else (model, provider_prefixed)
|
||||
for key in candidates:
|
||||
entry = litellm.model_cost.get(key)
|
||||
if isinstance(entry, dict):
|
||||
price = entry.get(CACHE_STORAGE_PRICE_KEY)
|
||||
if isinstance(price, (int, float)) and not isinstance(price, bool) and price >= 0:
|
||||
return float(price)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_time(value: object) -> datetime | None:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def cache_lifetime_hours(created: dict, requested_ttl: str | None) -> float:
|
||||
"""Lifetime Google will bill storage for: the created cache's expireTime - createTime, else the requested TTL,
|
||||
else Google's 1 hour default."""
|
||||
create_time: Final = _parse_time(created.get("createTime"))
|
||||
expire_time: Final = _parse_time(created.get("expireTime"))
|
||||
if create_time is not None and expire_time is not None and expire_time > create_time:
|
||||
return (expire_time - create_time).total_seconds() / 3600.0
|
||||
if requested_ttl:
|
||||
try:
|
||||
return float(requested_ttl[:-1]) / 3600.0
|
||||
except ValueError:
|
||||
pass
|
||||
return GOOGLE_DEFAULT_CACHE_TTL_SECONDS / 3600.0
|
||||
|
||||
|
||||
def record_cache_storage(
|
||||
logging_obj: object, created: dict, requested_ttl: str | None, model: str, custom_llm_provider: str
|
||||
) -> None:
|
||||
"""Store the storage cost of a cache this request created on the logging object; the cost calculator adds it."""
|
||||
tokens: Final = (created.get("usageMetadata") or {}).get("totalTokenCount") or 0
|
||||
price: Final = cache_storage_price(model, custom_llm_provider) or 0.0
|
||||
hours: Final = cache_lifetime_hours(created, requested_ttl)
|
||||
cost: Final = max(tokens, 0) * hours * price
|
||||
logging_obj.vertex_ai_cache_storage_token_hours = max(tokens, 0) * hours # type: ignore[attr-defined] # rebind-ok
|
||||
logging_obj.vertex_ai_cache_storage_cost = cost # type: ignore[attr-defined] # rebind-ok
|
||||
|
||||
|
||||
def recorded_cache_storage_cost(logging_obj: object | None) -> float:
|
||||
recorded: Final[object] = getattr(logging_obj, "vertex_ai_cache_storage_cost", 0.0) if logging_obj else 0.0
|
||||
return float(recorded) if isinstance(recorded, (int, float)) and recorded > 0 else 0.0
|
||||
|
|
@ -15,6 +15,7 @@ from litellm.types.llms.vertex_ai import CachedContentRequestBody
|
|||
from litellm.utils import is_cached_message
|
||||
|
||||
from ..common_utils import get_supports_system_message
|
||||
from .storage_pricing import cap_cache_ttl
|
||||
from ..gemini.transformation import (
|
||||
gemini_convert_messages_with_history,
|
||||
transform_system_message,
|
||||
|
|
@ -167,7 +168,8 @@ def transform_openai_messages_to_gemini_context_caching(
|
|||
vertex_location: str | None,
|
||||
) -> CachedContentRequestBody:
|
||||
# Extract TTL from cached messages BEFORE system message transformation
|
||||
ttl: Final = extract_ttl_from_cached_messages(messages)
|
||||
# Google bills storage per token-hour for the whole TTL; keep the client's TTL within the operator's cap.
|
||||
ttl: Final = cap_cache_ttl(extract_ttl_from_cached_messages(messages))
|
||||
|
||||
supports_system_message: Final = get_supports_system_message(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.utils import is_prompt_caching_valid_prompt
|
|||
from ..common_utils import VertexAIError, get_vertex_base_url
|
||||
from ..vertex_llm_base import VertexBase
|
||||
from .known_cached_contents import KnownCachedContents, known_cached_contents
|
||||
from .storage_pricing import cache_storage_price, record_cache_storage
|
||||
from .transformation import (
|
||||
cached_messages_end_on_supported_turn,
|
||||
separate_cached_messages,
|
||||
|
|
@ -53,6 +54,7 @@ class _CachedContentUsageMetadata(TypedDict):
|
|||
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
|
||||
class _CreatedCachedContent(TypedDict):
|
||||
name: ReadOnly[NotRequired[str | None]]
|
||||
createTime: ReadOnly[NotRequired[str | None]]
|
||||
expireTime: ReadOnly[NotRequired[str | None]]
|
||||
usageMetadata: ReadOnly[NotRequired[_CachedContentUsageMetadata | None]]
|
||||
|
||||
|
|
@ -393,6 +395,14 @@ class ContextCachingEndpoints(VertexBase):
|
|||
)
|
||||
return messages, optional_params, None
|
||||
|
||||
# Storage is billed per token-hour for the cache's lifetime; never buy it without a price.
|
||||
if cache_storage_price(model, custom_llm_provider) is None:
|
||||
verbose_logger.debug(
|
||||
"Vertex AI context caching: no cache_storage_cost_per_token_per_hour for %s. Skipping context caching.",
|
||||
model,
|
||||
)
|
||||
return messages, optional_params, None
|
||||
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
|
|
@ -487,6 +497,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
|
||||
raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root
|
||||
_record_cache_creation_tokens(logging_obj, raw_response_cached)
|
||||
record_cache_storage(
|
||||
logging_obj, raw_response_cached, cached_content_request_body.get("ttl"), model, custom_llm_provider
|
||||
)
|
||||
self._known_caches.remember(
|
||||
KnownCachedContents.scope(
|
||||
url, custom_llm_provider, headers, vertex_project=vertex_project, vertex_location=vertex_location
|
||||
|
|
@ -559,6 +572,14 @@ class ContextCachingEndpoints(VertexBase):
|
|||
)
|
||||
return messages, optional_params, None
|
||||
|
||||
# Storage is billed per token-hour for the cache's lifetime; never buy it without a price.
|
||||
if cache_storage_price(model, custom_llm_provider) is None:
|
||||
verbose_logger.debug(
|
||||
"Vertex AI context caching: no cache_storage_cost_per_token_per_hour for %s. Skipping context caching.",
|
||||
model,
|
||||
)
|
||||
return messages, optional_params, None
|
||||
|
||||
tools: Final = optional_params.pop("tools", None)
|
||||
tool_choice: Final = optional_params.pop("tool_choice", None)
|
||||
|
||||
|
|
@ -654,6 +675,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
|
||||
raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root
|
||||
_record_cache_creation_tokens(logging_obj, raw_response_cached)
|
||||
record_cache_storage(
|
||||
logging_obj, raw_response_cached, cached_content_request_body.get("ttl"), model, custom_llm_provider
|
||||
)
|
||||
self._known_caches.remember(
|
||||
scope, generated_cache_key, raw_response_cached.get("name"), raw_response_cached.get("expireTime")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28927,6 +28927,7 @@
|
|||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
|
|
@ -29285,6 +29286,7 @@
|
|||
},
|
||||
"gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -29337,6 +29339,7 @@
|
|||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -29399,6 +29402,7 @@
|
|||
"gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
|
|
@ -29496,6 +29500,7 @@
|
|||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
|
|
@ -29553,6 +29558,7 @@
|
|||
},
|
||||
"gemini-2.5-flash-lite-preview-09-2025": {
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -29600,6 +29606,7 @@
|
|||
},
|
||||
"gemini-2.5-flash-preview-09-2025": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -29784,6 +29791,7 @@
|
|||
"gemini-2.5-pro": {
|
||||
"deprecation_date": "2026-10-20",
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07,
|
||||
|
|
@ -29896,11 +29904,13 @@
|
|||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"web_search_billing_unit": "per_query"
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06
|
||||
},
|
||||
"gemini-3.1-pro-preview-customtools": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
|
|
@ -30882,6 +30892,7 @@
|
|||
"gemini/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -31042,6 +31053,7 @@
|
|||
"gemini/gemini-2.5-flash": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
|
|
@ -31371,6 +31383,7 @@
|
|||
"gemini/gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
|
|
@ -31572,6 +31585,7 @@
|
|||
},
|
||||
"gemini/gemini-2.5-pro": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07,
|
||||
"cache_read_input_token_cost_batches": 1.25e-07,
|
||||
|
|
@ -31665,6 +31679,7 @@
|
|||
"gemini/gemini-3.1-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -31729,6 +31744,7 @@
|
|||
},
|
||||
"gemini/gemini-3.5-flash-lite": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 2e-08,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"cache_read_input_token_cost_priority": 5e-08,
|
||||
|
|
@ -31791,6 +31807,7 @@
|
|||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 5e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
|
|
@ -31854,6 +31871,7 @@
|
|||
"gemini/gemini-3.5-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
|
|
@ -31917,6 +31935,7 @@
|
|||
"gemini/gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -31979,6 +31998,7 @@
|
|||
"gemini/gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32042,6 +32062,7 @@
|
|||
"gemini/gemini-3.8-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32139,6 +32160,7 @@
|
|||
"gemini/gemini-3.1-pro-preview": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4e-06,
|
||||
|
|
@ -32203,6 +32225,7 @@
|
|||
"gemini/gemini-3.1-pro-preview-customtools": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4e-06,
|
||||
|
|
@ -32316,7 +32339,8 @@
|
|||
"supports_url_context": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"web_search_billing_unit": "per_query"
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"gemini-omni-flash-preview": {
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
|
|
@ -32353,6 +32377,7 @@
|
|||
"prompt_cache_min_tokens": 4096,
|
||||
"deprecation_date": "2027-05-19",
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
|
|
@ -32414,6 +32439,7 @@
|
|||
"gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32475,6 +32501,7 @@
|
|||
"gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32589,10 +32616,12 @@
|
|||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"web_search_billing_unit": "per_query"
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"gemini-3.8-flash-cyber": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
|
|
@ -52486,6 +52515,7 @@
|
|||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -52538,6 +52568,7 @@
|
|||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -52601,6 +52632,7 @@
|
|||
"vertex_ai/gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
|
|
|
|||
|
|
@ -28927,6 +28927,7 @@
|
|||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
|
|
@ -29285,6 +29286,7 @@
|
|||
},
|
||||
"gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -29337,6 +29339,7 @@
|
|||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -29399,6 +29402,7 @@
|
|||
"gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
|
|
@ -29496,6 +29500,7 @@
|
|||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
|
|
@ -29553,6 +29558,7 @@
|
|||
},
|
||||
"gemini-2.5-flash-lite-preview-09-2025": {
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -29600,6 +29606,7 @@
|
|||
},
|
||||
"gemini-2.5-flash-preview-09-2025": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -29784,6 +29791,7 @@
|
|||
"gemini-2.5-pro": {
|
||||
"deprecation_date": "2026-10-20",
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07,
|
||||
|
|
@ -29896,11 +29904,13 @@
|
|||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"web_search_billing_unit": "per_query"
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06
|
||||
},
|
||||
"gemini-3.1-pro-preview-customtools": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
|
|
@ -30882,6 +30892,7 @@
|
|||
"gemini/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -31042,6 +31053,7 @@
|
|||
"gemini/gemini-2.5-flash": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
|
|
@ -31371,6 +31383,7 @@
|
|||
"gemini/gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
|
|
@ -31572,6 +31585,7 @@
|
|||
},
|
||||
"gemini/gemini-2.5-pro": {
|
||||
"cache_read_input_token_cost": 1.25e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07,
|
||||
"cache_read_input_token_cost_batches": 1.25e-07,
|
||||
|
|
@ -31665,6 +31679,7 @@
|
|||
"gemini/gemini-3.1-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -31729,6 +31744,7 @@
|
|||
},
|
||||
"gemini/gemini-3.5-flash-lite": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 2e-08,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"cache_read_input_token_cost_priority": 5e-08,
|
||||
|
|
@ -31791,6 +31807,7 @@
|
|||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 5e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
|
|
@ -31854,6 +31871,7 @@
|
|||
"gemini/gemini-3.5-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
|
|
@ -31917,6 +31935,7 @@
|
|||
"gemini/gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -31979,6 +31998,7 @@
|
|||
"gemini/gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32042,6 +32062,7 @@
|
|||
"gemini/gemini-3.8-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32139,6 +32160,7 @@
|
|||
"gemini/gemini-3.1-pro-preview": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4e-06,
|
||||
|
|
@ -32203,6 +32225,7 @@
|
|||
"gemini/gemini-3.1-pro-preview-customtools": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4e-06,
|
||||
|
|
@ -32316,7 +32339,8 @@
|
|||
"supports_url_context": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"web_search_billing_unit": "per_query"
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"gemini-omni-flash-preview": {
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
|
|
@ -32353,6 +32377,7 @@
|
|||
"prompt_cache_min_tokens": 4096,
|
||||
"deprecation_date": "2027-05-19",
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"input_cost_per_audio_token": 1.5e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
|
|
@ -32414,6 +32439,7 @@
|
|||
"gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32475,6 +32501,7 @@
|
|||
"gemini-3.7-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
|
|
@ -32589,10 +32616,12 @@
|
|||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"web_search_billing_unit": "per_query"
|
||||
"web_search_billing_unit": "per_query",
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"gemini-3.8-flash-cyber": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
|
|
@ -52486,6 +52515,7 @@
|
|||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -52538,6 +52568,7 @@
|
|||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
|
|
@ -52601,6 +52632,7 @@
|
|||
"vertex_ai/gemini-3.5-flash-lite": {
|
||||
"deprecation_date": "2027-07-21",
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1.0e-06,
|
||||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,174 @@
|
|||
"""Storage pricing for Gemini explicit context caches created from a client ``cache_control`` marker.
|
||||
|
||||
Google bills a cachedContents object for storage per cached token per hour until it expires; LiteLLM picks the TTL
|
||||
from the client and never deletes the cache. The spend LiteLLM records for the request that created the cache must
|
||||
include that storage, and the TTL a client can buy must be bounded by the operator.
|
||||
"""
|
||||
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.vertex_ai.context_caching.storage_pricing import (
|
||||
cache_lifetime_hours,
|
||||
cache_storage_price,
|
||||
cap_cache_ttl,
|
||||
recorded_cache_storage_cost,
|
||||
)
|
||||
from litellm.llms.vertex_ai.context_caching.transformation import (
|
||||
transform_openai_messages_to_gemini_context_caching,
|
||||
)
|
||||
from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import ContextCachingEndpoints
|
||||
|
||||
PRO_RATE = 4.5e-06 # USD per cached token per hour, Gemini 2.5 Pro (Vertex AI pricing, context cache storage)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def local_cost_map(monkeypatch):
|
||||
original = litellm.model_cost
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
litellm.get_model_info.cache_clear()
|
||||
monkeypatch.delattr(litellm, "gemini_context_cache_max_ttl_seconds", raising=False)
|
||||
yield
|
||||
litellm.model_cost = original
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _messages(ttl: str | None) -> list:
|
||||
control = {"type": "ephemeral"} | ({"ttl": ttl} if ttl else {})
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "long prefix " * 400, "cache_control": control}]},
|
||||
{"role": "user", "content": "question"},
|
||||
]
|
||||
|
||||
|
||||
def test_storage_rate_is_in_the_cost_map():
|
||||
assert cache_storage_price("gemini-2.5-pro", "gemini") == PRO_RATE
|
||||
assert cache_storage_price("gemini-2.5-pro", "vertex_ai") == PRO_RATE
|
||||
assert cache_storage_price("gemini-2.5-flash", "gemini") == 1e-06
|
||||
assert cache_storage_price("no-such-model", "gemini") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requested, cap, expected",
|
||||
[
|
||||
("31536000s", None, "3600s"),
|
||||
("600s", None, "600s"),
|
||||
(None, None, None),
|
||||
("31536000s", 86400, "86400s"),
|
||||
(None, 1800, "1800s"),
|
||||
],
|
||||
)
|
||||
def test_client_ttl_is_capped(monkeypatch, requested, cap, expected):
|
||||
if cap is not None:
|
||||
monkeypatch.setattr(litellm, "gemini_context_cache_max_ttl_seconds", cap, raising=False)
|
||||
assert cap_cache_ttl(requested) == expected
|
||||
|
||||
|
||||
def test_transformed_cache_request_carries_the_capped_ttl():
|
||||
body = transform_openai_messages_to_gemini_context_caching(
|
||||
model="gemini-2.5-pro",
|
||||
messages=_messages("8760h")[:1],
|
||||
custom_llm_provider="gemini",
|
||||
cache_key="k",
|
||||
vertex_project=None,
|
||||
vertex_location=None,
|
||||
)
|
||||
assert body["ttl"] == "3600s"
|
||||
|
||||
|
||||
def test_lifetime_comes_from_the_created_cache():
|
||||
created = {"createTime": "2026-10-14T00:00:00Z", "expireTime": "2026-10-14T05:00:00Z"}
|
||||
assert cache_lifetime_hours(created, "60s") == 5.0
|
||||
assert cache_lifetime_hours({}, "7200s") == 2.0
|
||||
assert cache_lifetime_hours({}, None) == 1.0
|
||||
|
||||
|
||||
def _created_response(tokens: int, hours: float) -> httpx.Response:
|
||||
created = datetime(2026, 10, 14, tzinfo=timezone.utc)
|
||||
expires = datetime.fromtimestamp(created.timestamp() + hours * 3600, timezone.utc)
|
||||
body = {
|
||||
"name": "cachedContents/abc",
|
||||
"usageMetadata": {"totalTokenCount": tokens},
|
||||
"createTime": created.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"expireTime": expires.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
}
|
||||
return httpx.Response(200, json=body, request=httpx.Request("POST", "https://x/cachedContents"))
|
||||
|
||||
|
||||
def _endpoints_call(model: str, client: MagicMock, logging_obj: Logging, ttl: str | None):
|
||||
with patch(
|
||||
"litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.is_prompt_caching_valid_prompt",
|
||||
return_value=True,
|
||||
):
|
||||
return ContextCachingEndpoints().check_and_create_cache(
|
||||
messages=_messages(ttl),
|
||||
optional_params={},
|
||||
api_key="k",
|
||||
api_base=None,
|
||||
model=model,
|
||||
client=client,
|
||||
timeout=10,
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider="gemini",
|
||||
vertex_project=None,
|
||||
vertex_location=None,
|
||||
vertex_auth_header=None,
|
||||
)
|
||||
|
||||
|
||||
def _logging_obj() -> Logging:
|
||||
return Logging(
|
||||
model="gemini/gemini-2.5-pro",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=datetime.now(timezone.utc),
|
||||
litellm_call_id="c",
|
||||
function_id="f",
|
||||
)
|
||||
|
||||
|
||||
def test_created_cache_storage_is_recorded_and_priced():
|
||||
client = MagicMock(spec=HTTPHandler)
|
||||
client.get.return_value = httpx.Response(200, json={}, request=httpx.Request("GET", "https://x"))
|
||||
client.post.return_value = _created_response(tokens=2000, hours=1.0)
|
||||
logging_obj = _logging_obj()
|
||||
_, _, name = _endpoints_call("gemini-2.5-pro", client, logging_obj, "8760h")
|
||||
assert name == "cachedContents/abc"
|
||||
assert client.post.call_args.kwargs["json"]["ttl"] == "3600s"
|
||||
assert math.isclose(recorded_cache_storage_cost(logging_obj), 2000 * 1.0 * PRO_RATE, rel_tol=1e-9)
|
||||
|
||||
response = ModelResponse(
|
||||
model="gemini-2.5-pro",
|
||||
usage=Usage(prompt_tokens=2100, completion_tokens=10, total_tokens=2110, cache_read_input_tokens=2000),
|
||||
)
|
||||
with_storage = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="gemini/gemini-2.5-pro",
|
||||
custom_llm_provider="gemini",
|
||||
litellm_logging_obj=logging_obj,
|
||||
)
|
||||
without = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="gemini/gemini-2.5-pro",
|
||||
custom_llm_provider="gemini",
|
||||
litellm_logging_obj=_logging_obj(),
|
||||
)
|
||||
assert math.isclose(with_storage - without, 2000 * PRO_RATE, rel_tol=1e-6)
|
||||
|
||||
|
||||
def test_model_without_storage_price_gets_no_explicit_cache():
|
||||
client = MagicMock(spec=HTTPHandler)
|
||||
messages, _, name = _endpoints_call("gemini-unpriced-test-model", client, _logging_obj(), "8760h")
|
||||
assert name is None
|
||||
client.post.assert_not_called()
|
||||
assert len(messages) == 2
|
||||
|
|
@ -339,8 +339,11 @@ class TestTransformationWithTTL:
|
|||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]
|
||||
)
|
||||
def test_transform_with_system_message_and_ttl(self, custom_llm_provider):
|
||||
"""Test transformation with system message and TTL"""
|
||||
def test_transform_with_system_message_and_ttl(self, custom_llm_provider, monkeypatch):
|
||||
"""Test transformation with system message and TTL (operator allows TTLs above the 1 hour default cap)"""
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "gemini_context_cache_max_ttl_seconds", 86400, raising=False)
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
|
|
|
|||
|
|
@ -52,6 +52,13 @@ class TestContextCachingEndpoints:
|
|||
return_value=True,
|
||||
)
|
||||
self._token_check_patcher.start()
|
||||
# Storage pricing is covered by test_context_cache_storage_pricing.py; these tests use
|
||||
# models without a storage rate in the cost map, so give every model one here.
|
||||
self._storage_price_patcher = patch(
|
||||
"litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching.cache_storage_price",
|
||||
return_value=4.5e-06,
|
||||
)
|
||||
self._storage_price_patcher.start()
|
||||
|
||||
# Sample messages for testing
|
||||
self.sample_messages = [
|
||||
|
|
@ -83,6 +90,7 @@ class TestContextCachingEndpoints:
|
|||
def teardown_method(self):
|
||||
"""Teardown for each test method"""
|
||||
self._token_check_patcher.stop()
|
||||
self._storage_price_patcher.stop()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider", ["gemini", "vertex_ai", "vertex_ai_beta"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue