mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
Merge 04833e4afb into eb65b8f017
This commit is contained in:
roasted-penguin
GitHub
commit
f40e58634c
17 changed files with 688 additions and 6 deletions
|
|
@ -58,6 +58,7 @@ pub struct ModelInfoFields {
|
|||
pub cache_creation_input_token_cost_above_12hr_ultrafast: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches: Option<f64>,
|
||||
pub cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast: Option<f64>,
|
||||
pub cache_storage_cost_per_token_per_hour: Option<f64>,
|
||||
pub off_peak_pricing: Option<Value>,
|
||||
pub input_cost_per_character: Option<f64>,
|
||||
pub input_cost_per_token_above_128k_tokens: Option<f64>,
|
||||
|
|
|
|||
|
|
@ -53,6 +53,8 @@ attributes! {
|
|||
bedrock_converse_supports_strict_tools: bool,
|
||||
/// Highest reasoning effort the Bedrock output_config accepts for this model.
|
||||
bedrock_output_config_effort_ceiling: EffortLevel,
|
||||
/// USD per token per hour to keep an explicitly created context cache stored.
|
||||
cache_storage_cost_per_token_per_hour: UnitPrice,
|
||||
comment: String,
|
||||
/// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
|
||||
default_reasoning_effort: EffortLevel,
|
||||
|
|
@ -313,6 +315,8 @@ mod tests {
|
|||
#[case::deprecation_source_not_a_url_string("deprecation_source", json!(7), false)]
|
||||
#[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)]
|
||||
#[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)]
|
||||
#[case::cache_storage_rate("cache_storage_cost_per_token_per_hour", json!(4.5e-6), true)]
|
||||
#[case::negative_cache_storage_rate("cache_storage_cost_per_token_per_hour", json!(-1), false)]
|
||||
#[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)]
|
||||
#[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
|
||||
#[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
|
||||
|
|
|
|||
|
|
@ -328,6 +328,45 @@ def _get_additional_costs(
|
|||
return None
|
||||
|
||||
|
||||
def _recorded_cache_storage_token_hours(litellm_logging_obj: LitellmLoggingObject | None) -> object:
|
||||
return litellm_logging_obj.context_cache_storage_token_hours if litellm_logging_obj else 0.0
|
||||
|
||||
|
||||
def _context_cache_storage_cost(
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
*,
|
||||
requested_model: str | None,
|
||||
completion_response: object | None,
|
||||
base_model: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
router_model_id: str | None,
|
||||
region_name: str | None,
|
||||
) -> float:
|
||||
token_hours: Final = _recorded_cache_storage_token_hours(litellm_logging_obj)
|
||||
if not isinstance(token_hours, int | float) or token_hours <= 0:
|
||||
return 0.0
|
||||
underlying_model: Final = select_model_name_for_cost_calc(
|
||||
model=requested_model,
|
||||
completion_response=completion_response,
|
||||
base_model=base_model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
region_name=region_name,
|
||||
)
|
||||
rate: Final = next(
|
||||
(
|
||||
price
|
||||
for source in (
|
||||
_deployment_model_info(litellm_logging_obj, custom_pricing=True, router_model_id=router_model_id),
|
||||
_cost_map_model_info(underlying_model, custom_llm_provider) if underlying_model else None,
|
||||
)
|
||||
if source is not None
|
||||
and isinstance(price := source.get("cache_storage_cost_per_token_per_hour"), int | float)
|
||||
),
|
||||
0.0,
|
||||
)
|
||||
return token_hours * rate
|
||||
|
||||
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second",
|
||||
"output_cost_per_second",
|
||||
|
|
@ -2350,6 +2389,20 @@ def completion_cost(
|
|||
)
|
||||
else:
|
||||
additional_costs = None
|
||||
_storage_response: ModelResponse | None = (
|
||||
completion_response if isinstance(completion_response, ModelResponse) else None
|
||||
)
|
||||
_storage_provider: str | None = custom_llm_provider if isinstance(custom_llm_provider, str) else None
|
||||
_storage_region: str | None = region_name if isinstance(region_name, str) else None
|
||||
cache_storage_cost: float = _context_cache_storage_cost(
|
||||
litellm_logging_obj,
|
||||
requested_model=requested_model,
|
||||
completion_response=_storage_response,
|
||||
base_model=base_model,
|
||||
custom_llm_provider=_storage_provider,
|
||||
router_model_id=router_model_id,
|
||||
region_name=_storage_region,
|
||||
)
|
||||
|
||||
_final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
|
||||
cost_for_built_in_tools = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
|
|
@ -2362,6 +2415,12 @@ def completion_cost(
|
|||
_final_cost += cost_for_built_in_tools
|
||||
if additional_costs:
|
||||
_final_cost += sum(additional_costs.values())
|
||||
if cache_storage_cost > 0:
|
||||
_final_cost += cache_storage_cost
|
||||
if additional_costs is None:
|
||||
additional_costs = {"cache_storage_cost": cache_storage_cost}
|
||||
else:
|
||||
additional_costs["cache_storage_cost"] = cache_storage_cost
|
||||
|
||||
original_cost = _final_cost
|
||||
if litellm.cost_discount_config:
|
||||
|
|
|
|||
|
|
@ -691,6 +691,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
baseline_cache_context: "BaselineCacheContext | None" = None
|
||||
baseline_observation: "CapturedBaselineObservation | None" = None
|
||||
_on_failure_payload_built: "Callable[[], None] | None" = None
|
||||
context_cache_storage_token_hours: float = 0.0
|
||||
|
||||
def record_context_cache_storage(self, token_hours: float) -> None:
|
||||
self.context_cache_storage_token_hours = token_hours
|
||||
|
||||
@property
|
||||
def _defer_async_logging(self) -> bool:
|
||||
|
|
|
|||
56
litellm/llms/vertex_ai/context_caching/storage_cost.py
Normal file
56
litellm/llms/vertex_ai/context_caching/storage_cost.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, ValidationError
|
||||
|
||||
DEFAULT_CONTEXT_CACHE_TTL_SECONDS: Final = 3600.0
|
||||
_SECONDS_PER_HOUR: Final = 3600.0
|
||||
|
||||
|
||||
class _CachedContentUsage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
totalTokenCount: int = 0
|
||||
|
||||
|
||||
class _CreatedCachedContent(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
usageMetadata: _CachedContentUsage = _CachedContentUsage()
|
||||
createTime: datetime | None = None
|
||||
expireTime: datetime | None = None
|
||||
|
||||
|
||||
def _reported_lifetime_seconds(created: _CreatedCachedContent) -> float | None:
|
||||
if created.createTime is None or created.expireTime is None:
|
||||
return None
|
||||
seconds: Final = (created.expireTime - created.createTime).total_seconds()
|
||||
return seconds if seconds > 0 else None
|
||||
|
||||
|
||||
def _requested_ttl_seconds(requested_ttl: str | None) -> float | None:
|
||||
if requested_ttl is None or not requested_ttl.endswith("s"):
|
||||
return None
|
||||
try:
|
||||
seconds: Final = float(requested_ttl[:-1])
|
||||
except ValueError:
|
||||
return None
|
||||
return seconds if seconds > 0 else None
|
||||
|
||||
|
||||
def context_cache_storage_token_hours(created_cached_content: object, requested_ttl: str | None) -> float:
|
||||
"""Token-hours a newly created cachedContents resource is billed for over its whole TTL.
|
||||
|
||||
The lifetime comes from the create response (expireTime - createTime), then the TTL that was
|
||||
requested, then the provider default of one hour.
|
||||
"""
|
||||
try:
|
||||
created: Final = _CreatedCachedContent.model_validate(created_cached_content)
|
||||
except ValidationError:
|
||||
return 0.0
|
||||
lifetime_seconds: Final = (
|
||||
_reported_lifetime_seconds(created)
|
||||
or _requested_ttl_seconds(requested_ttl)
|
||||
or DEFAULT_CONTEXT_CACHE_TTL_SECONDS
|
||||
)
|
||||
return created.usageMetadata.totalTokenCount * lifetime_seconds / _SECONDS_PER_HOUR
|
||||
|
|
@ -22,6 +22,7 @@ from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_p
|
|||
from ..common_utils import VertexAIError, get_vertex_base_url
|
||||
from ..vertex_llm_base import VertexBase, publisher_model_location
|
||||
from .known_cached_contents import KnownCachedContents, known_cached_contents
|
||||
from .storage_cost import context_cache_storage_token_hours
|
||||
from .transformation import (
|
||||
cached_messages_end_on_supported_turn,
|
||||
separate_cached_messages,
|
||||
|
|
@ -544,6 +545,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
|
||||
raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root
|
||||
_record_cache_creation_tokens(logging_obj, raw_response_cached)
|
||||
logging_obj.record_context_cache_storage(
|
||||
context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl"))
|
||||
)
|
||||
self._known_caches.remember(
|
||||
KnownCachedContents.scope(
|
||||
url, custom_llm_provider, headers, vertex_project=target.project, vertex_location=target.location
|
||||
|
|
@ -696,6 +700,9 @@ class ContextCachingEndpoints(VertexBase):
|
|||
|
||||
raw_response_cached: Final = _CreatedCachedContentRoot.model_validate(response.json()).root
|
||||
_record_cache_creation_tokens(logging_obj, raw_response_cached)
|
||||
logging_obj.record_context_cache_storage(
|
||||
context_cache_storage_token_hours(raw_response_cached, cached_content_request_body.get("ttl"))
|
||||
)
|
||||
self._known_caches.remember(
|
||||
scope, generated_cache_key, raw_response_cached.get("name"), raw_response_cached.get("expireTime")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@
|
|||
"max_output_tokens": 4096
|
||||
},
|
||||
"sample_spec": {
|
||||
"cache_storage_cost_per_token_per_hour": 0.0,
|
||||
"code_interpreter_cost_per_session": 0.0,
|
||||
"computer_use_input_cost_per_1k_tokens": 0.0,
|
||||
"computer_use_output_cost_per_1k_tokens": 0.0,
|
||||
|
|
@ -29582,6 +29583,7 @@
|
|||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
|
|
@ -29948,6 +29950,7 @@
|
|||
},
|
||||
"gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -30005,6 +30008,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
|
|
@ -30070,6 +30074,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_batches": 1.5e-07,
|
||||
"input_cost_per_token_flex": 1.5e-07,
|
||||
|
|
@ -30166,6 +30171,7 @@
|
|||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
|
|
@ -30458,6 +30464,7 @@
|
|||
"cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07,
|
||||
"cache_read_input_token_cost_flex": 1.25e-07,
|
||||
"cache_read_input_token_cost_priority": 2.25e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 2.5e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -30541,6 +30548,7 @@
|
|||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -30673,6 +30681,7 @@
|
|||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
"cache_read_input_token_cost_priority": 3.6e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
|
|
@ -30686,6 +30695,7 @@
|
|||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_priority": 9e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens": 1e-06,
|
||||
|
|
@ -30801,7 +30811,8 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"vertex_ai/gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -30854,6 +30865,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-11-19",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "vertex_ai/gemini-3.8-flash",
|
||||
|
|
@ -30918,6 +30930,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-01-28",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "vertex_ai/gemini-3.8-flash",
|
||||
|
|
@ -30935,6 +30948,7 @@
|
|||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
|
|
@ -30992,6 +31006,7 @@
|
|||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
|
|
@ -31101,6 +31116,7 @@
|
|||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -32989,6 +33005,7 @@
|
|||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_priority": 9e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens": 1e-06,
|
||||
|
|
@ -33137,7 +33154,8 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -33190,6 +33208,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-11-19",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "gemini-3.8-flash",
|
||||
|
|
@ -33254,6 +33273,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-01-28",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "gemini-3.8-flash",
|
||||
|
|
@ -33271,6 +33291,7 @@
|
|||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
|
|
@ -33328,6 +33349,7 @@
|
|||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
|
|
@ -53795,6 +53817,7 @@
|
|||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -53852,6 +53875,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
|
|
@ -53918,6 +53942,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_batches": 1.5e-07,
|
||||
"input_cost_per_token_flex": 1.5e-07,
|
||||
|
|
|
|||
|
|
@ -331,6 +331,7 @@ class ModelInfo(MirroredPricingParams):
|
|||
ptu_effective_from: datetime.datetime | None = None
|
||||
ptu_effective_to: datetime.datetime | None = None
|
||||
ptu_shares: Mapping[str, Annotated[StrictInt, BeforeValidator(_whole_float_as_int)]] | None = None
|
||||
cache_storage_cost_per_token_per_hour: float | None = None
|
||||
|
||||
# when tag-based routing's "!" or "&" constraints eliminate every deployment
|
||||
# in this model group, fall back to the default-tagged pool instead of
|
||||
|
|
|
|||
|
|
@ -377,6 +377,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_creation_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
cache_storage_cost_per_token_per_hour: ReadOnly[float | None]
|
||||
# Smallest prefix this model will actually cache, whatever caching mechanism its provider uses.
|
||||
# Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT.
|
||||
prompt_cache_min_tokens: int | None
|
||||
|
|
@ -4297,6 +4298,8 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
|
||||
|
||||
DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS: Final[frozenset[str]] = frozenset({"cache_storage_cost_per_token_per_hour"})
|
||||
|
||||
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"supports_regex_lookaround",
|
||||
|
|
@ -4327,6 +4330,7 @@ SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
|
|||
frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)
|
||||
- frozenset(CustomPricingLiteLLMParams.model_fields)
|
||||
- DEPLOYMENT_SCOPED_PRICING_FIELDS
|
||||
- DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS
|
||||
- DEPLOYMENT_SCOPED_CAPABILITY_FIELDS
|
||||
- DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7438,6 +7438,7 @@ def get_model_info_helper(
|
|||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast", None
|
||||
),
|
||||
cache_storage_cost_per_token_per_hour=_model_info.get("cache_storage_cost_per_token_per_hour"),
|
||||
off_peak_pricing=_model_info.get("off_peak_pricing", None),
|
||||
input_cost_per_character=_model_info.get("input_cost_per_character", None),
|
||||
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@
|
|||
"max_output_tokens": 4096
|
||||
},
|
||||
"sample_spec": {
|
||||
"cache_storage_cost_per_token_per_hour": 0.0,
|
||||
"code_interpreter_cost_per_session": 0.0,
|
||||
"computer_use_input_cost_per_1k_tokens": 0.0,
|
||||
"computer_use_output_cost_per_1k_tokens": 0.0,
|
||||
|
|
@ -29582,6 +29583,7 @@
|
|||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
|
|
@ -29948,6 +29950,7 @@
|
|||
},
|
||||
"gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -30005,6 +30008,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
|
|
@ -30070,6 +30074,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_batches": 1.5e-07,
|
||||
"input_cost_per_token_flex": 1.5e-07,
|
||||
|
|
@ -30166,6 +30171,7 @@
|
|||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-10-20",
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
|
|
@ -30458,6 +30464,7 @@
|
|||
"cache_read_input_token_cost_above_200k_tokens_priority": 4.5e-07,
|
||||
"cache_read_input_token_cost_flex": 1.25e-07,
|
||||
"cache_read_input_token_cost_priority": 2.25e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"input_cost_per_token": 1.25e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 2.5e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -30541,6 +30548,7 @@
|
|||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -30673,6 +30681,7 @@
|
|||
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
|
||||
"cache_read_input_token_cost_priority": 3.6e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": 0.014,
|
||||
"search_context_size_medium": 0.014,
|
||||
|
|
@ -30686,6 +30695,7 @@
|
|||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_priority": 9e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens": 1e-06,
|
||||
|
|
@ -30801,7 +30811,8 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"vertex_ai/gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -30854,6 +30865,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-11-19",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "vertex_ai/gemini-3.8-flash",
|
||||
|
|
@ -30918,6 +30930,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-01-28",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "vertex_ai/gemini-3.8-flash",
|
||||
|
|
@ -30935,6 +30948,7 @@
|
|||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
|
|
@ -30992,6 +31006,7 @@
|
|||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
|
|
@ -31101,6 +31116,7 @@
|
|||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"cache_storage_cost_per_token_per_hour": 4.5e-06,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -32989,6 +33005,7 @@
|
|||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_priority": 9e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_above_200k_tokens": 1e-06,
|
||||
|
|
@ -33137,7 +33154,8 @@
|
|||
"output_cost_per_token_batches": 4.5e-06,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"output_cost_per_token_flex": 4.5e-06,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06
|
||||
},
|
||||
"gemini-3.6-flash": {
|
||||
"prompt_cache_min_tokens": 4096,
|
||||
|
|
@ -33190,6 +33208,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2026-11-19",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "gemini-3.8-flash",
|
||||
|
|
@ -33254,6 +33273,7 @@
|
|||
"input_cost_per_token_priority": 1.35e-06,
|
||||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-01-28",
|
||||
"deprecation_source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versions",
|
||||
"successor_model": "gemini-3.8-flash",
|
||||
|
|
@ -33271,6 +33291,7 @@
|
|||
"cache_read_input_token_cost_batches": 3.75e-08,
|
||||
"cache_read_input_token_cost_flex": 3.75e-08,
|
||||
"cache_read_input_token_cost_priority": 1.35e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"input_cost_per_token_batches": 3.75e-07,
|
||||
|
|
@ -33328,6 +33349,7 @@
|
|||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
|
|
@ -53795,6 +53817,7 @@
|
|||
},
|
||||
"vertex_ai/gemini-3.1-flash-lite-preview": {
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
|
|
@ -53852,6 +53875,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
|
|
@ -53918,6 +53942,7 @@
|
|||
"cache_read_input_token_cost_batches": 1.5e-08,
|
||||
"cache_read_input_token_cost_flex": 1.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"cache_storage_cost_per_token_per_hour": 1e-06,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"input_cost_per_token_batches": 1.5e-07,
|
||||
"input_cost_per_token_flex": 1.5e-07,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.vertex_ai.context_caching.storage_cost import (
|
||||
context_cache_storage_token_hours,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"created, requested_ttl, expected",
|
||||
[
|
||||
(
|
||||
{
|
||||
"usageMetadata": {"totalTokenCount": 5000},
|
||||
"createTime": "2026-01-01T00:00:00.123456789Z",
|
||||
"expireTime": "2026-01-01T02:00:00.123456789Z",
|
||||
},
|
||||
"60s",
|
||||
10000.0,
|
||||
),
|
||||
({"usageMetadata": {"totalTokenCount": 5000}}, "1800s", 2500.0),
|
||||
({"usageMetadata": {"totalTokenCount": 5000}}, "90.5s", 5000 * 90.5 / 3600),
|
||||
(
|
||||
{"usageMetadata": {"totalTokenCount": 4000}, "expireTime": "2026-01-01T02:00:00Z"},
|
||||
"900s",
|
||||
1000.0,
|
||||
),
|
||||
({"name": "cachedContents/1"}, "3600s", 0.0),
|
||||
({"usageMetadata": {"totalTokenCount": "many"}}, "3600s", 0.0),
|
||||
({"usageMetadata": {"totalTokenCount": 5000}, "expireTime": "not-a-time"}, "3600s", 0.0),
|
||||
("not-an-object", "3600s", 0.0),
|
||||
],
|
||||
)
|
||||
def test_token_hours_follow_reported_lifetime_then_requested_ttl(
|
||||
created: object, requested_ttl: str | None, expected: float
|
||||
) -> None:
|
||||
assert context_cache_storage_token_hours(created, requested_ttl) == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_token_hours_default_to_one_hour_without_lifetime_or_ttl() -> None:
|
||||
# Source: https://cloud.google.com/vertex-ai/generative-ai/docs/context-cache/context-cache-create
|
||||
# "The default expiration time of a context cache is 60 minutes after it's created" (read 2026-10-07)
|
||||
assert context_cache_storage_token_hours({"usageMetadata": {"totalTokenCount": 5000}}, None) == 5000.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested_ttl", ["", "s", "abc", "10m", "-5s"])
|
||||
def test_unparseable_requested_ttl_falls_back_to_default_lifetime(requested_ttl: str) -> None:
|
||||
created: Final = {"usageMetadata": {"totalTokenCount": 3600}}
|
||||
|
||||
assert context_cache_storage_token_hours(created, requested_ttl) == context_cache_storage_token_hours(created, None)
|
||||
|
|
@ -1,9 +1,13 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
||||
|
|
@ -14,6 +18,7 @@ from litellm.llms.vertex_ai.context_caching.known_cached_contents import KnownCa
|
|||
from litellm.llms.vertex_ai.context_caching.vertex_ai_context_caching import (
|
||||
ContextCachingEndpoints,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -2378,6 +2383,212 @@ class TestContextCachingMultiRegionUrls:
|
|||
assert "/locations/global/cachedContents" in url
|
||||
|
||||
|
||||
_STORAGE_TEST_MODEL: Final = "gemini-2.5-pro"
|
||||
|
||||
|
||||
_STORAGE_TEST_CACHE_NAME: Final = "cachedContents/storage-cost"
|
||||
|
||||
|
||||
def _storage_logging_obj() -> Logging:
|
||||
logging_obj: Final = Logging(
|
||||
model=_STORAGE_TEST_MODEL,
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=None,
|
||||
litellm_call_id="context-cache-storage",
|
||||
function_id="context-cache-storage",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model=_STORAGE_TEST_MODEL, litellm_params={}, optional_params={}, custom_llm_provider="gemini"
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _storage_messages() -> list[AllMessageValues]:
|
||||
return [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "policy " * 6000, "cache_control": {"type": "ephemeral"}}],
|
||||
},
|
||||
{"role": "user", "content": "Hello"},
|
||||
]
|
||||
|
||||
|
||||
def _json_response(method: str, payload: Mapping[str, object]) -> httpx.Response:
|
||||
return httpx.Response(200, json=payload, request=httpx.Request(method, "https://example.invalid"))
|
||||
|
||||
|
||||
_CREATED_CACHE: Final = {
|
||||
"name": _STORAGE_TEST_CACHE_NAME,
|
||||
"model": f"models/{_STORAGE_TEST_MODEL}",
|
||||
"usageMetadata": {"totalTokenCount": 7200},
|
||||
"createTime": "2026-01-01T00:00:00Z",
|
||||
"expireTime": "2026-01-01T00:30:00Z",
|
||||
}
|
||||
|
||||
|
||||
class _CreateCacheKwargs(TypedDict):
|
||||
messages: ReadOnly[list[AllMessageValues]]
|
||||
optional_params: ReadOnly[dict[str, object]]
|
||||
api_key: ReadOnly[str]
|
||||
api_base: ReadOnly[str | None]
|
||||
model: ReadOnly[str]
|
||||
client: ReadOnly[MagicMock]
|
||||
timeout: ReadOnly[float]
|
||||
logging_obj: ReadOnly[Logging]
|
||||
custom_llm_provider: ReadOnly[Literal["gemini"]]
|
||||
vertex_project: ReadOnly[str | None]
|
||||
vertex_location: ReadOnly[str | None]
|
||||
vertex_auth_header: ReadOnly[str | None]
|
||||
|
||||
|
||||
def _create_kwargs(logging_obj: Logging, client: MagicMock) -> _CreateCacheKwargs:
|
||||
return {
|
||||
"messages": _storage_messages(),
|
||||
"optional_params": {},
|
||||
"api_key": "test_key",
|
||||
"api_base": None,
|
||||
"model": _STORAGE_TEST_MODEL,
|
||||
"client": client,
|
||||
"timeout": 30.0,
|
||||
"logging_obj": logging_obj,
|
||||
"custom_llm_provider": "gemini",
|
||||
"vertex_project": None,
|
||||
"vertex_location": None,
|
||||
"vertex_auth_header": None,
|
||||
}
|
||||
|
||||
|
||||
def test_creating_a_cache_records_its_storage_token_hours(local_model_cost_map: None) -> None:
|
||||
logging_obj: Final = _storage_logging_obj()
|
||||
client: Final = MagicMock(spec=HTTPHandler)
|
||||
client.get.return_value = _json_response("GET", {})
|
||||
client.post.return_value = _json_response("POST", _CREATED_CACHE)
|
||||
|
||||
_, _, cache_name = ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(logging_obj, client))
|
||||
|
||||
assert cache_name == _STORAGE_TEST_CACHE_NAME
|
||||
assert logging_obj.context_cache_storage_token_hours == pytest.approx(7200 * 0.5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_creating_a_cache_records_its_storage_token_hours(local_model_cost_map: None) -> None:
|
||||
logging_obj: Final = _storage_logging_obj()
|
||||
client: Final = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.get = AsyncMock(return_value=_json_response("GET", {}))
|
||||
client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE))
|
||||
|
||||
_, _, cache_name = await ContextCachingEndpoints().async_check_and_create_cache(
|
||||
**_create_kwargs(logging_obj, client)
|
||||
)
|
||||
|
||||
assert cache_name == _STORAGE_TEST_CACHE_NAME
|
||||
assert logging_obj.context_cache_storage_token_hours == pytest.approx(7200 * 0.5)
|
||||
|
||||
|
||||
def _listing_with(created_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {"cachedContents": [{"name": _STORAGE_TEST_CACHE_NAME, "displayName": created_body["displayName"]}]}
|
||||
|
||||
|
||||
def test_reusing_an_existing_cache_records_no_storage(local_model_cost_map: None) -> None:
|
||||
client: Final = MagicMock(spec=HTTPHandler)
|
||||
client.get.return_value = _json_response("GET", {})
|
||||
client.post.return_value = _json_response("POST", _CREATED_CACHE)
|
||||
ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(_storage_logging_obj(), client))
|
||||
client.get.return_value = _json_response("GET", _listing_with(client.post.call_args.kwargs["json"]))
|
||||
reusing_logging_obj: Final = _storage_logging_obj()
|
||||
|
||||
_, _, cache_name = ContextCachingEndpoints().check_and_create_cache(**_create_kwargs(reusing_logging_obj, client))
|
||||
|
||||
assert cache_name == _STORAGE_TEST_CACHE_NAME
|
||||
assert client.post.call_count == 1
|
||||
assert reusing_logging_obj.context_cache_storage_token_hours == 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_reusing_an_existing_cache_records_no_storage(local_model_cost_map: None) -> None:
|
||||
client: Final = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.get = AsyncMock(return_value=_json_response("GET", {}))
|
||||
client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE))
|
||||
await ContextCachingEndpoints().async_check_and_create_cache(**_create_kwargs(_storage_logging_obj(), client))
|
||||
client.get.return_value = _json_response("GET", _listing_with(client.post.call_args.kwargs["json"]))
|
||||
reusing_logging_obj: Final = _storage_logging_obj()
|
||||
|
||||
_, _, cache_name = await ContextCachingEndpoints().async_check_and_create_cache(
|
||||
**_create_kwargs(reusing_logging_obj, client)
|
||||
)
|
||||
|
||||
assert cache_name == _STORAGE_TEST_CACHE_NAME
|
||||
assert client.post.call_count == 1
|
||||
assert reusing_logging_obj.context_cache_storage_token_hours == 0.0
|
||||
|
||||
|
||||
_STORAGE_TEST_CREATED_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc).timestamp()
|
||||
|
||||
|
||||
def _remembering_endpoints() -> ContextCachingEndpoints:
|
||||
return ContextCachingEndpoints(known_caches=KnownCachedContents(clock=lambda: _STORAGE_TEST_CREATED_AT))
|
||||
|
||||
|
||||
def test_reusing_a_remembered_cache_records_no_storage(local_model_cost_map: None) -> None:
|
||||
endpoints: Final = _remembering_endpoints()
|
||||
client: Final = MagicMock(spec=HTTPHandler)
|
||||
client.get.return_value = _json_response("GET", {})
|
||||
client.post.return_value = _json_response("POST", _CREATED_CACHE)
|
||||
creating_logging_obj: Final = _storage_logging_obj()
|
||||
endpoints.check_and_create_cache(**_create_kwargs(creating_logging_obj, client))
|
||||
reusing_logging_obj: Final = _storage_logging_obj()
|
||||
|
||||
_, _, cache_name = endpoints.check_and_create_cache(**_create_kwargs(reusing_logging_obj, client))
|
||||
|
||||
assert cache_name == _STORAGE_TEST_CACHE_NAME
|
||||
assert (client.get.call_count, client.post.call_count) == (1, 1)
|
||||
assert creating_logging_obj.context_cache_storage_token_hours == 7200 * 0.5
|
||||
assert reusing_logging_obj.context_cache_storage_token_hours == 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_reusing_a_remembered_cache_records_no_storage(local_model_cost_map: None) -> None:
|
||||
endpoints: Final = _remembering_endpoints()
|
||||
client: Final = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.get = AsyncMock(return_value=_json_response("GET", {}))
|
||||
client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE))
|
||||
creating_logging_obj: Final = _storage_logging_obj()
|
||||
await endpoints.async_check_and_create_cache(**_create_kwargs(creating_logging_obj, client))
|
||||
reusing_logging_obj: Final = _storage_logging_obj()
|
||||
|
||||
_, _, cache_name = await endpoints.async_check_and_create_cache(**_create_kwargs(reusing_logging_obj, client))
|
||||
|
||||
assert cache_name == _STORAGE_TEST_CACHE_NAME
|
||||
assert (client.get.await_count, client.post.await_count) == (1, 1)
|
||||
assert creating_logging_obj.context_cache_storage_token_hours == 7200 * 0.5
|
||||
assert reusing_logging_obj.context_cache_storage_token_hours == 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_requests_for_one_prefix_record_storage_once(local_model_cost_map: None) -> None:
|
||||
endpoints: Final = _remembering_endpoints()
|
||||
client: Final = MagicMock(spec=AsyncHTTPHandler)
|
||||
|
||||
async def slow_list(**_: object) -> httpx.Response:
|
||||
await asyncio.sleep(0.01)
|
||||
return _json_response("GET", {})
|
||||
|
||||
client.get = AsyncMock(side_effect=slow_list)
|
||||
client.post = AsyncMock(return_value=_json_response("POST", _CREATED_CACHE))
|
||||
logging_objs: Final = [_storage_logging_obj() for _ in range(3)]
|
||||
|
||||
results: Final = await asyncio.gather(
|
||||
*(endpoints.async_check_and_create_cache(**_create_kwargs(logging_obj, client)) for logging_obj in logging_objs)
|
||||
)
|
||||
|
||||
assert {cache_name for _, _, cache_name in results} == {_STORAGE_TEST_CACHE_NAME}
|
||||
assert client.post.await_count == 1
|
||||
recorded: Final = sorted(logging_obj.context_cache_storage_token_hours for logging_obj in logging_objs)
|
||||
assert recorded == [0.0, 0.0, 7200 * 0.5]
|
||||
|
||||
|
||||
def _created_cache_body(total_token_count: int) -> dict:
|
||||
return {
|
||||
"name": "cachedContents/created",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
import asyncio
|
||||
import datetime
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
|
|
@ -7272,6 +7273,211 @@ def test_completion_cost_bills_base_when_gemini_serves_on_demand(
|
|||
assert cost == pytest.approx(100 * 0.001 + 50 * 0.002)
|
||||
|
||||
|
||||
_STORAGE_RATE_KEY: Final = "cache_storage_cost_per_token_per_hour"
|
||||
|
||||
|
||||
_FIXED_START_TIME: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _storage_cost_map(_local_model_cost_map: None) -> Iterator[None]:
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _first_chat_model(provider_prefix: str, *, with_storage_rate: bool) -> str:
|
||||
names: Final = [name for name in litellm.model_cost if isinstance(name, str)]
|
||||
return next(
|
||||
name
|
||||
for name in names
|
||||
if _is_priced_chat_entry(_cost_map_entry(name), provider_prefix, with_storage_rate=with_storage_rate)
|
||||
)
|
||||
|
||||
|
||||
def _cost_map_entry(name: str) -> object:
|
||||
entry: Final[object] = litellm.model_cost.get(name)
|
||||
return entry
|
||||
|
||||
|
||||
def _is_priced_chat_entry(entry: object, provider_prefix: str, *, with_storage_rate: bool) -> bool:
|
||||
return (
|
||||
isinstance(entry, dict)
|
||||
and entry.get("mode") == "chat"
|
||||
and isinstance(provider := entry.get("litellm_provider"), str)
|
||||
and provider.startswith(provider_prefix)
|
||||
and entry.get("input_cost_per_token") is not None
|
||||
and (_STORAGE_RATE_KEY in entry) == with_storage_rate
|
||||
)
|
||||
|
||||
|
||||
def _cache_storage_logging_obj(model: str, token_hours: float) -> Logging:
|
||||
logging_obj: Final = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=_FIXED_START_TIME,
|
||||
litellm_call_id="context-cache-storage",
|
||||
function_id="f",
|
||||
)
|
||||
if token_hours:
|
||||
logging_obj.record_context_cache_storage(token_hours)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _chat_cost(model: str, logging_obj: Logging, **kwargs: object) -> float:
|
||||
usage: Final = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100)
|
||||
return completion_cost(
|
||||
completion_response=ModelResponse(model=model, usage=usage),
|
||||
model=model,
|
||||
litellm_logging_obj=logging_obj,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_context_cache_storage_is_billed_on_the_creating_request(_storage_cost_map: None) -> None:
|
||||
model: Final = _first_chat_model("vertex_ai", with_storage_rate=True)
|
||||
rate: Final = litellm.get_model_info(model)[_STORAGE_RATE_KEY]
|
||||
creating_request: Final = _cache_storage_logging_obj(model, token_hours=3_600.0)
|
||||
|
||||
with_storage: Final = _chat_cost(model, creating_request)
|
||||
without_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0))
|
||||
|
||||
assert rate is not None and rate > 0
|
||||
assert with_storage - without_storage == pytest.approx(3_600.0 * rate)
|
||||
assert creating_request.cost_breakdown["additional_costs"] == pytest.approx({"cache_storage_cost": 3_600.0 * rate})
|
||||
assert creating_request.cost_breakdown["total_cost"] == pytest.approx(with_storage)
|
||||
|
||||
|
||||
def test_context_cache_storage_costs_nothing_for_models_without_a_storage_rate(_storage_cost_map: None) -> None:
|
||||
model: Final = _first_chat_model("gemini", with_storage_rate=False)
|
||||
creating_request: Final = _cache_storage_logging_obj(model, token_hours=3_600.0)
|
||||
|
||||
with_storage: Final = _chat_cost(model, creating_request)
|
||||
|
||||
assert with_storage == _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0))
|
||||
assert creating_request.cost_breakdown.get("additional_costs") is None
|
||||
|
||||
|
||||
def test_context_cache_storage_uses_the_deployment_storage_rate(_storage_cost_map: None) -> None:
|
||||
model: Final = _first_chat_model("vertex_ai", with_storage_rate=True)
|
||||
deployment_rate: Final = litellm.get_model_info(model)[_STORAGE_RATE_KEY] * 3
|
||||
litellm.register_model(
|
||||
{
|
||||
"storage-cost-deployment": {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
_STORAGE_RATE_KEY: deployment_rate,
|
||||
}
|
||||
}
|
||||
)
|
||||
pricing: Final = {"custom_pricing": True, "router_model_id": "storage-cost-deployment"}
|
||||
|
||||
with_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=500.0), **pricing)
|
||||
without_storage: Final = _chat_cost(model, _cache_storage_logging_obj(model, token_hours=0.0), **pricing)
|
||||
|
||||
assert with_storage - without_storage == pytest.approx(500.0 * deployment_rate)
|
||||
|
||||
|
||||
def test_context_cache_storage_joins_the_provider_additional_costs(_storage_cost_map: None) -> None:
|
||||
litellm.register_model(
|
||||
{"router-storage-deployment": {"litellm_provider": "azure_ai", "mode": "chat", _STORAGE_RATE_KEY: 1e-6}}
|
||||
)
|
||||
logging_obj: Final = _cache_storage_logging_obj("azure_ai/model-router", token_hours=500.0)
|
||||
usage: Final = Usage(prompt_tokens=1_000_000, completion_tokens=100, total_tokens=1_000_100)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=ModelResponse(model="azure_ai/gpt-4o-2024-08-06", usage=usage),
|
||||
model="azure_ai/model-router",
|
||||
custom_llm_provider="azure_ai",
|
||||
litellm_logging_obj=logging_obj,
|
||||
custom_pricing=True,
|
||||
router_model_id="router-storage-deployment",
|
||||
)
|
||||
|
||||
breakdown: Final = logging_obj.cost_breakdown
|
||||
assert breakdown is not None
|
||||
additional_costs: Final = breakdown.get("additional_costs") or {}
|
||||
assert additional_costs["Azure Model Router Flat Cost"] > 0
|
||||
assert math.isclose(additional_costs["cache_storage_cost"], 500.0 * 1e-6)
|
||||
assert math.isclose(cost, sum(additional_costs.values()))
|
||||
|
||||
|
||||
_STORAGE_DEPLOYMENT_MODEL: Final = "gemini-2.5-flash"
|
||||
|
||||
|
||||
def _routed_request_cost(
|
||||
token_prices: Mapping[str, float], token_hours: float, *, storage_rate: float | None = None
|
||||
) -> float | None:
|
||||
from litellm import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "storage-cost-deployment",
|
||||
"litellm_params": {"model": f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}", **token_prices},
|
||||
"model_info": {} if storage_rate is None else {_STORAGE_RATE_KEY: storage_rate},
|
||||
}
|
||||
]
|
||||
)
|
||||
deployment_id: Final = router.model_list[0]["model_info"]["id"]
|
||||
assert isinstance(deployment_id, str)
|
||||
return _vertex_request_cost({**token_prices, "metadata": {"model_info": {"id": deployment_id}}}, token_hours)
|
||||
|
||||
|
||||
def _vertex_request_cost(litellm_params: Mapping[str, object], token_hours: float) -> float | None:
|
||||
logging_obj: Final = _cache_storage_logging_obj(_STORAGE_DEPLOYMENT_MODEL, token_hours)
|
||||
logging_obj.update_environment_variables(
|
||||
model=_STORAGE_DEPLOYMENT_MODEL,
|
||||
litellm_params=dict(litellm_params),
|
||||
optional_params={},
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
usage: Final = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100)
|
||||
return logging_obj.response_cost_calculator(ModelResponse(model=_STORAGE_DEPLOYMENT_MODEL, usage=usage))
|
||||
|
||||
|
||||
def test_custom_token_prices_without_a_storage_rate_bill_storage_at_the_underlying_model_rate(
|
||||
_storage_cost_map: None,
|
||||
) -> None:
|
||||
cost_map_rate: Final = litellm.get_model_info(f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}")[_STORAGE_RATE_KEY]
|
||||
token_prices: Final = {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}
|
||||
|
||||
with_storage: Final = _routed_request_cost(token_prices, token_hours=500.0)
|
||||
without_storage: Final = _routed_request_cost(token_prices, token_hours=0.0)
|
||||
|
||||
assert cost_map_rate is not None and cost_map_rate > 0
|
||||
assert without_storage == pytest.approx(1_000 * 1e-6 + 100 * 2e-6)
|
||||
assert with_storage == pytest.approx(without_storage + 500.0 * cost_map_rate)
|
||||
|
||||
|
||||
def test_a_storage_only_override_keeps_the_underlying_token_prices(_storage_cost_map: None) -> None:
|
||||
cost_map_rate: Final = litellm.get_model_info(f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}")[_STORAGE_RATE_KEY]
|
||||
assert isinstance(cost_map_rate, float)
|
||||
deployment_rate: Final = cost_map_rate * 3
|
||||
|
||||
no_override: Final = _vertex_request_cost({}, token_hours=0.0)
|
||||
without_storage: Final = _routed_request_cost({}, token_hours=0.0, storage_rate=deployment_rate)
|
||||
with_storage: Final = _routed_request_cost({}, token_hours=500.0, storage_rate=deployment_rate)
|
||||
|
||||
assert no_override is not None and no_override > 0
|
||||
assert without_storage == pytest.approx(no_override)
|
||||
assert with_storage == pytest.approx(no_override + 500.0 * deployment_rate)
|
||||
|
||||
|
||||
def test_a_deployment_storage_rate_stays_off_the_shared_backend_key(_storage_cost_map: None) -> None:
|
||||
backend_key: Final = f"vertex_ai/{_STORAGE_DEPLOYMENT_MODEL}"
|
||||
cost_map_rate: Final = litellm.get_model_info(backend_key)[_STORAGE_RATE_KEY]
|
||||
assert isinstance(cost_map_rate, float)
|
||||
|
||||
_routed_request_cost({}, token_hours=0.0, storage_rate=cost_map_rate * 3)
|
||||
|
||||
assert litellm.get_model_info(backend_key)[_STORAGE_RATE_KEY] == cost_map_rate
|
||||
|
||||
|
||||
def test_an_unpriced_provider_slug_does_not_bill_the_turn_at_zero(
|
||||
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -1484,13 +1484,17 @@ def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields():
|
|||
"""
|
||||
import typing
|
||||
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfoBase
|
||||
from litellm.types.utils import (
|
||||
DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS,
|
||||
CustomPricingLiteLLMParams,
|
||||
ModelInfoBase,
|
||||
)
|
||||
|
||||
pricing_markers = ("cost", "price", "uplift", "vector_size", "tiered_pricing")
|
||||
builtin_pricing_fields = {
|
||||
name for name in typing.get_type_hints(ModelInfoBase) if any(marker in name for marker in pricing_markers)
|
||||
}
|
||||
denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys())
|
||||
denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) | DEPLOYMENT_SCOPED_SEPARATE_CHARGE_FIELDS
|
||||
|
||||
uncovered = sorted(builtin_pricing_fields - denylisted_fields)
|
||||
assert not uncovered, (
|
||||
|
|
|
|||
|
|
@ -1002,6 +1002,7 @@ _COST_MAP_ROW_SCHEMA: Final = {
|
|||
"cache_creation_input_token_cost_flex": {"type": "number"},
|
||||
"cache_creation_input_token_cost_priority": {"type": "number"},
|
||||
"cache_creation_input_token_cost_ultrafast": {"type": "number"},
|
||||
"cache_storage_cost_per_token_per_hour": {"type": "number"},
|
||||
"cache_read_input_token_cost": {"type": "number"},
|
||||
"explicit_cache_read_input_token_cost": {"type": "number"},
|
||||
"cache_read_input_token_cost_above_32k_tokens": {"type": "number"},
|
||||
|
|
@ -9148,6 +9149,26 @@ async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch:
|
|||
assert counting_logger.logged_results[0] is inner_result
|
||||
|
||||
|
||||
def test_get_model_info_surfaces_context_cache_storage_rate(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
local_cost_map: Final[Mapping[str, object]] = litellm.get_model_cost_map(url="")
|
||||
monkeypatch.setattr(litellm, "model_cost", local_cost_map)
|
||||
litellm.get_model_info.cache_clear()
|
||||
priced: Final = {
|
||||
name: entry["cache_storage_cost_per_token_per_hour"]
|
||||
for name, entry in litellm.model_cost.items()
|
||||
if isinstance(entry, dict) and "cache_storage_cost_per_token_per_hour" in entry
|
||||
}
|
||||
|
||||
surfaced: Final = {
|
||||
name: litellm.get_model_info(name).get("cache_storage_cost_per_token_per_hour") for name in priced
|
||||
}
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
assert priced
|
||||
assert surfaced == priced
|
||||
|
||||
|
||||
def test_basic_trimming():
|
||||
litellm.turn_on_debug()
|
||||
messages = [
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -52224,6 +52224,8 @@ export interface components {
|
|||
cache_creation_input_token_cost_above_1hr?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
cache_read_input_token_cost?: number | null;
|
||||
/** Cache Storage Cost Per Token Per Hour */
|
||||
cache_storage_cost_per_token_per_hour?: number | null;
|
||||
/** Cost Per Ptu Per Hour */
|
||||
cost_per_ptu_per_hour?: number | null;
|
||||
/** Created At */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue