scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

fix: charge 12h OpenAI Responses cache writes

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-10-21 05:45:07 +00:00
parent 159b5b2f72
commit c443f83a79
9 changed files with 223 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/model_prices_and_context_window_backup.json
View file

@ -36699,6 +36699,8 @@
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 0.000025,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00005,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
@ -36784,6 +36786,8 @@
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 0.000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -36861,6 +36865,8 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 0.00000025,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000005,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
@ -36947,6 +36953,8 @@
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_1hr": 0.00001,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37021,6 +37029,8 @@
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_1hr": 0.00001,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37097,6 +37107,8 @@
},
"gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 0.000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -37172,6 +37184,8 @@
},
"gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_1hr": 0.0000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.000001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
@ -37247,6 +37261,8 @@
},
"gpt-5.6-cyber": {
"cache_creation_input_token_cost": 1.5625e-05,
"cache_creation_input_token_cost_above_1hr": 0.00003125,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000625,
"cache_creation_input_token_cost_above_272k_tokens": 3.125e-05,
"cache_read_input_token_cost": 1.25e-06,
"cache_read_input_token_cost_above_272k_tokens": 2.5e-06,
@ -74677,6 +74693,8 @@
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 0.000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,

50
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -22860,6 +22860,35 @@
"title": "Body_image_edit_api_openai_deployments__model__images_edits_post",
"type": "object"
},
"CacheWriteTokensDetails": {
"additionalProperties": true,
"properties": {
"ttl_12h_tokens": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Ttl 12H Tokens"
},
"ttl_30m_tokens": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Ttl 30M Tokens"
}
},
"title": "CacheWriteTokensDetails",
"type": "object"
},
"CachedTokensDetails": {
"properties": {
"audio_tokens": {
@ -24354,6 +24383,27 @@
],
"title": "Audio Tokens"
},
"cache_write_tokens": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Cache Write Tokens"
},
"cache_write_tokens_details": {
"anyOf": [
{
"$ref": "#/components/schemas/CacheWriteTokensDetails"
},
{
"type": "null"
}
]
},
"cached_tokens": {
"default": 0,
"title": "Cached Tokens",

31
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -3401,11 +3401,6 @@ class LiteLLMCompletionResponsesConfig:
cache_write_tokens: Final = getattr(prompt_details, "cache_write_tokens", None) or getattr(
prompt_details, "cache_creation_tokens", None
)
cache_write_extra: Final[Mapping[str, int]] = (
MappingProxyType({"cache_write_tokens": cache_write_tokens})
if cache_write_tokens is not None
else MappingProxyType({})
)
cache_type: Final = getattr(prompt_details, "cache_type", None)
cache_type_extra: Final[Mapping[str, str]] = (
MappingProxyType({"cache_type": cache_type}) if cache_type is not None else MappingProxyType({})
@ -3426,18 +3421,20 @@ class LiteLLMCompletionResponsesConfig:
if count is not None
}
)
response_usage.input_tokens_details = InputTokensDetails(
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
text_tokens=prompt_details.text_tokens,
audio_tokens=prompt_details.audio_tokens,
image_tokens=prompt_details.image_tokens,
video_tokens=prompt_details.video_tokens,
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
**cache_type_extra,
**grounding_request_counts,
response_usage.input_tokens_details = InputTokensDetails.model_validate(
{
"cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
"text_tokens": prompt_details.text_tokens,
"audio_tokens": prompt_details.audio_tokens,
"image_tokens": prompt_details.image_tokens,
"video_tokens": prompt_details.video_tokens,
"cached_tokens_details": (
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
"cache_write_tokens": cache_write_tokens,
**cache_type_extra,
**grounding_request_counts,
}
)
# Translate completion_tokens_details to output_tokens_details

58
litellm/responses/utils.py
View file

@ -13,6 +13,8 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
from litellm.types.llms.openai import (
AllMessageValues,
CacheWriteTokensDetails,
InputTokensDetails,
OutputTokensDetails,
ResponseAPIUsage,
ResponseInputParam,
@ -23,6 +25,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.responses.main import DecodedResponseId
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
SpecialEnums,
@ -41,6 +44,23 @@ def _output_token_detail(details: object, field: str) -> int | None:
return value if isinstance(value, int) else None
def _normalize_input_tokens_details(details: object) -> InputTokensDetails:
if isinstance(details, InputTokensDetails):
return details
return InputTokensDetails.model_validate(details)
def _cache_creation_token_details(
details: CacheWriteTokensDetails | None,
) -> CacheCreationTokenDetails | None:
if details is None:
return None
return CacheCreationTokenDetails(
ephemeral_5m_input_tokens=details.ttl_30m_tokens,
ephemeral_1h_input_tokens=details.ttl_12h_tokens,
)
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
return isinstance(value, list)
@ -1246,25 +1266,25 @@ class ResponseAPILoggingUtils:
completion_tokens: Final[int] = response_api_usage.output_tokens or 0
prompt_tokens_details: PromptTokensDetailsWrapper | None = None
if response_api_usage.input_tokens_details:
if isinstance(response_api_usage.input_tokens_details, dict):
prompt_tokens_details = PromptTokensDetailsWrapper(**response_api_usage.input_tokens_details)
else:
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
cached_tokens_details=getattr(
response_api_usage.input_tokens_details, "cached_tokens_details", None
),
video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None),
cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None),
web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None),
google_maps_grounding_requests=getattr(
response_api_usage.input_tokens_details, "google_maps_grounding_requests", None
),
)
input_tokens_details: Final[InputTokensDetails] = _normalize_input_tokens_details(
response_api_usage.input_tokens_details
)
cache_write_tokens_details: Final[CacheWriteTokensDetails | None] = (
input_tokens_details.cache_write_tokens_details
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=input_tokens_details.cached_tokens,
audio_tokens=input_tokens_details.audio_tokens,
text_tokens=input_tokens_details.text_tokens,
image_tokens=input_tokens_details.image_tokens,
cached_tokens_details=input_tokens_details.cached_tokens_details,
video_tokens=input_tokens_details.video_tokens,
cache_write_tokens=input_tokens_details.cache_write_tokens,
cache_type=getattr(input_tokens_details, "cache_type", None),
web_search_requests=getattr(input_tokens_details, "web_search_requests", None),
google_maps_grounding_requests=getattr(input_tokens_details, "google_maps_grounding_requests", None),
cache_creation_token_details=_cache_creation_token_details(cache_write_tokens_details),
)
completion_tokens_details: CompletionTokensDetailsWrapper | None = None
output_tokens_details: Final[OutputTokensDetails | None] = getattr(
response_api_usage, "output_tokens_details", None

13
litellm/types/llms/openai.py
View file

@ -1381,10 +1381,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
model_config = {"extra": "allow"}
class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject):
ttl_30m_tokens: int | None = None
ttl_12h_tokens: int | None = None
model_config = ConfigDict(frozen=True)
def _is_none(value: object) -> bool:
return value is None
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none)
cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none)
image_tokens: int | None = None
text_tokens: int | None = None
video_tokens: int | None = None

18
model_prices_and_context_window.json
View file

@ -36699,6 +36699,8 @@
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 0.000025,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00005,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
@ -36784,6 +36786,8 @@
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 0.000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -36861,6 +36865,8 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 0.00000025,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000005,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
@ -36947,6 +36953,8 @@
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_1hr": 0.00001,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37021,6 +37029,8 @@
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_1hr": 0.00001,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00002,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37097,6 +37107,8 @@
},
"gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 0.000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -37172,6 +37184,8 @@
},
"gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_1hr": 0.0000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.000001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
@ -37247,6 +37261,8 @@
},
"gpt-5.6-cyber": {
"cache_creation_input_token_cost": 1.5625e-05,
"cache_creation_input_token_cost_above_1hr": 0.00003125,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.0000625,
"cache_creation_input_token_cost_above_272k_tokens": 3.125e-05,
"cache_read_input_token_cost": 1.25e-06,
"cache_read_input_token_cost_above_272k_tokens": 2.5e-06,
@ -74677,6 +74693,8 @@
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 0.000005,
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": 0.00001,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,

58
tests/unit/litellm_core_utils/llm_cost_calc/test_responses_cache_cost_breakdown.py
View file

@ -13,13 +13,16 @@ break-outs that overlap it, matching the Anthropic path. A request with no cache
activity leaves both fields unset, which is the documented behavior.
"""
from collections.abc import Mapping
from datetime import datetime
from typing import Final, cast
import pytest
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.llms.openai import CacheWriteTokensDetails, InputTokensDetails, ResponseAPIUsage
from litellm.types.utils import Choices, Message, ModelResponse
MODEL = "gpt-5.6"
@ -93,6 +96,61 @@ def test_responses_api_cache_costs_are_itemized_in_the_breakdown(local_model_cos
assert total == pytest.approx(breakdown["total_cost"])
@pytest.mark.parametrize("details_as_object", [False, True], ids=["dict", "model"])
def test_responses_api_12h_cache_writes_use_the_12h_rate(local_model_cost_map, details_as_object: bool):
rates: Final[Mapping[str, float]] = cast(Mapping[str, float], litellm.model_cost[MODEL])
usage_input: Final[dict[str, object] | ResponseAPIUsage] = (
ResponseAPIUsage(
input_tokens=12000,
output_tokens=200,
total_tokens=12200,
input_tokens_details=InputTokensDetails(
cached_tokens=4000,
cache_write_tokens=7000,
cache_write_tokens_details=CacheWriteTokensDetails(ttl_30m_tokens=3000, ttl_12h_tokens=4000),
),
)
if details_as_object
else {
"input_tokens": 12000,
"output_tokens": 200,
"total_tokens": 12200,
"input_tokens_details": {
"cached_tokens": 4000,
"cache_write_tokens": 7000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 4000},
},
}
)
usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_input)
completion_response: Final = ModelResponse(
id="x",
created=1,
model=MODEL,
object="chat.completion",
choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")],
usage=usage,
)
logging_obj: Final = _logging_obj()
litellm.completion_cost(
completion_response=completion_response,
model=MODEL,
custom_llm_provider="openai",
litellm_logging_obj=logging_obj,
)
breakdown: Final = logging_obj.cost_breakdown
assert breakdown is not None
expected_input: Final = (
1000 * rates["input_cost_per_token"]
+ 4000 * rates["cache_read_input_token_cost"]
+ 3000 * rates["cache_creation_input_token_cost"]
+ 4000 * rates["cache_creation_input_token_cost_above_1hr"]
)
assert breakdown["input_cost"] == pytest.approx(expected_input)
def test_cache_fields_stay_unset_when_there_was_no_cache_activity(local_model_cost_map):
"""A request that neither read nor wrote the cache leaves both break-out fields
off the breakdown, so they serialize as null rather than a misleading $0."""

1
tests/unit/test_utils.py
View file

@ -940,6 +940,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_audio_token_cost": {"type": "number"},
"cache_creation_input_token_cost": {"type": "number"},
"cache_creation_input_token_cost_above_1hr": {"type": "number"},
"cache_creation_input_token_cost_above_1hr_above_272k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_32k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_100k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"},

12
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -28452,6 +28452,15 @@ export interface components {
*/
uncached_input_tokens: number;
};
/** CacheWriteTokensDetails */
CacheWriteTokensDetails: {
/** Ttl 12H Tokens */
ttl_12h_tokens?: number | null;
/** Ttl 30M Tokens */
ttl_30m_tokens?: number | null;
} & {
[key: string]: unknown;
};
/** CachedTokensDetails */
CachedTokensDetails: {
/** Audio Tokens */
@ -33607,6 +33616,9 @@ export interface components {
InputTokensDetails: {
/** Audio Tokens */
audio_tokens?: number | null;
/** Cache Write Tokens */
cache_write_tokens?: number | null;
cache_write_tokens_details?: components["schemas"]["CacheWriteTokensDetails"] | null;
/**
* Cached Tokens
* @default 0