mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
Fix/cache token (#28009)
Squash-merged by litellm-agent from escon1004's PR.
This commit is contained in:
escon1004
Sameer Kankute
parent
67af5e0205
commit
36685849a0
8 changed files with 581 additions and 20 deletions
|
|
@ -173,17 +173,45 @@ def _cost_per_token_custom_pricing_helper(
|
|||
prompt_tokens: float = 0,
|
||||
completion_tokens: float = 0,
|
||||
response_time_ms: Optional[float] = 0.0,
|
||||
cached_tokens: float = 0,
|
||||
cache_creation_tokens: float = 0,
|
||||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: Optional[CostPerToken] = None,
|
||||
custom_cost_per_second: Optional[float] = None,
|
||||
) -> Optional[Tuple[float, float]]:
|
||||
"""Internal helper function for calculating cost, if custom pricing given"""
|
||||
"""Internal helper function for calculating cost, if custom pricing given.
|
||||
|
||||
prompt_tokens is assumed to include both cached_tokens and cache_creation_tokens
|
||||
(OpenAI-compatible convention). Anthropic-style usage where prompt_tokens excludes
|
||||
cache tokens is handled at the caller (cost_per_token) before invoking this helper.
|
||||
"""
|
||||
if custom_cost_per_token is None and custom_cost_per_second is None:
|
||||
return None
|
||||
|
||||
if custom_cost_per_token is not None:
|
||||
input_cost = custom_cost_per_token["input_cost_per_token"] * prompt_tokens
|
||||
output_cost = custom_cost_per_token["output_cost_per_token"] * completion_tokens
|
||||
input_cost_per_token = custom_cost_per_token["input_cost_per_token"]
|
||||
output_cost_per_token = custom_cost_per_token["output_cost_per_token"]
|
||||
|
||||
cache_read_input_token_cost = custom_cost_per_token.get(
|
||||
"cache_read_input_token_cost",
|
||||
input_cost_per_token,
|
||||
)
|
||||
cache_creation_input_token_cost = custom_cost_per_token.get(
|
||||
"cache_creation_input_token_cost",
|
||||
input_cost_per_token,
|
||||
)
|
||||
|
||||
regular_prompt_tokens = max(
|
||||
prompt_tokens - cached_tokens - cache_creation_tokens,
|
||||
0,
|
||||
)
|
||||
|
||||
input_cost = (
|
||||
regular_prompt_tokens * input_cost_per_token
|
||||
+ cached_tokens * cache_read_input_token_cost
|
||||
+ cache_creation_tokens * cache_creation_input_token_cost
|
||||
)
|
||||
output_cost = completion_tokens * output_cost_per_token
|
||||
return input_cost, output_cost
|
||||
elif custom_cost_per_second is not None:
|
||||
output_cost = custom_cost_per_second * response_time_ms / 1000 # type: ignore
|
||||
|
|
@ -323,10 +351,56 @@ def cost_per_token( # noqa: PLR0915
|
|||
)
|
||||
|
||||
## CUSTOM PRICING ##
|
||||
# Normalize cache token counts across providers:
|
||||
# - OpenAI-compatible: usage.prompt_tokens_details.cached_tokens
|
||||
# (prompt_tokens already INCLUDES cached_tokens)
|
||||
# - Anthropic: usage.cache_read_input_tokens / cache_creation_input_tokens
|
||||
# (prompt_tokens does NOT include these — adjust before calling helper)
|
||||
_cache_read_tokens: float = 0
|
||||
_cache_creation_tokens: float = 0
|
||||
_is_anthropic_style = False
|
||||
|
||||
if usage_object is not None:
|
||||
_pt_details = getattr(usage_object, "prompt_tokens_details", None)
|
||||
if _pt_details is not None:
|
||||
_cache_read_tokens = float(getattr(_pt_details, "cached_tokens", 0) or 0)
|
||||
# OpenAI-compatible providers report cache-write tokens under
|
||||
# either `cache_creation_tokens` or `cache_write_tokens` (kimi-k2
|
||||
# uses the latter). Mirror db_spend_update_writer to stay symmetric.
|
||||
_cache_creation_tokens = float(
|
||||
getattr(_pt_details, "cache_creation_tokens", 0)
|
||||
or getattr(_pt_details, "cache_write_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
|
||||
_anthropic_read = getattr(usage_object, "cache_read_input_tokens", None)
|
||||
_anthropic_create = getattr(usage_object, "cache_creation_input_tokens", None)
|
||||
if _anthropic_read or _anthropic_create:
|
||||
_is_anthropic_style = True
|
||||
if _anthropic_read:
|
||||
_cache_read_tokens = float(_anthropic_read)
|
||||
if _anthropic_create:
|
||||
_cache_creation_tokens = float(_anthropic_create)
|
||||
|
||||
if not _cache_read_tokens and cache_read_input_tokens:
|
||||
_cache_read_tokens = float(cache_read_input_tokens)
|
||||
_is_anthropic_style = True
|
||||
if not _cache_creation_tokens and cache_creation_input_tokens:
|
||||
_cache_creation_tokens = float(cache_creation_input_tokens)
|
||||
_is_anthropic_style = True
|
||||
|
||||
# Anthropic reports prompt_tokens as input_tokens (excluding cache tokens).
|
||||
# Adjust so the helper's "prompt_tokens includes cache tokens" invariant holds.
|
||||
_normalized_prompt_tokens = float(prompt_tokens)
|
||||
if _is_anthropic_style:
|
||||
_normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
|
||||
|
||||
response_cost = _cost_per_token_custom_pricing_helper(
|
||||
prompt_tokens=prompt_tokens,
|
||||
prompt_tokens=_normalized_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time_ms=response_time_ms,
|
||||
cached_tokens=_cache_read_tokens,
|
||||
cache_creation_tokens=_cache_creation_tokens,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -69,6 +69,35 @@ else:
|
|||
ProxyLogging = Any
|
||||
|
||||
|
||||
def _extract_cache_read_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_read_input_tokens field.
|
||||
OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_read_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(details.get("cached_tokens", 0) or 0)
|
||||
|
||||
|
||||
def _extract_cache_creation_tokens(usage_obj: dict) -> int:
|
||||
"""
|
||||
Anthropic: top-level cache_creation_input_tokens field.
|
||||
OpenAI-compatible (kimi-k2 etc.): prompt_tokens_details.cache_write_tokens
|
||||
or prompt_tokens_details.cache_creation_tokens.
|
||||
"""
|
||||
explicit = usage_obj.get("cache_creation_input_tokens", 0) or 0
|
||||
if explicit:
|
||||
return int(explicit)
|
||||
details = usage_obj.get("prompt_tokens_details") or {}
|
||||
return int(
|
||||
details.get("cache_write_tokens", 0)
|
||||
or details.get("cache_creation_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
class DBSpendUpdateWriter:
|
||||
"""
|
||||
Module responsible for
|
||||
|
|
@ -1992,12 +2021,8 @@ class DBSpendUpdateWriter:
|
|||
api_requests=1,
|
||||
successful_requests=1 if request_status == "success" else 0,
|
||||
failed_requests=1 if request_status != "success" else 0,
|
||||
cache_read_input_tokens=usage_obj.get("cache_read_input_tokens", 0)
|
||||
or 0,
|
||||
cache_creation_input_tokens=usage_obj.get(
|
||||
"cache_creation_input_tokens", 0
|
||||
)
|
||||
or 0,
|
||||
cache_read_input_tokens=_extract_cache_read_tokens(usage_obj),
|
||||
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
|
||||
)
|
||||
return daily_transaction
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -107,9 +107,13 @@ class LiteLLMCommonStrings(Enum):
|
|||
SupportedCacheControls = ["ttl", "s-maxage", "no-cache", "no-store"]
|
||||
|
||||
|
||||
class CostPerToken(TypedDict):
|
||||
input_cost_per_token: float
|
||||
output_cost_per_token: float
|
||||
class CostPerToken(TypedDict, total=False):
|
||||
# Required base rates — kept under total=False so we can mark them
|
||||
# Required individually while leaving the cache rates NotRequired.
|
||||
input_cost_per_token: Required[float]
|
||||
output_cost_per_token: Required[float]
|
||||
cache_read_input_token_cost: float
|
||||
cache_creation_input_token_cost: float
|
||||
|
||||
|
||||
class ProviderField(TypedDict):
|
||||
|
|
|
|||
|
|
@ -2057,3 +2057,317 @@ def test_openrouter_gemini_3_1_flash_lite_preview_pricing():
|
|||
assert model_info["output_cost_per_token"] == 1.5e-06
|
||||
assert model_info["max_input_tokens"] == 1048576
|
||||
assert model_info["max_output_tokens"] == 65536
|
||||
|
||||
|
||||
def test_custom_pricing_applies_cache_read_input_cost():
|
||||
"""
|
||||
Bug 1 reproduction: custom_cost_per_token with cache_read_input_token_cost
|
||||
should bill cached prompt tokens at the cache rate, not the full input rate.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=6074,
|
||||
completion_tokens=285,
|
||||
total_tokens=6359,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=3456,
|
||||
audio_tokens=0,
|
||||
),
|
||||
)
|
||||
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 0.0000025,
|
||||
"output_cost_per_token": 0.000015,
|
||||
"cache_read_input_token_cost": 0.00000025,
|
||||
},
|
||||
)
|
||||
|
||||
expected = (6074 - 3456) * 0.0000025 + 3456 * 0.00000025 + 285 * 0.000015
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_custom_pricing_applies_cache_creation_input_cost_via_prompt_details():
|
||||
"""
|
||||
OpenAI-compatible providers report cache-write tokens under
|
||||
prompt_tokens_details.cache_creation_tokens. The custom-pricing helper must
|
||||
bill those at cache_creation_input_token_cost, not the full input rate.
|
||||
"""
|
||||
pt_details = PromptTokensDetailsWrapper(cached_tokens=1000, audio_tokens=0)
|
||||
pt_details.cache_creation_tokens = 500
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=4000,
|
||||
completion_tokens=100,
|
||||
total_tokens=4100,
|
||||
prompt_tokens_details=pt_details,
|
||||
)
|
||||
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 0.0000025,
|
||||
"output_cost_per_token": 0.000015,
|
||||
"cache_read_input_token_cost": 0.00000025,
|
||||
"cache_creation_input_token_cost": 0.000003125,
|
||||
},
|
||||
)
|
||||
|
||||
expected = (
|
||||
(4000 - 1000 - 500) * 0.0000025
|
||||
+ 1000 * 0.00000025
|
||||
+ 500 * 0.000003125
|
||||
+ 100 * 0.000015
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_custom_pricing_applies_cache_creation_input_cost_via_cache_write_tokens_alias():
|
||||
"""
|
||||
Some OpenAI-compatible providers (e.g. kimi-k2) emit cache-write tokens as
|
||||
`cache_write_tokens` rather than `cache_creation_tokens`. The cost
|
||||
calculator must mirror db_spend_update_writer and accept either name —
|
||||
otherwise daily aggregation counts the tokens but the per-request cost
|
||||
bills them at the full input rate.
|
||||
|
||||
Drives `cost_per_token` directly with a SimpleNamespace usage stub so the
|
||||
`cache_write_tokens` alias survives the call (Pydantic's Usage init
|
||||
rebuilds prompt_tokens_details and drops dynamic attributes).
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.cost_calculator import cost_per_token
|
||||
|
||||
pt_details = SimpleNamespace(cached_tokens=1000, cache_write_tokens=500)
|
||||
usage_stub = SimpleNamespace(
|
||||
prompt_tokens=4000,
|
||||
completion_tokens=100,
|
||||
total_tokens=4100,
|
||||
prompt_tokens_details=pt_details,
|
||||
cache_read_input_tokens=None,
|
||||
cache_creation_input_tokens=None,
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model="moonshotai/kimi-k2",
|
||||
prompt_tokens=4000,
|
||||
completion_tokens=100,
|
||||
custom_llm_provider="openai",
|
||||
usage_object=usage_stub,
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 0.0000025,
|
||||
"output_cost_per_token": 0.000015,
|
||||
"cache_read_input_token_cost": 0.00000025,
|
||||
"cache_creation_input_token_cost": 0.000003125,
|
||||
},
|
||||
)
|
||||
|
||||
expected_prompt = (
|
||||
(4000 - 1000 - 500) * 0.0000025 + 1000 * 0.00000025 + 500 * 0.000003125
|
||||
)
|
||||
expected_completion = 100 * 0.000015
|
||||
|
||||
assert prompt_cost == pytest.approx(expected_prompt)
|
||||
assert completion_cost == pytest.approx(expected_completion)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bug 2 — db_spend_update_writer cache token extraction helpers.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_extract_cache_read_tokens_anthropic_top_level():
|
||||
from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens
|
||||
|
||||
usage_obj = {
|
||||
"prompt_tokens": 100,
|
||||
"cache_read_input_tokens": 80,
|
||||
"prompt_tokens_details": {"cached_tokens": 80},
|
||||
}
|
||||
# Anthropic top-level value should win over prompt_tokens_details fallback.
|
||||
assert _extract_cache_read_tokens(usage_obj) == 80
|
||||
|
||||
|
||||
def test_extract_cache_read_tokens_openai_compatible_fallback():
|
||||
from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens
|
||||
|
||||
# Anthropic field absent — fall back to prompt_tokens_details.cached_tokens.
|
||||
usage_obj = {
|
||||
"prompt_tokens": 22583,
|
||||
"prompt_tokens_details": {"cached_tokens": 22016},
|
||||
}
|
||||
assert _extract_cache_read_tokens(usage_obj) == 22016
|
||||
|
||||
|
||||
def test_extract_cache_read_tokens_zero_when_missing():
|
||||
from litellm.proxy.db.db_spend_update_writer import _extract_cache_read_tokens
|
||||
|
||||
assert _extract_cache_read_tokens({}) == 0
|
||||
assert _extract_cache_read_tokens({"cache_read_input_tokens": None}) == 0
|
||||
assert (
|
||||
_extract_cache_read_tokens({"prompt_tokens_details": {"cached_tokens": None}})
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_anthropic_top_level():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
usage_obj = {
|
||||
"prompt_tokens": 100,
|
||||
"cache_creation_input_tokens": 50,
|
||||
"prompt_tokens_details": {"cache_write_tokens": 50},
|
||||
}
|
||||
# Anthropic top-level should short-circuit the fallback.
|
||||
assert _extract_cache_creation_tokens(usage_obj) == 50
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_openai_cache_write_alias():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
# kimi-k2 emits cache_write_tokens.
|
||||
usage_obj = {
|
||||
"prompt_tokens": 1000,
|
||||
"prompt_tokens_details": {"cache_write_tokens": 200},
|
||||
}
|
||||
assert _extract_cache_creation_tokens(usage_obj) == 200
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_openai_cache_creation_alias():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
# Other OpenAI-compatible providers emit cache_creation_tokens.
|
||||
usage_obj = {
|
||||
"prompt_tokens": 1000,
|
||||
"prompt_tokens_details": {"cache_creation_tokens": 300},
|
||||
}
|
||||
assert _extract_cache_creation_tokens(usage_obj) == 300
|
||||
|
||||
|
||||
def test_extract_cache_creation_tokens_zero_when_missing():
|
||||
from litellm.proxy.db.db_spend_update_writer import (
|
||||
_extract_cache_creation_tokens,
|
||||
)
|
||||
|
||||
assert _extract_cache_creation_tokens({}) == 0
|
||||
assert _extract_cache_creation_tokens({"cache_creation_input_tokens": None}) == 0
|
||||
assert (
|
||||
_extract_cache_creation_tokens(
|
||||
{"prompt_tokens_details": {"cache_write_tokens": None}}
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
def test_custom_pricing_anthropic_style_cache_tokens_not_double_counted():
|
||||
"""
|
||||
Anthropic providers report cache tokens at the top level of Usage, and
|
||||
`prompt_tokens` EXCLUDES them. The helper expects `prompt_tokens` to
|
||||
include cache tokens, so cost_per_token must adjust before invoking it —
|
||||
otherwise regular_prompt_tokens goes negative and clamps to 0.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=2000,
|
||||
completion_tokens=100,
|
||||
total_tokens=2100,
|
||||
cache_read_input_tokens=1500,
|
||||
cache_creation_input_tokens=300,
|
||||
)
|
||||
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="anthropic/claude-3-5-sonnet",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="anthropic/claude-3-5-sonnet",
|
||||
custom_llm_provider="anthropic",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 0.000003,
|
||||
"output_cost_per_token": 0.000015,
|
||||
"cache_read_input_token_cost": 0.0000003,
|
||||
"cache_creation_input_token_cost": 0.00000375,
|
||||
},
|
||||
)
|
||||
|
||||
# Anthropic prompt_tokens=2000 excludes cache. After normalization the
|
||||
# helper sees 2000 + 1500 + 300 = 3800, of which 2000 are uncached.
|
||||
expected = 2000 * 0.000003 + 1500 * 0.0000003 + 300 * 0.00000375 + 100 * 0.000015
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_custom_pricing_without_cache_keys_preserves_legacy_behavior():
|
||||
"""
|
||||
Backward compatibility: when custom_cost_per_token omits both cache rates,
|
||||
cached tokens must be billed at input_cost_per_token (matching the pre-fix
|
||||
behavior) so existing callers see no change.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=100,
|
||||
total_tokens=1100,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=400,
|
||||
audio_tokens=0,
|
||||
),
|
||||
)
|
||||
|
||||
response = ModelResponse(
|
||||
id="test-id",
|
||||
created=1234567890,
|
||||
model="openai/gpt-5.4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="openai/gpt-5.4",
|
||||
custom_llm_provider="openai",
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": 0.0000025,
|
||||
"output_cost_per_token": 0.000015,
|
||||
},
|
||||
)
|
||||
|
||||
# All 1000 prompt tokens billed at input rate, regardless of cached_tokens.
|
||||
expected = 1000 * 0.0000025 + 100 * 0.000015
|
||||
|
||||
assert cost == pytest.approx(expected)
|
||||
|
|
|
|||
|
|
@ -654,12 +654,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
|
|||
<Card>
|
||||
<Title>Input Tokens</Title>
|
||||
<Text className="text-2xl font-bold mt-2 text-blue-600">
|
||||
{Math.max(
|
||||
0,
|
||||
(userSpendData.metadata?.total_prompt_tokens || 0) -
|
||||
(userSpendData.metadata?.total_cache_read_input_tokens || 0) -
|
||||
(userSpendData.metadata?.total_cache_creation_input_tokens || 0)
|
||||
).toLocaleString()}
|
||||
{(userSpendData.metadata?.total_prompt_tokens || 0).toLocaleString()}
|
||||
</Text>
|
||||
</Card>
|
||||
<Card>
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
|
|||
form.setFieldsValue({
|
||||
input_cost_per_token: undefined,
|
||||
output_cost_per_token: undefined,
|
||||
cache_read_input_token_cost: undefined,
|
||||
cache_creation_input_token_cost: undefined,
|
||||
input_cost_per_second: undefined,
|
||||
});
|
||||
}
|
||||
|
|
@ -211,6 +213,24 @@ const AdvancedSettings: React.FC<AdvancedSettingsProps> = ({
|
|||
>
|
||||
<TextInput />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Cache Read Cost (per 1M tokens)"
|
||||
name="cache_read_input_token_cost"
|
||||
rules={[{ validator: validateNumber }]}
|
||||
tooltip="If left blank, defaults to Input Cost."
|
||||
className="mb-4"
|
||||
>
|
||||
<TextInput placeholder="Defaults to Input Cost if blank" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Cache Write Cost (per 1M tokens)"
|
||||
name="cache_creation_input_token_cost"
|
||||
rules={[{ validator: validateNumber }]}
|
||||
tooltip="If left blank, defaults to Input Cost (the backend falls back to input_cost_per_token when no cache-write rate is set)."
|
||||
className="mb-4"
|
||||
>
|
||||
<TextInput placeholder="Defaults to Input Cost if blank" />
|
||||
</Form.Item>
|
||||
</>
|
||||
) : (
|
||||
<Form.Item
|
||||
|
|
|
|||
|
|
@ -45,6 +45,39 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
|
|||
if (formValues.output_cost_per_token !== undefined && formValues.output_cost_per_token !== null && formValues.output_cost_per_token !== "") {
|
||||
formValues.output_cost_per_token = Number(formValues.output_cost_per_token) / 1000000;
|
||||
}
|
||||
|
||||
// Cache Read Cost: if blank, default to Input Cost (already token-unit converted above)
|
||||
if (
|
||||
formValues.cache_read_input_token_cost !== undefined &&
|
||||
formValues.cache_read_input_token_cost !== null &&
|
||||
formValues.cache_read_input_token_cost !== ""
|
||||
) {
|
||||
formValues.cache_read_input_token_cost =
|
||||
Number(formValues.cache_read_input_token_cost) / 1000000;
|
||||
} else if (
|
||||
formValues.input_cost_per_token !== undefined &&
|
||||
formValues.input_cost_per_token !== null &&
|
||||
formValues.input_cost_per_token !== ""
|
||||
) {
|
||||
formValues.cache_read_input_token_cost = Number(formValues.input_cost_per_token);
|
||||
} else {
|
||||
delete formValues.cache_read_input_token_cost;
|
||||
}
|
||||
|
||||
// Cache Write Cost: explicit value if provided, else leave unset so the
|
||||
// backend keeps the model-level default (per-second pricing, model_prices
|
||||
// entries, etc.). Sending 0 here would overwrite that default.
|
||||
// The backend falls back to input_cost_per_token when this key is absent.
|
||||
if (
|
||||
formValues.cache_creation_input_token_cost !== undefined &&
|
||||
formValues.cache_creation_input_token_cost !== null &&
|
||||
formValues.cache_creation_input_token_cost !== ""
|
||||
) {
|
||||
formValues.cache_creation_input_token_cost =
|
||||
Number(formValues.cache_creation_input_token_cost) / 1000000;
|
||||
} else {
|
||||
delete formValues.cache_creation_input_token_cost;
|
||||
}
|
||||
// Keep input_cost_per_second as is, no conversion needed
|
||||
|
||||
// Iterate through the key-value pairs in formValues
|
||||
|
|
@ -119,7 +152,13 @@ export const prepareModelAddRequest = async (formValues: Record<string, any>, ac
|
|||
}
|
||||
|
||||
// Handle the pricing fields
|
||||
else if (key === "input_cost_per_token" || key === "output_cost_per_token" || key === "input_cost_per_second") {
|
||||
else if (
|
||||
key === "input_cost_per_token" ||
|
||||
key === "output_cost_per_token" ||
|
||||
key === "input_cost_per_second" ||
|
||||
key === "cache_read_input_token_cost" ||
|
||||
key === "cache_creation_input_token_cost"
|
||||
) {
|
||||
if (value !== undefined && value !== null && value !== "") {
|
||||
litellmParamsObj[key] = Number(value);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -263,6 +263,34 @@ export default function ModelInfoView({
|
|||
updatedLitellmParams.output_cost_per_token = Number(values.output_cost) / 1_000_000;
|
||||
}
|
||||
|
||||
// Cache Read Cost: explicit value if provided, else fall back to input cost (when input cost touched).
|
||||
if (form.isFieldTouched("cache_read_cost") || form.isFieldTouched("input_cost")) {
|
||||
if (
|
||||
values.cache_read_cost !== undefined &&
|
||||
values.cache_read_cost !== null &&
|
||||
values.cache_read_cost !== ""
|
||||
) {
|
||||
updatedLitellmParams.cache_read_input_token_cost = Number(values.cache_read_cost) / 1_000_000;
|
||||
} else if (updatedLitellmParams.input_cost_per_token !== undefined) {
|
||||
updatedLitellmParams.cache_read_input_token_cost = updatedLitellmParams.input_cost_per_token;
|
||||
}
|
||||
}
|
||||
|
||||
// Cache Write Cost: explicit value if provided, else clear the override
|
||||
// so the backend falls back to the model-level default. Sending 0 here
|
||||
// would persist a zero rate even when the user intended to unset it.
|
||||
if (form.isFieldTouched("cache_write_cost")) {
|
||||
if (
|
||||
values.cache_write_cost !== undefined &&
|
||||
values.cache_write_cost !== null &&
|
||||
values.cache_write_cost !== ""
|
||||
) {
|
||||
updatedLitellmParams.cache_creation_input_token_cost = Number(values.cache_write_cost) / 1_000_000;
|
||||
} else {
|
||||
delete updatedLitellmParams.cache_creation_input_token_cost;
|
||||
}
|
||||
}
|
||||
|
||||
if (values.litellm_credential_name) {
|
||||
updatedLitellmParams.litellm_credential_name = values.litellm_credential_name;
|
||||
} else {
|
||||
|
|
@ -638,6 +666,22 @@ export default function ModelInfoView({
|
|||
output_cost: localModelData.litellm_params?.output_cost_per_token
|
||||
? localModelData.litellm_params.output_cost_per_token * 1_000_000
|
||||
: localModelData.model_info?.output_cost_per_token * 1_000_000 || null,
|
||||
cache_read_cost:
|
||||
localModelData.litellm_params?.cache_read_input_token_cost !== undefined &&
|
||||
localModelData.litellm_params?.cache_read_input_token_cost !== null
|
||||
? localModelData.litellm_params.cache_read_input_token_cost * 1_000_000
|
||||
: localModelData.model_info?.cache_read_input_token_cost !== undefined &&
|
||||
localModelData.model_info?.cache_read_input_token_cost !== null
|
||||
? localModelData.model_info.cache_read_input_token_cost * 1_000_000
|
||||
: null,
|
||||
cache_write_cost:
|
||||
localModelData.litellm_params?.cache_creation_input_token_cost !== undefined &&
|
||||
localModelData.litellm_params?.cache_creation_input_token_cost !== null
|
||||
? localModelData.litellm_params.cache_creation_input_token_cost * 1_000_000
|
||||
: localModelData.model_info?.cache_creation_input_token_cost !== undefined &&
|
||||
localModelData.model_info?.cache_creation_input_token_cost !== null
|
||||
? localModelData.model_info.cache_creation_input_token_cost * 1_000_000
|
||||
: null,
|
||||
cache_control: localModelData.litellm_params?.cache_control_injection_points ? true : false,
|
||||
cache_control_injection_points: localModelData.litellm_params?.cache_control_injection_points || [],
|
||||
model_access_group: Array.isArray(localModelData.model_info?.access_groups)
|
||||
|
|
@ -725,6 +769,52 @@ export default function ModelInfoView({
|
|||
)}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Text className="font-medium">Cache Read Cost (per 1M tokens)</Text>
|
||||
{isEditing ? (
|
||||
<Form.Item
|
||||
name="cache_read_cost"
|
||||
className="mb-0"
|
||||
tooltip="If left blank on save, defaults to Input Cost."
|
||||
>
|
||||
<NumericalInput placeholder="Defaults to Input Cost if blank" />
|
||||
</Form.Item>
|
||||
) : (
|
||||
<div className="mt-1 p-2 bg-gray-50 rounded">
|
||||
{localModelData?.litellm_params?.cache_read_input_token_cost !== undefined &&
|
||||
localModelData?.litellm_params?.cache_read_input_token_cost !== null
|
||||
? (localModelData.litellm_params.cache_read_input_token_cost * 1_000_000).toFixed(4)
|
||||
: localModelData?.model_info?.cache_read_input_token_cost !== undefined &&
|
||||
localModelData?.model_info?.cache_read_input_token_cost !== null
|
||||
? (localModelData.model_info.cache_read_input_token_cost * 1_000_000).toFixed(4)
|
||||
: "Not Set"}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Text className="font-medium">Cache Write Cost (per 1M tokens)</Text>
|
||||
{isEditing ? (
|
||||
<Form.Item
|
||||
name="cache_write_cost"
|
||||
className="mb-0"
|
||||
tooltip="If left blank on save, defaults to Input Cost (backend falls back to input_cost_per_token)."
|
||||
>
|
||||
<NumericalInput placeholder="Defaults to Input Cost if blank" />
|
||||
</Form.Item>
|
||||
) : (
|
||||
<div className="mt-1 p-2 bg-gray-50 rounded">
|
||||
{localModelData?.litellm_params?.cache_creation_input_token_cost !== undefined &&
|
||||
localModelData?.litellm_params?.cache_creation_input_token_cost !== null
|
||||
? (localModelData.litellm_params.cache_creation_input_token_cost * 1_000_000).toFixed(4)
|
||||
: localModelData?.model_info?.cache_creation_input_token_cost !== undefined &&
|
||||
localModelData?.model_info?.cache_creation_input_token_cost !== null
|
||||
? (localModelData.model_info.cache_creation_input_token_cost * 1_000_000).toFixed(4)
|
||||
: "Not Set"}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Text className="font-medium">API Base</Text>
|
||||
{isEditing ? (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue