mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(router): isolate all per-deployment pricing overrides from sibling deployments (#31021)
* fix(router): isolate all per-deployment pricing overrides from sibling deployments CustomPricingLiteLLMParams is the authoritative set of per-deployment pricing fields, used to strip overrides from the shared backend-alias key so one deployment cannot pollute a sibling that shares the same backend model. It had drifted from ModelInfoBase: tiered and per-unit cost fields such as input_cost_per_token_above_272k_tokens, cache_read_input_token_cost_above_*, output_vector_size, ocr_cost_per_*, and the regional uplift multipliers were absent, so a deployment overriding any of them leaked the override into litellm.model_cost under the shared key and every sibling read the wrong rate via /model/info (LIT-3897). Add the missing fields so the denylist covers every ModelInfoBase pricing field, and guard against future drift with a test asserting the two stay in sync, plus a regression test that a tiered override stays isolated to its own deployment model_id key. * chore(ui): regenerate schema.d.ts for custom pricing fields
This commit is contained in:
mubashir1osmani
GitHub
parent
c18a870746
commit
21cd1d1a4f
3 changed files with 209 additions and 0 deletions
|
|
@ -3148,6 +3148,19 @@ class CustomPricingLiteLLMParams(BaseModel):
|
|||
search_context_cost_per_query: Optional[Dict[str, Any]] = None
|
||||
citation_cost_per_token: Optional[float] = None
|
||||
tiered_pricing: Optional[List[Dict[str, Any]]] = None
|
||||
cache_read_input_token_cost_above_272k_tokens: Optional[float] = None
|
||||
cache_read_input_token_cost_above_512k_tokens: Optional[float] = None
|
||||
input_cost_per_image_token: Optional[float] = None
|
||||
input_cost_per_token_above_272k_tokens: Optional[float] = None
|
||||
input_cost_per_token_above_512k_tokens: Optional[float] = None
|
||||
output_cost_per_token_above_272k_tokens: Optional[float] = None
|
||||
output_cost_per_token_above_512k_tokens: Optional[float] = None
|
||||
output_vector_size: Optional[int] = None
|
||||
ocr_cost_per_page: Optional[float] = None
|
||||
ocr_cost_per_credit: Optional[float] = None
|
||||
annotation_cost_per_page: Optional[float] = None
|
||||
regional_processing_uplift_multiplier_eu: Optional[float] = None
|
||||
regional_processing_uplift_multiplier_us: Optional[float] = None
|
||||
|
||||
|
||||
all_litellm_params = (
|
||||
|
|
|
|||
|
|
@ -537,3 +537,147 @@ def test_inherit_builtin_cache_pricing_noop_for_unknown_backend():
|
|||
)
|
||||
|
||||
assert model_info == {"input_cost_per_token": 0.000003}
|
||||
|
||||
|
||||
def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields():
|
||||
"""The shared-backend-key stripping in Router relies on
|
||||
CustomPricingLiteLLMParams enumerating every per-deployment pricing field.
|
||||
If a new pricing field is added to ModelInfoBase but not mirrored here, a
|
||||
deployment override on that field leaks into the shared backend key and
|
||||
every sibling deployment reads the wrong rate (LIT-3897). This guard fails
|
||||
fast when the two drift apart.
|
||||
"""
|
||||
import typing
|
||||
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfoBase
|
||||
|
||||
pricing_markers = ("cost", "price", "uplift", "vector_size", "tiered_pricing")
|
||||
builtin_pricing_fields = {
|
||||
name
|
||||
for name in typing.get_type_hints(ModelInfoBase)
|
||||
if any(marker in name for marker in pricing_markers)
|
||||
}
|
||||
denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys())
|
||||
|
||||
uncovered = sorted(builtin_pricing_fields - denylisted_fields)
|
||||
assert not uncovered, (
|
||||
"ModelInfoBase pricing fields missing from CustomPricingLiteLLMParams; "
|
||||
f"these would leak into shared backend keys: {uncovered}"
|
||||
)
|
||||
|
||||
|
||||
def test_tiered_pricing_override_isolated_from_sibling_via_model_info_lookup():
|
||||
"""LIT-3897: a deployment that overrides a tiered pricing field
|
||||
(input_cost_per_token_above_272k_tokens) must not pollute the shared
|
||||
backend key, so a sibling sharing the same backend resolves its pricing
|
||||
via litellm.get_model_info (the path /model/info uses) without seeing the
|
||||
override.
|
||||
"""
|
||||
backend_model = "gemini/gemini-2.5-flash"
|
||||
override = 0.000999
|
||||
|
||||
builtin_info = litellm.get_model_info(model=backend_model)
|
||||
assert builtin_info.get("input_cost_per_token_above_272k_tokens") != override
|
||||
|
||||
model_keys = {
|
||||
"lit3897-tiered-custom": litellm.model_cost.get("lit3897-tiered-custom"),
|
||||
"lit3897-tiered-sibling": litellm.model_cost.get("lit3897-tiered-sibling"),
|
||||
backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)),
|
||||
}
|
||||
try:
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "custom-priced-flash",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key-tiered-1",
|
||||
},
|
||||
"model_info": {
|
||||
"id": "lit3897-tiered-custom",
|
||||
"input_cost_per_token_above_272k_tokens": override,
|
||||
"cache_read_input_token_cost_above_272k_tokens": override,
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "gemini-2.5-flash",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key-tiered-2",
|
||||
},
|
||||
"model_info": {"id": "lit3897-tiered-sibling"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
shared = litellm.get_model_info(model=backend_model)
|
||||
assert shared.get("input_cost_per_token_above_272k_tokens") != override, (
|
||||
"Tiered override leaked into the shared backend key; siblings read "
|
||||
"the wrong rate via /model/info"
|
||||
)
|
||||
assert shared.get("cache_read_input_token_cost_above_272k_tokens") != override
|
||||
|
||||
custom_entry = litellm.model_cost["lit3897-tiered-custom"]
|
||||
assert custom_entry["input_cost_per_token_above_272k_tokens"] == override
|
||||
assert custom_entry["cache_read_input_token_cost_above_272k_tokens"] == override
|
||||
finally:
|
||||
_restore_model_cost_entries(model_keys)
|
||||
|
||||
|
||||
def test_custom_pricing_isolated_from_sibling_via_proxy_model_info_path():
|
||||
"""LIT-3897 end to end through the proxy resolution helper: the override
|
||||
deployment reports its custom input rate while the sibling keeps the
|
||||
canonical gemini rate when /model/info resolves each deployment. Mirrors the
|
||||
ticket config where the override is set on litellm_params.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import _get_proxy_model_info
|
||||
|
||||
backend_model = "gemini/gemini-2.5-flash"
|
||||
override_input = 5e-05
|
||||
override_output = 1e-04
|
||||
|
||||
builtin_info = litellm.get_model_info(model=backend_model)
|
||||
builtin_input = builtin_info["input_cost_per_token"]
|
||||
assert builtin_input != override_input
|
||||
|
||||
model_keys = {
|
||||
"lit3897-proxy-custom": litellm.model_cost.get("lit3897-proxy-custom"),
|
||||
"lit3897-proxy-sibling": litellm.model_cost.get("lit3897-proxy-sibling"),
|
||||
backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)),
|
||||
}
|
||||
try:
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "custom-priced-flash",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key-proxy-1",
|
||||
"input_cost_per_token": override_input,
|
||||
"output_cost_per_token": override_output,
|
||||
},
|
||||
"model_info": {"id": "lit3897-proxy-custom"},
|
||||
},
|
||||
{
|
||||
"model_name": "gemini-2.5-flash",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key-proxy-2",
|
||||
},
|
||||
"model_info": {"id": "lit3897-proxy-sibling"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
resolved = {
|
||||
m["model_name"]: _get_proxy_model_info(model=copy.deepcopy(m))[
|
||||
"model_info"
|
||||
]["input_cost_per_token"]
|
||||
for m in router.model_list
|
||||
}
|
||||
|
||||
assert resolved["custom-priced-flash"] == override_input
|
||||
assert resolved["gemini-2.5-flash"] == builtin_input
|
||||
assert resolved["gemini-2.5-flash"] != resolved["custom-priced-flash"]
|
||||
finally:
|
||||
_restore_model_cost_entries(model_keys)
|
||||
|
|
|
|||
52
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
52
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25112,6 +25112,8 @@ export interface components {
|
|||
} | null;
|
||||
/** Adaptive Router Default Model */
|
||||
adaptive_router_default_model?: string | null;
|
||||
/** Annotation Cost Per Page */
|
||||
annotation_cost_per_page?: number | null;
|
||||
/** Api Base */
|
||||
api_base?: string | null;
|
||||
/** Api Key */
|
||||
|
|
@ -25156,8 +25158,12 @@ export interface components {
|
|||
cache_read_input_token_cost_above_200k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Above 200K Tokens Priority */
|
||||
cache_read_input_token_cost_above_200k_tokens_priority?: number | null;
|
||||
/** Cache Read Input Token Cost Above 272K Tokens */
|
||||
cache_read_input_token_cost_above_272k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Above 272K Tokens Priority */
|
||||
cache_read_input_token_cost_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Read Input Token Cost Above 512K Tokens */
|
||||
cache_read_input_token_cost_above_512k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Flex */
|
||||
cache_read_input_token_cost_flex?: number | null;
|
||||
/** Cache Read Input Token Cost Priority */
|
||||
|
|
@ -25194,6 +25200,8 @@ export interface components {
|
|||
input_cost_per_image?: number | null;
|
||||
/** Input Cost Per Image Above 128K Tokens */
|
||||
input_cost_per_image_above_128k_tokens?: number | null;
|
||||
/** Input Cost Per Image Token */
|
||||
input_cost_per_image_token?: number | null;
|
||||
/** Input Cost Per Pixel */
|
||||
input_cost_per_pixel?: number | null;
|
||||
/** Input Cost Per Query */
|
||||
|
|
@ -25208,8 +25216,12 @@ export interface components {
|
|||
input_cost_per_token_above_200k_tokens?: number | null;
|
||||
/** Input Cost Per Token Above 200K Tokens Priority */
|
||||
input_cost_per_token_above_200k_tokens_priority?: number | null;
|
||||
/** Input Cost Per Token Above 272K Tokens */
|
||||
input_cost_per_token_above_272k_tokens?: number | null;
|
||||
/** Input Cost Per Token Above 272K Tokens Priority */
|
||||
input_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Input Cost Per Token Above 512K Tokens */
|
||||
input_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Input Cost Per Token Batches */
|
||||
input_cost_per_token_batches?: number | null;
|
||||
/** Input Cost Per Token Cache Hit */
|
||||
|
|
@ -25255,6 +25267,10 @@ export interface components {
|
|||
model_info?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Ocr Cost Per Credit */
|
||||
ocr_cost_per_credit?: number | null;
|
||||
/** Ocr Cost Per Page */
|
||||
ocr_cost_per_page?: number | null;
|
||||
/** Organization */
|
||||
organization?: string | null;
|
||||
/** Output Cost Per Audio Per Second */
|
||||
|
|
@ -25285,8 +25301,12 @@ export interface components {
|
|||
output_cost_per_token_above_200k_tokens?: number | null;
|
||||
/** Output Cost Per Token Above 200K Tokens Priority */
|
||||
output_cost_per_token_above_200k_tokens_priority?: number | null;
|
||||
/** Output Cost Per Token Above 272K Tokens */
|
||||
output_cost_per_token_above_272k_tokens?: number | null;
|
||||
/** Output Cost Per Token Above 272K Tokens Priority */
|
||||
output_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Output Cost Per Token Above 512K Tokens */
|
||||
output_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Output Cost Per Token Batches */
|
||||
output_cost_per_token_batches?: number | null;
|
||||
/** Output Cost Per Token Flex */
|
||||
|
|
@ -25295,6 +25315,8 @@ export interface components {
|
|||
output_cost_per_token_priority?: number | null;
|
||||
/** Output Cost Per Video Per Second */
|
||||
output_cost_per_video_per_second?: number | null;
|
||||
/** Output Vector Size */
|
||||
output_vector_size?: number | null;
|
||||
/** Quality Router Config */
|
||||
quality_router_config?: {
|
||||
[key: string]: unknown;
|
||||
|
|
@ -25303,6 +25325,10 @@ export interface components {
|
|||
quality_router_default_model?: string | null;
|
||||
/** Region Name */
|
||||
region_name?: string | null;
|
||||
/** Regional Processing Uplift Multiplier Eu */
|
||||
regional_processing_uplift_multiplier_eu?: number | null;
|
||||
/** Regional Processing Uplift Multiplier Us */
|
||||
regional_processing_uplift_multiplier_us?: number | null;
|
||||
/** Rpm */
|
||||
rpm?: number | null;
|
||||
/** S3 Bucket Name */
|
||||
|
|
@ -32794,6 +32820,8 @@ export interface components {
|
|||
} | null;
|
||||
/** Adaptive Router Default Model */
|
||||
adaptive_router_default_model?: string | null;
|
||||
/** Annotation Cost Per Page */
|
||||
annotation_cost_per_page?: number | null;
|
||||
/** Api Base */
|
||||
api_base?: string | null;
|
||||
/** Api Key */
|
||||
|
|
@ -32838,8 +32866,12 @@ export interface components {
|
|||
cache_read_input_token_cost_above_200k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Above 200K Tokens Priority */
|
||||
cache_read_input_token_cost_above_200k_tokens_priority?: number | null;
|
||||
/** Cache Read Input Token Cost Above 272K Tokens */
|
||||
cache_read_input_token_cost_above_272k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Above 272K Tokens Priority */
|
||||
cache_read_input_token_cost_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Read Input Token Cost Above 512K Tokens */
|
||||
cache_read_input_token_cost_above_512k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Flex */
|
||||
cache_read_input_token_cost_flex?: number | null;
|
||||
/** Cache Read Input Token Cost Priority */
|
||||
|
|
@ -32876,6 +32908,8 @@ export interface components {
|
|||
input_cost_per_image?: number | null;
|
||||
/** Input Cost Per Image Above 128K Tokens */
|
||||
input_cost_per_image_above_128k_tokens?: number | null;
|
||||
/** Input Cost Per Image Token */
|
||||
input_cost_per_image_token?: number | null;
|
||||
/** Input Cost Per Pixel */
|
||||
input_cost_per_pixel?: number | null;
|
||||
/** Input Cost Per Query */
|
||||
|
|
@ -32890,8 +32924,12 @@ export interface components {
|
|||
input_cost_per_token_above_200k_tokens?: number | null;
|
||||
/** Input Cost Per Token Above 200K Tokens Priority */
|
||||
input_cost_per_token_above_200k_tokens_priority?: number | null;
|
||||
/** Input Cost Per Token Above 272K Tokens */
|
||||
input_cost_per_token_above_272k_tokens?: number | null;
|
||||
/** Input Cost Per Token Above 272K Tokens Priority */
|
||||
input_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Input Cost Per Token Above 512K Tokens */
|
||||
input_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Input Cost Per Token Batches */
|
||||
input_cost_per_token_batches?: number | null;
|
||||
/** Input Cost Per Token Cache Hit */
|
||||
|
|
@ -32937,6 +32975,10 @@ export interface components {
|
|||
model_info?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Ocr Cost Per Credit */
|
||||
ocr_cost_per_credit?: number | null;
|
||||
/** Ocr Cost Per Page */
|
||||
ocr_cost_per_page?: number | null;
|
||||
/** Organization */
|
||||
organization?: string | null;
|
||||
/** Output Cost Per Audio Per Second */
|
||||
|
|
@ -32967,8 +33009,12 @@ export interface components {
|
|||
output_cost_per_token_above_200k_tokens?: number | null;
|
||||
/** Output Cost Per Token Above 200K Tokens Priority */
|
||||
output_cost_per_token_above_200k_tokens_priority?: number | null;
|
||||
/** Output Cost Per Token Above 272K Tokens */
|
||||
output_cost_per_token_above_272k_tokens?: number | null;
|
||||
/** Output Cost Per Token Above 272K Tokens Priority */
|
||||
output_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Output Cost Per Token Above 512K Tokens */
|
||||
output_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Output Cost Per Token Batches */
|
||||
output_cost_per_token_batches?: number | null;
|
||||
/** Output Cost Per Token Flex */
|
||||
|
|
@ -32977,6 +33023,8 @@ export interface components {
|
|||
output_cost_per_token_priority?: number | null;
|
||||
/** Output Cost Per Video Per Second */
|
||||
output_cost_per_video_per_second?: number | null;
|
||||
/** Output Vector Size */
|
||||
output_vector_size?: number | null;
|
||||
/** Quality Router Config */
|
||||
quality_router_config?: {
|
||||
[key: string]: unknown;
|
||||
|
|
@ -32985,6 +33033,10 @@ export interface components {
|
|||
quality_router_default_model?: string | null;
|
||||
/** Region Name */
|
||||
region_name?: string | null;
|
||||
/** Regional Processing Uplift Multiplier Eu */
|
||||
regional_processing_uplift_multiplier_eu?: number | null;
|
||||
/** Regional Processing Uplift Multiplier Us */
|
||||
regional_processing_uplift_multiplier_us?: number | null;
|
||||
/** Rpm */
|
||||
rpm?: number | null;
|
||||
/** S3 Bucket Name */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue