From 21cd1d1a4f022aa8afe796992bb24ab509eb090b Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 23 Jun 2026 00:11:22 -0700 Subject: [PATCH] fix(router): isolate all per-deployment pricing overrides from sibling deployments (#31021) * fix(router): isolate all per-deployment pricing overrides from sibling deployments CustomPricingLiteLLMParams is the authoritative set of per-deployment pricing fields, used to strip overrides from the shared backend-alias key so one deployment cannot pollute a sibling that shares the same backend model. It had drifted from ModelInfoBase: tiered and per-unit cost fields such as input_cost_per_token_above_272k_tokens, cache_read_input_token_cost_above_*, output_vector_size, ocr_cost_per_*, and the regional uplift multipliers were absent, so a deployment overriding any of them leaked the override into litellm.model_cost under the shared key and every sibling read the wrong rate via /model/info (LIT-3897). Add the missing fields so the denylist covers every ModelInfoBase pricing field, and guard against future drift with a test asserting the two stay in sync, plus a regression test that a tiered override stays isolated to its own deployment model_id key. * chore(ui): regenerate schema.d.ts for custom pricing fields --- litellm/types/utils.py | 13 ++ .../test_router_model_cost_isolation.py | 144 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 52 +++++++ 3 files changed, 209 insertions(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ee4f9bca341..16e693c7d7c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3148,6 +3148,19 @@ class CustomPricingLiteLLMParams(BaseModel): search_context_cost_per_query: Optional[Dict[str, Any]] = None citation_cost_per_token: Optional[float] = None tiered_pricing: Optional[List[Dict[str, Any]]] = None + cache_read_input_token_cost_above_272k_tokens: Optional[float] = None + cache_read_input_token_cost_above_512k_tokens: Optional[float] = None + input_cost_per_image_token: Optional[float] = None + input_cost_per_token_above_272k_tokens: Optional[float] = None + input_cost_per_token_above_512k_tokens: Optional[float] = None + output_cost_per_token_above_272k_tokens: Optional[float] = None + output_cost_per_token_above_512k_tokens: Optional[float] = None + output_vector_size: Optional[int] = None + ocr_cost_per_page: Optional[float] = None + ocr_cost_per_credit: Optional[float] = None + annotation_cost_per_page: Optional[float] = None + regional_processing_uplift_multiplier_eu: Optional[float] = None + regional_processing_uplift_multiplier_us: Optional[float] = None all_litellm_params = ( diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index ee64f44d32c..d4ac9659f00 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -537,3 +537,147 @@ def test_inherit_builtin_cache_pricing_noop_for_unknown_backend(): ) assert model_info == {"input_cost_per_token": 0.000003} + + +def test_custom_pricing_field_denylist_covers_all_builtin_pricing_fields(): + """The shared-backend-key stripping in Router relies on + CustomPricingLiteLLMParams enumerating every per-deployment pricing field. + If a new pricing field is added to ModelInfoBase but not mirrored here, a + deployment override on that field leaks into the shared backend key and + every sibling deployment reads the wrong rate (LIT-3897). This guard fails + fast when the two drift apart. + """ + import typing + + from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfoBase + + pricing_markers = ("cost", "price", "uplift", "vector_size", "tiered_pricing") + builtin_pricing_fields = { + name + for name in typing.get_type_hints(ModelInfoBase) + if any(marker in name for marker in pricing_markers) + } + denylisted_fields = set(CustomPricingLiteLLMParams.model_fields.keys()) + + uncovered = sorted(builtin_pricing_fields - denylisted_fields) + assert not uncovered, ( + "ModelInfoBase pricing fields missing from CustomPricingLiteLLMParams; " + f"these would leak into shared backend keys: {uncovered}" + ) + + +def test_tiered_pricing_override_isolated_from_sibling_via_model_info_lookup(): + """LIT-3897: a deployment that overrides a tiered pricing field + (input_cost_per_token_above_272k_tokens) must not pollute the shared + backend key, so a sibling sharing the same backend resolves its pricing + via litellm.get_model_info (the path /model/info uses) without seeing the + override. + """ + backend_model = "gemini/gemini-2.5-flash" + override = 0.000999 + + builtin_info = litellm.get_model_info(model=backend_model) + assert builtin_info.get("input_cost_per_token_above_272k_tokens") != override + + model_keys = { + "lit3897-tiered-custom": litellm.model_cost.get("lit3897-tiered-custom"), + "lit3897-tiered-sibling": litellm.model_cost.get("lit3897-tiered-sibling"), + backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), + } + try: + Router( + model_list=[ + { + "model_name": "custom-priced-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-tiered-1", + }, + "model_info": { + "id": "lit3897-tiered-custom", + "input_cost_per_token_above_272k_tokens": override, + "cache_read_input_token_cost_above_272k_tokens": override, + }, + }, + { + "model_name": "gemini-2.5-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-tiered-2", + }, + "model_info": {"id": "lit3897-tiered-sibling"}, + }, + ], + ) + + shared = litellm.get_model_info(model=backend_model) + assert shared.get("input_cost_per_token_above_272k_tokens") != override, ( + "Tiered override leaked into the shared backend key; siblings read " + "the wrong rate via /model/info" + ) + assert shared.get("cache_read_input_token_cost_above_272k_tokens") != override + + custom_entry = litellm.model_cost["lit3897-tiered-custom"] + assert custom_entry["input_cost_per_token_above_272k_tokens"] == override + assert custom_entry["cache_read_input_token_cost_above_272k_tokens"] == override + finally: + _restore_model_cost_entries(model_keys) + + +def test_custom_pricing_isolated_from_sibling_via_proxy_model_info_path(): + """LIT-3897 end to end through the proxy resolution helper: the override + deployment reports its custom input rate while the sibling keeps the + canonical gemini rate when /model/info resolves each deployment. Mirrors the + ticket config where the override is set on litellm_params. + """ + from litellm.proxy.proxy_server import _get_proxy_model_info + + backend_model = "gemini/gemini-2.5-flash" + override_input = 5e-05 + override_output = 1e-04 + + builtin_info = litellm.get_model_info(model=backend_model) + builtin_input = builtin_info["input_cost_per_token"] + assert builtin_input != override_input + + model_keys = { + "lit3897-proxy-custom": litellm.model_cost.get("lit3897-proxy-custom"), + "lit3897-proxy-sibling": litellm.model_cost.get("lit3897-proxy-sibling"), + backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), + } + try: + router = Router( + model_list=[ + { + "model_name": "custom-priced-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-proxy-1", + "input_cost_per_token": override_input, + "output_cost_per_token": override_output, + }, + "model_info": {"id": "lit3897-proxy-custom"}, + }, + { + "model_name": "gemini-2.5-flash", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-proxy-2", + }, + "model_info": {"id": "lit3897-proxy-sibling"}, + }, + ], + ) + + resolved = { + m["model_name"]: _get_proxy_model_info(model=copy.deepcopy(m))[ + "model_info" + ]["input_cost_per_token"] + for m in router.model_list + } + + assert resolved["custom-priced-flash"] == override_input + assert resolved["gemini-2.5-flash"] == builtin_input + assert resolved["gemini-2.5-flash"] != resolved["custom-priced-flash"] + finally: + _restore_model_cost_entries(model_keys) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6ee089ae036..6fae14ee6ec 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25112,6 +25112,8 @@ export interface components { } | null; /** Adaptive Router Default Model */ adaptive_router_default_model?: string | null; + /** Annotation Cost Per Page */ + annotation_cost_per_page?: number | null; /** Api Base */ api_base?: string | null; /** Api Key */ @@ -25156,8 +25158,12 @@ export interface components { cache_read_input_token_cost_above_200k_tokens?: number | null; /** Cache Read Input Token Cost Above 200K Tokens Priority */ cache_read_input_token_cost_above_200k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens */ + cache_read_input_token_cost_above_272k_tokens?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 512K Tokens */ + cache_read_input_token_cost_above_512k_tokens?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -25194,6 +25200,8 @@ export interface components { input_cost_per_image?: number | null; /** Input Cost Per Image Above 128K Tokens */ input_cost_per_image_above_128k_tokens?: number | null; + /** Input Cost Per Image Token */ + input_cost_per_image_token?: number | null; /** Input Cost Per Pixel */ input_cost_per_pixel?: number | null; /** Input Cost Per Query */ @@ -25208,8 +25216,12 @@ export interface components { input_cost_per_token_above_200k_tokens?: number | null; /** Input Cost Per Token Above 200K Tokens Priority */ input_cost_per_token_above_200k_tokens_priority?: number | null; + /** Input Cost Per Token Above 272K Tokens */ + input_cost_per_token_above_272k_tokens?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ input_cost_per_token_above_272k_tokens_priority?: number | null; + /** Input Cost Per Token Above 512K Tokens */ + input_cost_per_token_above_512k_tokens?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -25255,6 +25267,10 @@ export interface components { model_info?: { [key: string]: unknown; } | null; + /** Ocr Cost Per Credit */ + ocr_cost_per_credit?: number | null; + /** Ocr Cost Per Page */ + ocr_cost_per_page?: number | null; /** Organization */ organization?: string | null; /** Output Cost Per Audio Per Second */ @@ -25285,8 +25301,12 @@ export interface components { output_cost_per_token_above_200k_tokens?: number | null; /** Output Cost Per Token Above 200K Tokens Priority */ output_cost_per_token_above_200k_tokens_priority?: number | null; + /** Output Cost Per Token Above 272K Tokens */ + output_cost_per_token_above_272k_tokens?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ output_cost_per_token_above_272k_tokens_priority?: number | null; + /** Output Cost Per Token Above 512K Tokens */ + output_cost_per_token_above_512k_tokens?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */ @@ -25295,6 +25315,8 @@ export interface components { output_cost_per_token_priority?: number | null; /** Output Cost Per Video Per Second */ output_cost_per_video_per_second?: number | null; + /** Output Vector Size */ + output_vector_size?: number | null; /** Quality Router Config */ quality_router_config?: { [key: string]: unknown; @@ -25303,6 +25325,10 @@ export interface components { quality_router_default_model?: string | null; /** Region Name */ region_name?: string | null; + /** Regional Processing Uplift Multiplier Eu */ + regional_processing_uplift_multiplier_eu?: number | null; + /** Regional Processing Uplift Multiplier Us */ + regional_processing_uplift_multiplier_us?: number | null; /** Rpm */ rpm?: number | null; /** S3 Bucket Name */ @@ -32794,6 +32820,8 @@ export interface components { } | null; /** Adaptive Router Default Model */ adaptive_router_default_model?: string | null; + /** Annotation Cost Per Page */ + annotation_cost_per_page?: number | null; /** Api Base */ api_base?: string | null; /** Api Key */ @@ -32838,8 +32866,12 @@ export interface components { cache_read_input_token_cost_above_200k_tokens?: number | null; /** Cache Read Input Token Cost Above 200K Tokens Priority */ cache_read_input_token_cost_above_200k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens */ + cache_read_input_token_cost_above_272k_tokens?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Read Input Token Cost Above 512K Tokens */ + cache_read_input_token_cost_above_512k_tokens?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -32876,6 +32908,8 @@ export interface components { input_cost_per_image?: number | null; /** Input Cost Per Image Above 128K Tokens */ input_cost_per_image_above_128k_tokens?: number | null; + /** Input Cost Per Image Token */ + input_cost_per_image_token?: number | null; /** Input Cost Per Pixel */ input_cost_per_pixel?: number | null; /** Input Cost Per Query */ @@ -32890,8 +32924,12 @@ export interface components { input_cost_per_token_above_200k_tokens?: number | null; /** Input Cost Per Token Above 200K Tokens Priority */ input_cost_per_token_above_200k_tokens_priority?: number | null; + /** Input Cost Per Token Above 272K Tokens */ + input_cost_per_token_above_272k_tokens?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ input_cost_per_token_above_272k_tokens_priority?: number | null; + /** Input Cost Per Token Above 512K Tokens */ + input_cost_per_token_above_512k_tokens?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -32937,6 +32975,10 @@ export interface components { model_info?: { [key: string]: unknown; } | null; + /** Ocr Cost Per Credit */ + ocr_cost_per_credit?: number | null; + /** Ocr Cost Per Page */ + ocr_cost_per_page?: number | null; /** Organization */ organization?: string | null; /** Output Cost Per Audio Per Second */ @@ -32967,8 +33009,12 @@ export interface components { output_cost_per_token_above_200k_tokens?: number | null; /** Output Cost Per Token Above 200K Tokens Priority */ output_cost_per_token_above_200k_tokens_priority?: number | null; + /** Output Cost Per Token Above 272K Tokens */ + output_cost_per_token_above_272k_tokens?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ output_cost_per_token_above_272k_tokens_priority?: number | null; + /** Output Cost Per Token Above 512K Tokens */ + output_cost_per_token_above_512k_tokens?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */ @@ -32977,6 +33023,8 @@ export interface components { output_cost_per_token_priority?: number | null; /** Output Cost Per Video Per Second */ output_cost_per_video_per_second?: number | null; + /** Output Vector Size */ + output_vector_size?: number | null; /** Quality Router Config */ quality_router_config?: { [key: string]: unknown; @@ -32985,6 +33033,10 @@ export interface components { quality_router_default_model?: string | null; /** Region Name */ region_name?: string | null; + /** Regional Processing Uplift Multiplier Eu */ + regional_processing_uplift_multiplier_eu?: number | null; + /** Regional Processing Uplift Multiplier Us */ + regional_processing_uplift_multiplier_us?: number | null; /** Rpm */ rpm?: number | null; /** S3 Bucket Name */