From 27be70891d68daaf12e908e8faefab9335e16fde Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Thu, 24 Sep 2026 05:24:14 +0000 Subject: [PATCH] fix(cost): satisfy type-discipline gates and sync generated types for balanced tier --- .../crates/model-catalog/src/model_info.rs | 9 ++++ litellm/cost_calculator.py | 9 ++-- litellm/llms/openai_like/dynamic_config.py | 48 +++++++++---------- litellm/llms/openai_like/json_loader.py | 3 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 ++++++++ 5 files changed, 58 insertions(+), 31 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 77a8b768e38..7ada47f8b62 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -286,6 +286,9 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(default, skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_above_512k_tokens: Option, + /// Balanced service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_balanced: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_batches: Option, /// Flex service-tier rate for the same-named base field. @@ -380,6 +383,9 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(default, skip_serializing_if = "Option::is_none")] pub input_cost_per_token_above_512k_tokens: Option, + /// Balanced service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_balanced: Option, /// USD per prompt token via the provider's batch API. #[serde(default, skip_serializing_if = "Option::is_none")] pub input_cost_per_token_batches: Option, @@ -501,6 +507,9 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(default, skip_serializing_if = "Option::is_none")] pub output_cost_per_token_above_512k_tokens: Option, + /// Balanced service-tier rate for the same-named base field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_balanced: Option, /// USD per generated token via the provider's batch API. #[serde(default, skip_serializing_if = "Option::is_none")] pub output_cost_per_token_batches: Option, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 30e676dee03..0fb3e62f378 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -970,7 +970,7 @@ def _completion_window_value(metadata: object) -> str | None: pricing, so it returns None.""" if not isinstance(metadata, dict): return None - window: Final = cast(dict[str, object], metadata).get("completion_window") + window: Final[object] = metadata.get("completion_window") if isinstance(window, str) and window in (ServiceTier.FLEX.value, ServiceTier.BALANCED.value): return window return None @@ -980,9 +980,8 @@ def _service_tier_from_completion_window(optional_params: dict[str, object]) -> """Read ``metadata.completion_window`` from ``extra_body`` or a top-level ``metadata`` param (the two shapes callers use to pick a provider completion window directly).""" extra_body: Final = optional_params.get("extra_body") - return _completion_window_value( - cast(dict[str, object], extra_body).get("metadata") if isinstance(extra_body, dict) else None - ) or _completion_window_value(optional_params.get("metadata")) + extra_metadata: Final[object] = extra_body.get("metadata") if isinstance(extra_body, dict) else None + return _completion_window_value(extra_metadata) or _completion_window_value(optional_params.get("metadata")) def get_usage_object( @@ -1398,7 +1397,7 @@ def completion_cost( if service_tier is None and optional_params is not None: service_tier = _normalize_service_tier(optional_params.get("service_tier")) if service_tier is None: - service_tier = _service_tier_from_completion_window(cast(dict[str, object], optional_params)) + service_tier = _service_tier_from_completion_window(optional_params) service_tier = _normalize_service_tier(service_tier) diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index d4952ffe7f2..01b381b26f7 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -4,7 +4,7 @@ Dynamic configuration class generator for JSON-based providers. from collections.abc import Coroutine, Mapping from types import MappingProxyType -from typing import Any, Final, Literal, cast, overload +from typing import Any, Final, Literal, overload from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -33,13 +33,17 @@ def _apply_service_tier_as_completion_window(body: Mapping[str, object]) -> dict window: Final[str | None] = ( _SERVICE_TIER_TO_COMPLETION_WINDOW.get(service_tier.lower()) if isinstance(service_tier, str) else None ) - metadata: Final[dict[str, object]] = ( - cast(dict[str, object], body["metadata"]) if isinstance(body.get("metadata"), dict) else {} - ) - new_body: Final[dict[str, object]] = {key: value for key, value in body.items() if key != "service_tier"} + raw_metadata: Final = body.get("metadata") + metadata: Final[Mapping[str, object]] = raw_metadata if isinstance(raw_metadata, dict) else MappingProxyType({}) + new_body: Final[dict[str, object]] = { # mutable-ok: transform_request returns a plain dict + key: value for key, value in body.items() if key != "service_tier" + } if window is None or "completion_window" in metadata: return new_body - return {**new_body, "metadata": {**metadata, "completion_window": window}} + return { # mutable-ok: transform_request returns a plain dict + **new_body, + "metadata": {**metadata, "completion_window": window}, # mutable-ok: transform_request returns a plain dict + } def _service_tier_as_completion_window_enabled(provider: SimpleProviderConfig) -> bool: @@ -126,15 +130,12 @@ def create_config_class(provider: SimpleProviderConfig): litellm_params: dict[str, object], headers: dict[str, object], ) -> dict[str, object]: - body: Final[dict[str, object]] = cast( - dict[str, object], - super().transform_request( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - headers=headers, - ), + body: Final = super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, ) if _service_tier_as_completion_window_enabled(provider): return _apply_service_tier_as_completion_window(body) @@ -169,7 +170,7 @@ def create_config_class(provider: SimpleProviderConfig): param for param in (*base_params, *extra_params) if param not in excluded_params ) - return list(supported_params) + return list(supported_params) # mutable-ok: get_supported_openai_params contract returns a list def map_openai_params( self, @@ -282,15 +283,12 @@ def create_responses_config_class(provider: SimpleProviderConfig): ) -> dict[str, object]: if provider.special_handling.get("force_store_false"): response_api_optional_request_params["store"] = False - body: Final[dict[str, object]] = cast( - dict[str, object], - super().transform_responses_api_request( - model=model, - input=input, - response_api_optional_request_params=response_api_optional_request_params, - litellm_params=litellm_params, - headers=headers, - ), + body: Final = super().transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, ) if _service_tier_as_completion_window_enabled(provider): return _apply_service_tier_as_completion_window(body) diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 140f7cba1eb..406bd840a97 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -5,6 +5,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers. import json from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType from typing import Final from litellm._logging import verbose_logger @@ -21,7 +22,7 @@ class SimpleProviderConfig: self.base_class = data.get("base_class", "openai_gpt") self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) - self.special_handling: Mapping[str, object] = data.get("special_handling", {}) + self.special_handling: Mapping[str, object] = data.get("special_handling") or MappingProxyType({}) self.supported_endpoints = data.get("supported_endpoints", []) self.unsupported_params: Final = tuple(data.get("unsupported_params", ())) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bdfd4aec316..d63915e2dd2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31323,6 +31323,8 @@ export interface components { cache_creation_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Priority */ cache_creation_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Balanced */ + cache_creation_input_token_cost_balanced?: number | null; /** Cache Creation Input Token Cost Batches */ cache_creation_input_token_cost_batches?: number | null; /** Cache Creation Input Token Cost Flex */ @@ -31351,6 +31353,8 @@ export interface components { cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Balanced */ + cache_read_input_token_cost_balanced?: number | null; /** Cache Read Input Token Cost Batches */ cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ @@ -31429,6 +31433,8 @@ export interface components { input_cost_per_token_above_272k_tokens_priority?: number | null; /** Input Cost Per Token Above 512K Tokens */ input_cost_per_token_above_512k_tokens?: number | null; + /** Input Cost Per Token Balanced */ + input_cost_per_token_balanced?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -31516,6 +31522,8 @@ export interface components { output_cost_per_pixel?: number | null; /** Output Cost Per Reasoning Token */ output_cost_per_reasoning_token?: number | null; + /** Output Cost Per Reasoning Token Balanced */ + output_cost_per_reasoning_token_balanced?: number | null; /** Output Cost Per Reasoning Token Flex */ output_cost_per_reasoning_token_flex?: number | null; /** Output Cost Per Reasoning Token Priority */ @@ -31552,6 +31560,8 @@ export interface components { output_cost_per_token_above_272k_tokens_priority?: number | null; /** Output Cost Per Token Above 512K Tokens */ output_cost_per_token_above_512k_tokens?: number | null; + /** Output Cost Per Token Balanced */ + output_cost_per_token_balanced?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */ @@ -42218,6 +42228,8 @@ export interface components { cache_creation_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Priority */ cache_creation_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Balanced */ + cache_creation_input_token_cost_balanced?: number | null; /** Cache Creation Input Token Cost Batches */ cache_creation_input_token_cost_batches?: number | null; /** Cache Creation Input Token Cost Flex */ @@ -42246,6 +42258,8 @@ export interface components { cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Balanced */ + cache_read_input_token_cost_balanced?: number | null; /** Cache Read Input Token Cost Batches */ cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ @@ -42324,6 +42338,8 @@ export interface components { input_cost_per_token_above_272k_tokens_priority?: number | null; /** Input Cost Per Token Above 512K Tokens */ input_cost_per_token_above_512k_tokens?: number | null; + /** Input Cost Per Token Balanced */ + input_cost_per_token_balanced?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -42411,6 +42427,8 @@ export interface components { output_cost_per_pixel?: number | null; /** Output Cost Per Reasoning Token */ output_cost_per_reasoning_token?: number | null; + /** Output Cost Per Reasoning Token Balanced */ + output_cost_per_reasoning_token_balanced?: number | null; /** Output Cost Per Reasoning Token Flex */ output_cost_per_reasoning_token_flex?: number | null; /** Output Cost Per Reasoning Token Priority */ @@ -42447,6 +42465,8 @@ export interface components { output_cost_per_token_above_272k_tokens_priority?: number | null; /** Output Cost Per Token Above 512K Tokens */ output_cost_per_token_above_512k_tokens?: number | null; + /** Output Cost Per Token Balanced */ + output_cost_per_token_balanced?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */