diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 8f7549c1c6d..bcdbc8b1388 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -30,6 +30,7 @@ pub struct ModelInfoFields { pub cache_read_input_audio_token_cost_batches: Option, pub explicit_cache_read_input_token_cost: Option, pub prompt_cache_min_tokens: Option, + pub min_billed_input_tokens: Option, pub cache_read_input_token_cost_above_200k_tokens: Option, pub cache_read_input_token_cost_above_200k_tokens_priority: Option, pub cache_read_input_token_cost_above_272k_tokens: Option, diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 93c10a81524..6528573eb78 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -73,6 +73,8 @@ attributes! { max_tokens: u64 as Count, /// Free-form notes about the entry (e.g. pricing derivation). metadata: BTreeMap, + /// Input tokens billed per request even when the prompt is shorter. + min_billed_input_tokens: u64 as Count, /// Primary API surface / task type of the model. mode: Mode, /// Databricks units, not USD. diff --git a/litellm/__init__.py b/litellm/__init__.py index 3e8cddac12c..03484e9d8ac 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -689,6 +689,7 @@ kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared mo streamlake_models: Final[Set[str]] = set() zerogpu_models: Final[Set[str]] = set() v0_models: Set = set() +vispark_models: Set[str] = set() morph_models: Set = set() opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads lambda_ai_models: Set = set() @@ -990,6 +991,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje zerogpu_models.add(key) elif value.get("litellm_provider") == "v0": v0_models.add(key) + elif value.get("litellm_provider") == "vispark": + vispark_models.add(key) elif value.get("litellm_provider") == "morph": morph_models.add(key) elif value.get("litellm_provider") == "opencode_go": @@ -1166,6 +1169,7 @@ model_list = list( | streamlake_models | zerogpu_models | v0_models + | vispark_models | morph_models | opencode_go_models | lambda_ai_models @@ -1298,6 +1302,7 @@ def _build_models_by_provider() -> dict: "streamlake": streamlake_models, "zerogpu": zerogpu_models, "v0": v0_models, + "vispark": vispark_models, "morph": morph_models, "opencode_go": opencode_go_models, "lambda_ai": lambda_ai_models, diff --git a/litellm/constants.py b/litellm/constants.py index d02114f56b1..549693c2546 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1061,6 +1061,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://inference.coralbricks.ai/v1", "https://api.cortecs.ai/v1", "https://api.scx.ai/v1", + "https://api.lab.vispark.in/v1", "https://api.prisminference.com/v1", "https://api.reka.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", @@ -1139,6 +1140,7 @@ openai_compatible_providers: Final[list[str]] = [ "pinstripes", # Pinstripes - JSON-configured provider "openinfer", "darkbloom", + "vispark", "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", "coralbricks", @@ -1173,6 +1175,7 @@ openai_text_completion_compatible_providers: Final[list] = [ # providers that s "poe", "chutes", "v0", + "vispark", "lambda_ai", "hyperbolic", "wandb", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a4a9a5bee39..ea873a61f0d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -523,8 +523,13 @@ def cost_per_token( if _is_anthropic_style: _normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens + billed_prompt_tokens: Final = ( + max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider)) + if custom_cost_per_token is not None and _normalized_prompt_tokens > 0 + else _normalized_prompt_tokens + ) response_cost: Final = _cost_per_token_custom_pricing_helper( - prompt_tokens=_normalized_prompt_tokens, + prompt_tokens=billed_prompt_tokens, completion_tokens=completion_tokens, response_time_ms=response_time_ms, cached_tokens=_cache_read_tokens, @@ -1467,6 +1472,23 @@ def _store_cost_breakdown_in_logging_obj( # Don't fail the main cost calculation if breakdown storage fails +def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | None) -> bool: + from litellm.litellm_core_utils.litellm_logging import MODEL_INFO_RATE_KEYS, use_custom_pricing_for_model + + logged_params: Final = getattr(litellm_logging_obj, "litellm_params", None) + return ( + isinstance(logged_params, dict) + and use_custom_pricing_for_model(logged_params) + and not use_custom_pricing_for_model(logged_params, MODEL_INFO_RATE_KEYS) + ) + + +def overrides_provider_stated_cost( + custom_pricing: bool | None, litellm_logging_obj: LitellmLoggingObject | None +) -> bool: + return custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj) + + def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: if usage is None or getattr(usage, "cost", None) is None: return usage @@ -1638,8 +1660,9 @@ def completion_cost( cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 provider_usage_object: Final = get_usage_object(completion_response=completion_response) + sets_prices: Final = overrides_provider_stated_cost(custom_pricing, litellm_logging_obj) cost_per_token_usage_object: Final[Usage | None] = ( - _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object + _without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None @@ -1654,7 +1677,10 @@ def completion_cost( explicit_pricing: Final = custom_pricing is True or base_model is not None caller_set_prices: Final = ( - explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + sets_prices + or base_model is not None + or custom_cost_per_token is not None + or custom_cost_per_second is not None ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( @@ -2337,7 +2363,9 @@ def response_cost_calculator( getattr(response_object, HIDDEN_PARAMS_ATTR), ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) - if provider_response_cost is not None and not custom_pricing: + if provider_response_cost is not None and not overrides_provider_stated_cost( + custom_pricing, litellm_logging_obj + ): return provider_response_cost response_cost = completion_cost( @@ -2483,6 +2511,12 @@ def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: return raw_entry if isinstance(raw_entry, Mapping) else None +def _min_billed_input_tokens(model: str, custom_llm_provider: str | None) -> int: + cost_entry: Final = _raw_cost_map_entry(model) or _raw_cost_map_entry(f"{custom_llm_provider}/{model}") or {} + minimum: Final = cost_entry.get("min_billed_input_tokens") + return minimum if isinstance(minimum, int) else 0 + + def pricing_entry_for_cost_calc( model: str | None, completion_response: object | None, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 34c98efc22f..ec47a092032 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -353,6 +353,7 @@ def _get_provider_request_id(original_exception: Exception) -> str | None: _CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) _METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS +MODEL_INFO_RATE_KEYS: Final[frozenset[str]] = _MODEL_INFO_CUSTOM_PRICING_KEYS - {"min_billed_input_tokens"} _UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset( ("user_api_key_auth", "user_api_key_budget_reservation") ) @@ -6463,13 +6464,16 @@ def _deployment_model_info(litellm_params: dict | None) -> Mapping[str, object]: return MappingProxyType({}) -def use_custom_pricing_for_model(litellm_params: dict | None) -> bool: +def use_custom_pricing_for_model( + litellm_params: dict | None, model_info_pricing_keys: frozenset[str] = _MODEL_INFO_CUSTOM_PRICING_KEYS +) -> bool: """ Check if the model uses custom pricing Returns True if any custom pricing field is present in `litellm_params`, or if - any custom pricing or deployment-scoped pricing field (such as - ``off_peak_pricing``) is present in the metadata ``model_info`` + any of `model_info_pricing_keys` (by default every custom pricing or + deployment-scoped pricing field, such as ``off_peak_pricing``) is present in + the metadata ``model_info`` """ if litellm_params is None: return False @@ -6487,7 +6491,7 @@ def use_custom_pricing_for_model(litellm_params: dict | None) -> bool: model_info: dict = metadata.get("model_info", {}) or {} if model_info: - matching_keys = _MODEL_INFO_CUSTOM_PRICING_KEYS & model_info.keys() + matching_keys = model_info_pricing_keys & model_info.keys() for key in matching_keys: if model_info.get(key) is not None: return True diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..de483efc072 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -659,6 +659,12 @@ def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) - return off_peak +def unbilled_minimum_input_tokens(model_info: ModelInfo, prompt_tokens: int) -> int: + if prompt_tokens <= 0: + return 0 + return max((model_info.get("min_billed_input_tokens") or 0) - prompt_tokens, 0) + + def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates: """Swap in off-peak per-token rates when the current UTC time is inside one of the model's off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in @@ -1560,14 +1566,17 @@ def generic_cost_per_token( base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost ) - prompt_cost = _calculate_input_cost( - prompt_tokens_details=prompt_tokens_details, - model_info=resolved_model_info, - prompt_base_cost=base_costs.input_rate, - cache_read_cost=effective_cache_read_cost, - cache_creation_cost=base_costs.cache_creation_rate, - cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - service_tier=service_tier, + prompt_cost = ( + _calculate_input_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=resolved_model_info, + prompt_base_cost=base_costs.input_rate, + cache_read_cost=effective_cache_read_cost, + cache_creation_cost=base_costs.cache_creation_rate, + cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, + service_tier=service_tier, + ) + + unbilled_minimum_input_tokens(resolved_model_info, usage.prompt_tokens) * base_costs.input_rate ) ## CALCULATE OUTPUT COST diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b3f89b33d1d..ffe5b770eba 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2341,7 +2341,7 @@ def migrate_file_to_image_url( image_url_object: Final = ChatCompletionImageObject( type="image_url", image_url=ChatCompletionImageUrlObject( - url=cast(str, file_id or file_data), + url=cast(str, file_data or file_id), ), ) if format and isinstance(image_url_object["image_url"], dict): diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 55d6fb1a0e2..30995494eb9 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_completion_tokens_details, parse_prompt_tokens_details, resolve_explicit_cache_read_cost_rate, + unbilled_minimum_input_tokens, ) from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -151,4 +152,6 @@ def cost_per_token( rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate) ) - return _bill(breakdown, billed_rates) + prompt_cost, completion_cost = _bill(breakdown, billed_rates) + minimum_cost: Final = unbilled_minimum_input_tokens(model_info, usage.prompt_tokens) * billed_rates.input_rate + return prompt_cost + minimum_cost, completion_cost diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index d7aafe9f06e..e4754367baf 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -2,25 +2,107 @@ Dynamic configuration class generator for JSON-based providers. """ -from collections.abc import Coroutine +from collections.abc import Callable, Coroutine, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, overload +from pydantic import JsonValue, TypeAdapter + +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, + migrate_file_to_image_url, ) from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionUserMessage, + OpenAIMessageContentListBlock, +) from .json_loader import SimpleProviderConfig +_JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_REQUESTED_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_SCHEMA_MAP_KEYWORDS: Final = frozenset({"properties", "patternProperties", "$defs", "definitions", "dependentSchemas"}) + if TYPE_CHECKING: from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig +def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "user" or not isinstance(message["content"], list): + return message + content: Final = [ + migrate_file_to_image_url(part) if part["type"] == "file" else part for part in message["content"] + ] + updated: Final[ChatCompletionUserMessage] = {**message, "content": content} + return updated + + +def _is_unreadable_file_part(part: OpenAIMessageContentListBlock) -> bool: + return part["type"] == "file" and not part["file"].get("file_data") + + +def _without_unreadable_file_parts(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "user" or not isinstance(message["content"], list): + return message + content: Final = [part for part in message["content"] if not _is_unreadable_file_part(part)] + updated: Final[ChatCompletionUserMessage] = {**message, "content": content} + return updated + + +def _messages_with_readable_file_parts( + messages: Sequence[AllMessageValues], provider_slug: str, drop_params: bool +) -> tuple[AllMessageValues, ...]: + readable: Final = tuple(_without_unreadable_file_parts(message) for message in messages) + if readable == tuple(messages) or drop_params: + return readable + raise UnsupportedParamsError( + message=( + f"{provider_slug} cannot read file parts sent by file_id. " + "Send the file as file_data, or set `drop_params=True` to drop them" + ), + llm_provider=provider_slug, + ) + + +def _without_additional_properties( + node: JsonValue, keep: Callable[[JsonValue], bool], is_schema_map: bool = False +) -> JsonValue: + if isinstance(node, list): + return [_without_additional_properties(item, keep) for item in node] + if not isinstance(node, dict): + return node + return { + key: _without_additional_properties(item, keep, not is_schema_map and key in _SCHEMA_MAP_KEYWORDS) + for key, item in node.items() + if is_schema_map or key != "additionalProperties" or keep(item) + } + + +def _response_format_without_additional_properties( + response_format: JsonValue, provider_slug: str, model: str, drop_params: bool +) -> JsonValue: + without_flags: Final = _without_additional_properties(response_format, lambda item: not isinstance(item, bool)) + without_any: Final = _without_additional_properties(response_format, lambda _: False) + if without_flags == without_any or drop_params: + return without_any + raise UnsupportedParamsError( + message=( + f"{provider_slug} does not support schema-valued additionalProperties (map fields) in response_format. " + "To drop them, set `drop_params=True`" + ), + llm_provider=provider_slug, + model=model, + ) + + def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" @@ -46,14 +128,37 @@ def create_config_class(provider: SimpleProviderConfig): ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: """Transform messages based on special_handling config""" - # Handle content list to string conversion if configured - if provider.special_handling.get("convert_content_list_to_string"): - messages = handle_messages_with_content_list_to_str_conversion(messages) - + content_flattened: Final = ( + handle_messages_with_content_list_to_str_conversion(messages) + if provider.special_handling.get("convert_content_list_to_string") + else messages + ) + transformed: Final = ( + [_with_file_parts_as_image_url(message) for message in content_flattened] + if provider.special_handling.get("convert_file_parts_to_image_url") + else content_flattened + ) if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) - else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed, model=model, is_async=True) + return super()._transform_messages(messages=transformed, model=model, is_async=False) + + def _prompt_cache_ordered_messages( + self, + messages: list[AllMessageValues], # mutable-ok: overrides OpenAIGPTConfig's list signature + litellm_params: Mapping[str, object], + ) -> list[AllMessageValues]: # mutable-ok: overrides OpenAIGPTConfig's list signature + readable: Final = ( + list( + _messages_with_readable_file_parts( + messages, + provider_slug=provider.slug, + drop_params=drops_unsupported_params(litellm_params.get("drop_params")), + ) + ) + if provider.special_handling.get("convert_file_parts_to_image_url") + else messages + ) + return super()._prompt_cache_ordered_messages(readable, litellm_params) def _should_preserve_cache_control_for_endpoint( self, @@ -149,12 +254,23 @@ def create_config_class(provider: SimpleProviderConfig): supported_params: Final = self.get_supported_openai_params(model) # Apply supported params - for param, value in non_default_params.items(): + requested_params: Final = _REQUESTED_PARAMS_ADAPTER.validate_python(non_default_params) + for param, value in requested_params.items(): # Check parameter mappings first if param in provider.param_mappings: optional_params[provider.param_mappings[param]] = value elif param in supported_params: - optional_params[param] = value + optional_params[param] = ( + _response_format_without_additional_properties( + _JSON_VALUE_ADAPTER.validate_python(value), + provider_slug=provider.slug, + model=model, + drop_params=drop_params or litellm.drop_params is True, + ) + if param == "response_format" + and provider.special_handling.get("remove_additional_properties_from_response_format") + else value + ) # Apply temperature constraints if present if "temperature" in optional_params: diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 6e99b9e7fec..6b3fe763277 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -24,9 +24,9 @@ class SimpleProviderConfig: self.base_url = data["base_url"] self.api_key_env: str = data["api_key_env"] self.api_base_env: str | None = data.get("api_base_env") - self.base_class = data.get("base_class", "openai_gpt") - self.param_mappings = data.get("param_mappings", {}) - self.constraints = data.get("constraints", {}) + self.base_class: str = data.get("base_class", "openai_gpt") + self.param_mappings: Mapping[str, str] = data.get("param_mappings", {}) + self.constraints: Mapping[str, float] = data.get("constraints", {}) self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( data.get("special_handling", {}) ) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index ad47b334ed4..3cd839da7a5 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -292,5 +292,18 @@ "preserve_cache_control": true }, "supported_endpoints": ["/v1/chat/completions"] + }, + "vispark": { + "base_url": "https://api.lab.vispark.in/v1", + "api_key_env": "VISPARK_LAB_API_KEY", + "api_base_env": "VISPARK_LAB_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "special_handling": { + "convert_file_parts_to_image_url": true, + "remove_additional_properties_from_response_format": true + }, + "supported_endpoints": ["/v1/chat/completions", "/v1/completions"] } } diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..e46d66c247a 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -6,7 +6,11 @@ Helper util for handling perplexity-specific cost calculation from datetime import datetime from typing import Final -from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + TokenRates, + apply_off_peak_pricing, + unbilled_minimum_input_tokens, +) from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -67,7 +71,10 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non output_cost_per_token: Final = rates.output_rate ## CALCULATE INPUT COST - prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token + prompt_tokens: Final = usage.prompt_tokens or 0 + prompt_cost: float = ( + prompt_tokens + unbilled_minimum_input_tokens(model_info, prompt_tokens) + ) * input_cost_per_token ## ADD CITATION TOKENS COST (if present) citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0 diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6bbb63b6fc..e899c190888 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -49863,6 +49863,105 @@ "supports_tool_choice": true, "supports_vision": true }, + "vispark/vision-small": { + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 1.05e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "min_billed_input_tokens": 600, + "mode": "chat", + "output_cost_per_token": 3.16e-06, + "source": "https://api.lab.vispark.in/v1/models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-medium": { + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 4.21e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "min_billed_input_tokens": 600, + "mode": "chat", + "output_cost_per_token": 1.263e-05, + "source": "https://api.lab.vispark.in/v1/models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-large": { + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 7.37e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "min_billed_input_tokens": 600, + "mode": "chat", + "output_cost_per_token": 2.211e-05, + "source": "https://api.lab.vispark.in/v1/models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vercel_ai_gateway/alibaba/qwen-3-14b": { "input_cost_per_token": 8e-08, "litellm_provider": "vercel_ai_gateway", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1d487f33835..418d0531cc2 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -4228,6 +4228,32 @@ "generateContent": true } }, + "vispark": { + "display_name": "Vispark (`vispark`)", + "url": "https://lab.vispark.in/#vision", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "vercel_ai_gateway": { "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c1fc6479421..66f39fcd3c9 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE +from litellm.cost_calculator import overrides_provider_stated_cost from litellm.litellm_core_utils.core_helpers import ( native_messages_reported_cost, override_stamped_response_cost, @@ -346,7 +347,7 @@ class AnthropicPassthroughLoggingHandler: if isinstance(litellm_model_response, ModelResponse) else None ) - if reported_cost is not None and not custom_pricing: + if reported_cost is not None and not overrides_provider_stated_cost(custom_pricing, logging_obj): return reported_cost return litellm.completion_cost( completion_response=litellm_model_response, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 04e6c5d8ba9..e0db33092f3 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -4284,5 +4284,33 @@ } ], "default_model_placeholder": "bigmodel/glm-5.3" + }, + { + "provider": "VISPARK", + "provider_display_name": "Vispark", + "litellm_provider": "vispark", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.lab.vispark.in/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "vispark/vision-medium" } ] diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 57dcb74b36b..d09784bf564 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -4,7 +4,7 @@ import asyncio import json import math import time -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone from itertools import chain @@ -1352,7 +1352,7 @@ def _estimate_request_input_cost_for_model( model_info=model_info, input_tokens=input_tokens, ) - for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) + for model_info in _get_model_cost_infos(model=model, llm_router=llm_router, group_billing_minimum=min) ] valid_estimates: Final = [estimate for estimate in estimates if estimate is not None] return max(valid_estimates) if valid_estimates else None @@ -1521,6 +1521,7 @@ def _get_model_cost_info( def _get_model_cost_infos( model: str, llm_router: Router | None, + group_billing_minimum: Callable[[Sequence[int]], int] = max, ) -> Sequence[Mapping[str, object]]: """Cost-info candidates to estimate a request against for one model group. @@ -1528,52 +1529,64 @@ def _get_model_cost_infos( is unknown. Rather than guess, we estimate the cost against every eligible pricing shape in the group (the group's flat rates plus each deployment's tiered table) and let the caller reserve the maximum, so a cheaper sibling - deployment can never leave the request under-reserved. + deployment can never leave the request under-reserved. The deployments' + billed input minimums fold into one through ``group_billing_minimum``, the + largest for admission and the smallest for the input cost a disconnect is charged. """ try: - base: Final = _get_model_cost_info(model=model, llm_router=llm_router) - if base is None: + group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router) + if group_info is None: return [] - tiered_tables: Final = _get_deployment_tiered_pricing_tables(model=model, llm_router=llm_router) + deployment_infos: Final = _get_deployment_model_infos(model=model, llm_router=llm_router) except Exception: verbose_proxy_logger.debug( "Unable to load model cost info for budget reservation", exc_info=True, ) return [] - if not tiered_tables: - return [base] + billing_minimums: Final = tuple( + _to_int(deployment_info.get("min_billed_input_tokens")) or 0 for deployment_info in deployment_infos + ) + base: Final = ( + {**group_info, "min_billed_input_tokens": group_billing_minimum(billing_minimums)} + if any(billing_minimums) + else group_info + ) + tiered_tables: Final = [ + table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None + ] return [base, *({**base, "tiered_pricing": table} for table in tiered_tables)] -def _deployment_tiered_pricing_table( - deployment: DeploymentTypedDict, - llm_router: Router, -) -> Sequence[Mapping[str, object]] | None: - model_id: Final = _get_value(_get_value(deployment, "model_info"), "id") - backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model") - if not isinstance(model_id, str) or not isinstance(backend_model, str): - return None - deployment_model_info: Final = llm_router.registry().deployment_model_info(model_id, backend_model, cached=True) - if deployment_model_info is None: - return None - tiered_pricing: Final = deployment_model_info.get("tiered_pricing") +def _tiered_pricing_table(deployment_info: Mapping[str, object]) -> Sequence[Mapping[str, object]] | None: + tiered_pricing: Final = deployment_info.get("tiered_pricing") if isinstance(tiered_pricing, list) and tiered_pricing: return tiered_pricing return None -def _get_deployment_tiered_pricing_tables( +def _deployment_model_info( + deployment: DeploymentTypedDict, + llm_router: Router, +) -> Mapping[str, object] | None: + model_id: Final = _get_value(_get_value(deployment, "model_info"), "id") + backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model") + if not isinstance(model_id, str) or not isinstance(backend_model, str): + return None + return llm_router.registry().deployment_model_info(model_id, backend_model, cached=True) + + +def _get_deployment_model_infos( model: str, llm_router: Router | None, -) -> Sequence[Sequence[Mapping[str, object]]]: +) -> Sequence[Mapping[str, object]]: if llm_router is None: return [] deployments: Final = llm_router.registry().deployments(model) or [] return [ - table + deployment_info for deployment in deployments - if (table := _deployment_tiered_pricing_table(deployment, llm_router)) is not None + if (deployment_info := _deployment_model_info(deployment, llm_router)) is not None ] @@ -1627,14 +1640,10 @@ def _estimate_input_tokens( if input_tokens is not None else count_input_tokens_for_model(request_body=request_body, model=model) ) - if counted is not None: - return counted - - max_input_tokens: Final = _to_int(model_info.get("max_input_tokens")) - if max_input_tokens is not None: - return max_input_tokens - - return None + estimated: Final = counted if counted is not None else _to_int(model_info.get("max_input_tokens")) + if estimated is None: + return None + return max(estimated, _to_int(model_info.get("min_billed_input_tokens")) or 0) DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK: Final = 16384 diff --git a/litellm/router.py b/litellm/router.py index 77958686957..ca82f071992 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10673,19 +10673,20 @@ class Router(DeprecatedRouterMembers): model_info[field] = copy.deepcopy(backend_value) @staticmethod - def _inherit_builtin_base_rates_for_off_peak( - model_info: dict, # mutable-ok: cost-map entry filled in place + def _inherit_builtin_base_rates_for_scoped_pricing( + model_info: dict[str, object], # mutable-ok: cost-map entry filled in place backend_model: str, custom_llm_provider: str | None, ) -> None: """Fill missing pricing fields on a deployment entry that only sets - ``off_peak_pricing``, from the backend model's built-in cost map entry. + ``off_peak_pricing`` or ``min_billed_input_tokens``, from the backend + model's built-in cost map entry. Cost lookup selects the deployment-scoped entry over the shared backend entry only when the deployment entry carries a base pricing field, and - ``off_peak_pricing`` is deliberately kept off the shared entry, so a - deployment spelling out only its off-peak schedule would otherwise - never receive the discount. The backend model's entire canonical cost + both fields are deliberately kept off the shared entry, so a deployment + spelling out only its off-peak schedule or input minimum would otherwise + never have it applied. The backend model's entire canonical cost map entry is copied, field by field, so threshold, tiered, service-tier, cache, character, and per-second rates as well as companion billing fields like ``web_search_billing_unit`` and the @@ -10699,7 +10700,7 @@ class Router(DeprecatedRouterMembers): no-op when any base pricing field is already set or the backend model has no canonical entry. """ - if not model_info.get("off_peak_pricing"): + if not model_info.get("off_peak_pricing") and model_info.get("min_billed_input_tokens") is None: return if any( model_info.get(field) is not None @@ -11894,7 +11895,7 @@ class Router(DeprecatedRouterMembers): model_info.update( {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} ) - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, @@ -11915,7 +11916,17 @@ class Router(DeprecatedRouterMembers): backend_model=backend_model, custom_llm_provider=pricing_provider, ) - return model_info + backend_info: Final = ( + lookup_model_info_for_registration(model=backend_model, custom_llm_provider=pricing_provider) + if model_info.get("input_cost_per_token") is not None or model_info.get("tiered_pricing") is not None + else None + ) + billing_minimum: Final = ( + None + if backend_info is None or isinstance(backend_info, ModelInfoLookupMiss) + else backend_info.get("min_billed_input_tokens") + ) + return model_info if billing_minimum is None else {"min_billed_input_tokens": billing_minimum, **model_info} @staticmethod def _register_deployment_pricing(deployment: Deployment) -> None: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1e59ec0c2cb..cdae7ae08d5 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -367,6 +367,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None + min_billed_input_tokens: ReadOnly[int | None] off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None @@ -4145,7 +4146,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): ) -DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) +DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing", "min_billed_input_tokens"}) DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( { @@ -4510,6 +4511,7 @@ class LlmProviders(str, Enum): MOONSHOT = "moonshot" PUBLICAI = "publicai" V0 = "v0" + VISPARK = "vispark" MORPH = "morph" OPENCODE_GO = "opencode_go" LAMBDA_AI = "lambda_ai" diff --git a/litellm/utils.py b/litellm/utils.py index e46e8ca1d52..4cfef36675e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7124,6 +7124,7 @@ def get_model_info_helper( ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), + min_billed_input_tokens=_model_info.get("min_billed_input_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None ), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6bbb63b6fc..e899c190888 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -49863,6 +49863,105 @@ "supports_tool_choice": true, "supports_vision": true }, + "vispark/vision-small": { + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 1.05e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "min_billed_input_tokens": 600, + "mode": "chat", + "output_cost_per_token": 3.16e-06, + "source": "https://api.lab.vispark.in/v1/models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-medium": { + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 4.21e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "min_billed_input_tokens": 600, + "mode": "chat", + "output_cost_per_token": 1.263e-05, + "source": "https://api.lab.vispark.in/v1/models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-large": { + "cache_read_input_token_cost": 0.0, + "input_cost_per_token": 7.37e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "min_billed_input_tokens": 600, + "mode": "chat", + "output_cost_per_token": 2.211e-05, + "source": "https://api.lab.vispark.in/v1/models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vercel_ai_gateway/alibaba/qwen-3-14b": { "input_cost_per_token": 8e-08, "litellm_provider": "vercel_ai_gateway", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d487f33835..418d0531cc2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -4228,6 +4228,32 @@ "generateContent": true } }, + "vispark": { + "display_name": "Vispark (`vispark`)", + "url": "https://lab.vispark.in/#vision", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "vercel_ai_gateway": { "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..7d65f4111f3 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -84,6 +84,7 @@ IGNORE_FUNCTIONS = [ "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. + "_without_additional_properties", # bounded by the nesting depth of a pydantic-validated JsonValue response_format (a finite JSON tree, no cycles possible). ] diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b527250c4..3b093815931 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -99,6 +99,43 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re assert completion_cost == pytest.approx(380 * 9.7e-7) +@pytest.mark.parametrize( + ("prompt_tokens", "billed_prompt_tokens"), + [(0, 0), (10, 600), (600, 600), (900, 900)], + ids=[ + "no_counted_prompt_bills_no_minimum", + "below_the_minimum_bills_the_minimum", + "at_the_minimum", + "above_the_minimum_bills_as_reported", + ], +) +def test_generic_cost_per_token_bills_at_least_the_minimum_input_tokens( + prompt_tokens: int, billed_prompt_tokens: int +) -> None: + model_info: Final[ModelInfo] = { + "key": "minimum-input-fixture", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 2e-6, + "output_cost_per_token": 5e-6, + "min_billed_input_tokens": 600, + "litellm_provider": "openai", + "mode": "chat", + "supported_openai_params": None, + } + + prompt_cost, completion_cost = generic_cost_per_token( + model="minimum-input-fixture", + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7), + custom_llm_provider="openai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(billed_prompt_tokens * 2e-6) + assert completion_cost == pytest.approx(7 * 5e-6) + + @pytest.mark.parametrize( ("model_info", "expected_prompt_cost"), [ diff --git a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py index 551caab7153..7dadbe653fc 100644 --- a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py @@ -994,6 +994,24 @@ class TestDashscopeCostCalculator: assert prompt_cost == 0.0 assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10) + @pytest.mark.parametrize(("prompt_tokens", "billed_prompt_tokens"), [(10, 600), (900, 900)]) + def test_bills_at_least_the_minimum_input_tokens(self, prompt_tokens: int, billed_prompt_tokens: int): + litellm.model_cost["dashscope/qwen-minimum-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "min_billed_input_tokens": 600, + } + + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-minimum-test", + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7), + ) + + assert math.isclose(prompt_cost, billed_prompt_tokens * 1e-06, rel_tol=1e-10) + assert math.isclose(completion_cost, 7 * 2e-06, rel_tol=1e-10) + OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc) diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py new file mode 100644 index 00000000000..129a7083df0 --- /dev/null +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -0,0 +1,480 @@ +import json +from typing import Final + +import pytest +import respx + +import litellm + +_CHAT_COMPLETION: Final = { + "id": "chatcmpl_vispark", + "object": "chat.completion", + "created": 1_790_000_000, + "model": "vision-medium", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Vispark"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} + +_TEXT_COMPLETION: Final = { + "id": "cmpl_vispark", + "object": "text_completion", + "created": 1_790_000_000, + "model": "vision-small", + "choices": [{"index": 0, "text": "Hello from Vispark", "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} + + +def test_vispark_provider_resolution(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + monkeypatch.delenv("VISPARK_LAB_API_BASE", raising=False) + + model, provider, api_key, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "vision-medium" + assert provider == "vispark" + assert api_key == "vispark-env-key" + assert api_base == "https://api.lab.vispark.in/v1" + + +def test_vispark_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + monkeypatch.setenv("VISPARK_LAB_API_BASE", "https://vispark.env.example/v1") + + _, provider, api_key, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base="https://vispark.internal.example/v1", + api_key="vispark-explicit-key", + ) + + assert provider == "vispark" + assert api_key == "vispark-explicit-key" + assert api_base == "https://vispark.internal.example/v1" + + +def test_vispark_api_base_env_overrides_default(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + monkeypatch.setenv("VISPARK_LAB_API_BASE", "https://vispark.env.example/v1") + + _, provider, _, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert provider == "vispark" + assert api_base == "https://vispark.env.example/v1" + + +def test_vispark_api_base_autodetects_provider(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + + model, provider, api_key, api_base = get_llm_provider( + model="vision-medium", + custom_llm_provider=None, + api_base="https://api.lab.vispark.in/v1", + api_key=None, + ) + + assert model == "vision-medium" + assert provider == "vispark" + assert api_key == "vispark-env-key" + assert api_base == "https://api.lab.vispark.in/v1" + + +def test_vispark_chat_completion_request_maps_max_completion_tokens(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + response: Final = litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=64, + api_key="vispark-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer vispark-test-key" + assert body["model"] == "vision-medium" + assert body["max_tokens"] == 64 + assert "max_completion_tokens" not in body + assert response.choices[0].message.content == "Hello from Vispark" + + +@pytest.mark.parametrize("additional_properties", [False, True]) +def test_vispark_strips_additional_properties_from_a_json_schema_response_format(additional_properties: bool): + city_schema: Final = { + "type": "object", + "properties": {"name": {"type": "string"}}, + "additionalProperties": additional_properties, + } + schema: Final = { + "type": "object", + "properties": {"city": city_schema}, + "required": ["city"], + "additionalProperties": False, + } + response_format: Final = {"type": "json_schema", "json_schema": {"name": "City", "schema": schema, "strict": True}} + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Capital of France as JSON"}], + response_format=response_format, + api_key="vispark-test-key", + ) + + sent_format: Final = json.loads(route.calls.last.request.content)["response_format"] + assert "additionalProperties" not in json.dumps(sent_format) + assert sent_format["json_schema"]["strict"] is True + assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}} + assert city_schema["additionalProperties"] == additional_properties + + +def _map_field_response_format() -> dict[str, object]: + schema: Final = { + "type": "object", + "properties": {"facts": {"type": "object", "additionalProperties": {"type": "string"}}}, + "required": ["facts"], + "additionalProperties": False, + } + return {"type": "json_schema", "json_schema": {"name": "Facts", "schema": schema}} + + +def test_vispark_rejects_a_map_field_response_format_without_drop_params(): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + with pytest.raises(litellm.UnsupportedParamsError) as error: + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Facts about Paris as JSON"}], + response_format=_map_field_response_format(), + api_key="vispark-test-key", + ) + + assert error.value.status_code == 400 + assert route.call_count == 0 + + +def test_vispark_drops_a_map_field_schema_with_drop_params(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Facts about Paris as JSON"}], + response_format=_map_field_response_format(), + drop_params=True, + api_key="vispark-test-key", + ) + + sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"] + assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]} + + +def test_vispark_keeps_a_schema_property_named_additional_properties(): + schema: Final = { + "type": "object", + "properties": {"additionalProperties": {"type": "string", "additionalProperties": False}}, + "required": ["additionalProperties"], + "additionalProperties": False, + } + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Describe Paris as JSON"}], + response_format={"type": "json_schema", "json_schema": {"name": "Notes", "schema": schema}}, + api_key="vispark-test-key", + ) + + sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"] + assert sent_schema == { + "type": "object", + "properties": {"additionalProperties": {"type": "string"}}, + "required": ["additionalProperties"], + } + + +def _file_id_messages() -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file"}, + {"type": "file", "file": {"file_id": "file-abc123"}}, + ], + } + ] + + +def test_vispark_rejects_a_file_part_sent_by_file_id_without_drop_params(): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + with pytest.raises(litellm.BadRequestError, match="file_id") as error: + litellm.completion(model="vispark/vision-medium", messages=_file_id_messages(), api_key="vispark-test-key") + + assert error.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.asyncio +async def test_vispark_drops_a_file_part_sent_by_file_id_with_drop_params(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + await litellm.acompletion( + model="vispark/vision-medium", + messages=_file_id_messages(), + drop_params=True, + api_key="vispark-test-key", + ) + + sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"] + assert sent_content == [{"type": "text", "text": "Summarize this file"}] + + +def test_other_json_providers_keep_additional_properties_and_file_parts(): + file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}} + response_format: Final = { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}, + }, + } + with respx.mock() as upstream: + route: Final = upstream.post("https://api.scaleway.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) + litellm.completion( + model="scaleway/llama-3.3-70b-instruct", + messages=[{"role": "user", "content": [{"type": "text", "text": "Read this"}, file_part]}], + response_format=response_format, + api_key="scaleway-test-key", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == response_format + assert body["messages"][0]["content"][1] == file_part + + +_PDF_DATA_URL: Final = "data:application/pdf;base64,JVBERi0xLjQK" + + +@pytest.mark.parametrize( + "file", + [{"file_data": _PDF_DATA_URL}, {"file_id": "file-abc123", "file_data": _PDF_DATA_URL}], + ids=["file_data", "file_id_and_file_data"], +) +def test_vispark_sends_a_pdf_file_part_as_an_image_url(file: dict[str, str]): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What code is in this PDF?"}, + {"type": "file", "file": file}, + ], + } + ], + api_key="vispark-test-key", + ) + + sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"] + assert sent_content[0] == {"type": "text", "text": "What code is in this PDF?"} + assert sent_content[1]["type"] == "image_url" + assert sent_content[1]["image_url"]["url"] == _PDF_DATA_URL + assert all(part["type"] != "file" for part in sent_content) + + +def test_vispark_text_completion_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/completions").respond(200, json=_TEXT_COMPLETION) + response: Final = litellm.text_completion( + model="vispark/vision-small", + prompt="Say hello", + api_key="vispark-test-key", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert route.call_count == 1 + assert body["model"] == "vision-small" + assert body["prompt"] == "Say hello" + assert response.choices[0].text == "Hello from Vispark" + + +def test_vispark_cost_follows_the_cost_map(): + cost_entry: Final = litellm.model_cost["vispark/vision-large"] + prompt_tokens: Final = cost_entry["min_billed_input_tokens"] + 1_000 + + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-large", + custom_llm_provider="vispark", + prompt_tokens=prompt_tokens, + completion_tokens=500, + ) + + assert prompt_cost == pytest.approx(prompt_tokens * cost_entry["input_cost_per_token"]) + assert completion_cost == pytest.approx(500 * cost_entry["output_cost_per_token"]) + + +@pytest.mark.parametrize("model", ["vision-small", "vision-medium", "vision-large"]) +def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str): + cost_entry: Final = litellm.model_cost[f"vispark/{model}"] + minimum: Final = cost_entry["min_billed_input_tokens"] + + prompt_cost, _ = litellm.cost_per_token( + model=model, + custom_llm_provider="vispark", + prompt_tokens=minimum // 2, + completion_tokens=1, + ) + + assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"]) + + +@pytest.mark.parametrize( + "custom_pricing", + [ + {"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + {"tiered_pricing": [{"range": [0, 1_000_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}]}, + ], + ids=["flat", "tiered"], +) +def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens( + custom_pricing: dict[str, object], +): + minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vispark-custom-priced", + "litellm_params": { + "model": "vispark/vision-small", + "api_key": "vispark-test-key", + **custom_pricing, + }, + } + ] + ) + with respx.mock() as upstream: + upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) + response: Final = router.completion( + model="vispark-custom-priced", messages=[{"role": "user", "content": "Say hello"}] + ) + + completion_tokens: Final = _CHAT_COMPLETION["usage"]["completion_tokens"] + assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6) + + +def test_vispark_deployment_minimum_override_stays_on_its_own_deployment(): + builtin: Final = litellm.model_cost["vispark/vision-small"] + minimum: Final = builtin["min_billed_input_tokens"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vispark-no-minimum", + "litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"}, + "model_info": {"min_billed_input_tokens": 0}, + }, + { + "model_name": "vispark-default", + "litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"}, + }, + ] + ) + with respx.mock() as upstream: + upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json={**_CHAT_COMPLETION, "model": "vision-small"} + ) + no_minimum: Final = router.completion( + model="vispark-no-minimum", messages=[{"role": "user", "content": "Say hello"}] + ) + default: Final = router.completion(model="vispark-default", messages=[{"role": "user", "content": "Say hello"}]) + + usage: Final = _CHAT_COMPLETION["usage"] + completion_cost: Final = usage["completion_tokens"] * builtin["output_cost_per_token"] + assert no_minimum._hidden_params["response_cost"] == pytest.approx( + usage["prompt_tokens"] * builtin["input_cost_per_token"] + completion_cost + ) + assert default._hidden_params["response_cost"] == pytest.approx( + minimum * builtin["input_cost_per_token"] + completion_cost + ) + assert litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] == minimum + + +def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens(): + minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] + + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-small", + custom_llm_provider="vispark", + prompt_tokens=minimum // 2, + completion_tokens=3, + custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + ) + + assert prompt_cost == pytest.approx(minimum * 2e-6) + assert completion_cost == pytest.approx(3 * 5e-6) + + +def test_vispark_explicit_custom_cost_per_token_bills_no_minimum_without_a_counted_prompt(): + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-small", + custom_llm_provider="vispark", + prompt_tokens=0, + completion_tokens=3, + custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + ) + + assert prompt_cost == 0.0 + assert completion_cost == pytest.approx(3 * 5e-6) + + +def test_vispark_models_are_listed_for_the_provider(): + from litellm.utils import get_valid_models + + expected: Final = {name for name, entry in litellm.model_cost.items() if entry.get("litellm_provider") == "vispark"} + + assert expected + assert set(litellm.models_by_provider["vispark"]) == expected + assert expected <= set(get_valid_models(custom_llm_provider="vispark")) diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..9f82514d6f4 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -140,6 +140,26 @@ class TestPerplexityCostCalculator: assert prompt_cost == 0.0 assert completion_cost == 0.008 + @pytest.mark.parametrize(("prompt_tokens", "billed_prompt_tokens"), [(10, 600), (900, 900)]) + def test_manual_pricing_bills_at_least_the_minimum_input_tokens( + self, prompt_tokens: int, billed_prompt_tokens: int + ): + litellm.model_cost["perplexity/sonar-minimum-test"] = { + "litellm_provider": "perplexity", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "min_billed_input_tokens": 600, + } + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-minimum-test", + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7), + ) + + assert math.isclose(prompt_cost, billed_prompt_tokens * 1e-06, rel_tol=1e-10) + assert math.isclose(completion_cost, 7 * 2e-06, rel_tol=1e-10) + OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 47b6af9aeb3..59743bfc522 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2951,6 +2951,24 @@ def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delt assert result["result"]._hidden_params["response_cost"] == 0.00412 +def test_streamed_messages_from_a_deployment_that_only_sets_a_billing_minimum_bill_the_reported_cost(): + logging_obj = _go_logging_obj() + logging_obj.litellm_params = {"metadata": {"model_info": {"min_billed_input_tokens": 100000}}} + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=_go_stream_chunks(), + end_time=datetime.now(), + ) + + assert result["kwargs"]["response_cost"] == 0.00412 + + def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field(): logging_obj = _go_logging_obj() logging_obj.model_call_details["custom_llm_provider"] = "anthropic" diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 035b7d9dcd3..5918e17be79 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -25,6 +25,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, + estimate_request_input_cost, estimate_request_max_cost, get_budget_window_start, release_unbound_budget_reservation, @@ -221,6 +222,73 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) +@pytest.mark.parametrize("deployment_minimum", [None, 0]) +def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_minimum: int | None) -> None: + backend_model: Final = "vispark/vision-small" + backend_info: Final = litellm.get_model_info(model=backend_model) + built_in_minimum: Final = backend_info.get("min_billed_input_tokens") + input_cost_per_token: Final = backend_info.get("input_cost_per_token") + assert built_in_minimum is not None and built_in_minimum > 0 + assert input_cost_per_token is not None + router: Final = Router( + model_list=[ + Deployment( + model_name="minimum-group", + litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"), + model_info=ModelInfo(id="minimum-deployment", min_billed_input_tokens=deployment_minimum), + ).model_dump() + ] + ) + prompt_tokens: Final = 12 + + reserved: Final = estimate_request_input_cost( + request_body={"model": "minimum-group", "messages": [{"role": "user", "content": "hi"}]}, + route="/chat/completions", + llm_router=router, + input_token_counts={"minimum-group": prompt_tokens}, + ) + + billed_tokens: Final = built_in_minimum if deployment_minimum is None else max(prompt_tokens, deployment_minimum) + assert reserved is not None + assert math.isclose(reserved, billed_tokens * input_cost_per_token) + + +@pytest.mark.parametrize("sibling_minimum", [0, None]) +def test_disconnect_charge_is_only_the_minimum_every_deployment_in_the_group_bills(sibling_minimum: int | None) -> None: + backend_model: Final = "vispark/vision-small" + backend_info: Final = litellm.get_model_info(model=backend_model) + built_in_minimum: Final = backend_info.get("min_billed_input_tokens") + input_cost_per_token: Final = backend_info.get("input_cost_per_token") + assert built_in_minimum is not None and built_in_minimum > 0 + assert input_cost_per_token is not None + largest_minimum: Final = built_in_minimum * 100 + router: Final = Router( + model_list=[ + Deployment( + model_name="mixed-minimum-group", + litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"), + model_info=ModelInfo(id=deployment_id, min_billed_input_tokens=minimum), + ).model_dump() + for deployment_id, minimum in (("largest-minimum", largest_minimum), ("sibling", sibling_minimum)) + ] + ) + prompt_tokens: Final = 12 + request_body: Final = {"model": "mixed-minimum-group", "messages": [{"role": "user", "content": "hi"}]} + token_counts: Final = {"mixed-minimum-group": prompt_tokens} + + disconnect_charge: Final = estimate_request_input_cost( + request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts + ) + admission_estimate: Final = estimate_request_max_cost( + request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts + ) + + shared_minimum: Final = built_in_minimum if sibling_minimum is None else sibling_minimum + assert disconnect_charge is not None and admission_estimate is not None + assert math.isclose(disconnect_charge, max(prompt_tokens, shared_minimum) * input_cost_per_token) + assert admission_estimate >= largest_minimum * input_cost_per_token + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", diff --git a/tests/unit/proxy/test_budget_reservation.py b/tests/unit/proxy/test_budget_reservation.py index 535c3da8c3a..4238992589b 100644 --- a/tests/unit/proxy/test_budget_reservation.py +++ b/tests/unit/proxy/test_budget_reservation.py @@ -1343,8 +1343,8 @@ def test_reservation_uses_most_expensive_deployment_in_group(): return_value={"max_output_tokens": 200000}, ), patch( - "litellm.proxy.spend_tracking.budget_reservation._get_deployment_tiered_pricing_tables", - return_value=[cheap, expensive], + "litellm.proxy.spend_tracking.budget_reservation._get_deployment_model_infos", + return_value=[{"tiered_pricing": cheap}, {"tiered_pricing": expensive}], ), patch( "litellm.proxy.spend_tracking.budget_reservation._estimate_input_tokens", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 84f6dab9eff..c6bedb758bd 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10589,3 +10589,50 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) + + +@pytest.mark.parametrize( + ("model", "chat_url", "usage_cost", "stated_cost"), + [ + ( + "perplexity/sonar", + "https://api.perplexity.ai/chat/completions", + {"request_cost": 0.0025, "total_cost": 0.00266}, + 0.00266, + ), + ("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721), + ("openrouter/openai/gpt-5.4-nano", "https://openrouter.ai/api/v1/chat/completions", 0.00721, 0.00721), + ], + ids=["perplexity", "kyma", "openrouter"], +) +def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost( + _local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float +): + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "deployment-with-minimum", + "litellm_params": {"model": model, "api_key": "provider-test-key"}, + "model_info": {"min_billed_input_tokens": 1000}, + } + ] + ) + with respx.mock() as upstream: + upstream.post(chat_url).respond( + 200, + json={ + "id": "chatcmpl_minimum", + "object": "chat.completion", + "created": 1_790_000_000, + "model": "sonar", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Paris"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8, "cost": usage_cost}, + }, + ) + response: Final = router.completion( + model="deployment-with-minimum", messages=[{"role": "user", "content": "Capital of France?"}] + ) + + assert response._hidden_params["response_cost"] == pytest.approx(stated_cost) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 91f0b2339c7..76fcf0375cc 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -1694,7 +1694,7 @@ def test_cost_map_backend_model_uses_canonical_model_precedence( assert Router._cost_map_backend_model(deployment) == expected -def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): +def test_inherit_builtin_base_rates_for_scoped_pricing_fills_missing_rates(): """Direct unit test of the helper: an entry carrying only an off_peak_pricing block inherits the backend model's built-in base token rates, so cost lookup via the deployment id can bill standard rates @@ -1709,7 +1709,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): } model_info = {"off_peak_pricing": off_peak_block} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="openai", @@ -1720,7 +1720,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): assert model_info["off_peak_pricing"] == off_peak_block -def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): +def test_inherit_builtin_base_rates_for_scoped_pricing_carries_threshold_rates(): """A backend with above-threshold pricing hands the whole rate structure to the deployment entry, so peak-hour billing of large prompts through that entry matches the shared backend entry instead of flattening to the base @@ -1734,7 +1734,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="gemini", @@ -1749,7 +1749,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): ) -def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields(): +def test_inherit_builtin_base_rates_for_scoped_pricing_carries_companion_billing_fields(): """Billing rules that are not literal cost rates, like the web search billing unit, must ride along, or grounding and regional uplifts would bill differently through the deployment entry than through the shared @@ -1763,7 +1763,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=None, @@ -1773,7 +1773,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"] -def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero(): +def test_inherit_builtin_base_rates_for_scoped_pricing_tiered_only_backend_stores_no_zero(): """A tiered-only backend has no flat token rates; get_model_info synthesizes zeros for them, and storing those would mark the deployment explicitly priced free. The tier table itself must carry over as an isolated copy so @@ -1786,7 +1786,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="dashscope", @@ -1803,7 +1803,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z assert raw_tiers[0] == original_first_tier -def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): +def test_inherit_builtin_base_rates_for_scoped_pricing_leaves_explicit_rates_alone(): """An entry that sets its own base rate beside the block already counts as a full custom pricing entry; the helper must not mix builtin rates into it. """ @@ -1812,7 +1812,7 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): "input_cost_per_token": 3e-06, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model="gpt-4o-mini", custom_llm_provider="openai", @@ -1822,12 +1822,12 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): assert "output_cost_per_token" not in model_info -def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend(): +def test_inherit_builtin_base_rates_for_scoped_pricing_noop_without_block_or_backend(): """Nothing happens without an off_peak_pricing block, and an unmapped backend model leaves the entry unchanged rather than raising. """ plain_info = {"id": "dep-1"} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=plain_info, backend_model="gpt-4o-mini", custom_llm_provider="openai", @@ -1835,7 +1835,7 @@ def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend() assert plain_info == {"id": "dep-1"} off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=off_peak_info, backend_model="this-backend-model-does-not-exist-x9y8z7", custom_llm_provider=None, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 243a20a2551..9f0fa79bcb3 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1166,6 +1166,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "prompt_cache_min_tokens": {"type": "number"}, + "min_billed_input_tokens": {"type": "number"}, "supports_prompt_cache_breakpoint": {"type": "boolean"}, "supports_thinking_cache_preservation": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"},