mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 0c0203f4e1 into 9252709cab
This commit is contained in:
commit
8078aaee7a
34 changed files with 1281 additions and 92 deletions
|
|
@ -30,6 +30,7 @@ pub struct ModelInfoFields {
|
|||
pub cache_read_input_audio_token_cost_batches: Option<f64>,
|
||||
pub explicit_cache_read_input_token_cost: Option<f64>,
|
||||
pub prompt_cache_min_tokens: Option<i64>,
|
||||
pub min_billed_input_tokens: Option<i64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
|
||||
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
|
||||
|
|
|
|||
|
|
@ -73,6 +73,8 @@ attributes! {
|
|||
max_tokens: u64 as Count,
|
||||
/// Free-form notes about the entry (e.g. pricing derivation).
|
||||
metadata: BTreeMap<String, Value>,
|
||||
/// Input tokens billed per request even when the prompt is shorter.
|
||||
min_billed_input_tokens: u64 as Count,
|
||||
/// Primary API surface / task type of the model.
|
||||
mode: Mode,
|
||||
/// Databricks units, not USD.
|
||||
|
|
|
|||
|
|
@ -689,6 +689,7 @@ kyma_models: Final[set[str]] = set() # mutable-ok: populated when the shared mo
|
|||
streamlake_models: Final[Set[str]] = set()
|
||||
zerogpu_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
vispark_models: Set[str] = set()
|
||||
morph_models: Set = set()
|
||||
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
lambda_ai_models: Set = set()
|
||||
|
|
@ -990,6 +991,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
zerogpu_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "vispark":
|
||||
vispark_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
morph_models.add(key)
|
||||
elif value.get("litellm_provider") == "opencode_go":
|
||||
|
|
@ -1166,6 +1169,7 @@ model_list = list(
|
|||
| streamlake_models
|
||||
| zerogpu_models
|
||||
| v0_models
|
||||
| vispark_models
|
||||
| morph_models
|
||||
| opencode_go_models
|
||||
| lambda_ai_models
|
||||
|
|
@ -1298,6 +1302,7 @@ def _build_models_by_provider() -> dict:
|
|||
"streamlake": streamlake_models,
|
||||
"zerogpu": zerogpu_models,
|
||||
"v0": v0_models,
|
||||
"vispark": vispark_models,
|
||||
"morph": morph_models,
|
||||
"opencode_go": opencode_go_models,
|
||||
"lambda_ai": lambda_ai_models,
|
||||
|
|
|
|||
|
|
@ -1061,6 +1061,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"https://inference.coralbricks.ai/v1",
|
||||
"https://api.cortecs.ai/v1",
|
||||
"https://api.scx.ai/v1",
|
||||
"https://api.lab.vispark.in/v1",
|
||||
"https://api.prisminference.com/v1",
|
||||
"https://api.reka.ai/v1",
|
||||
"https://gigachat.devices.sberbank.ru/api/v1",
|
||||
|
|
@ -1139,6 +1140,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"pinstripes", # Pinstripes - JSON-configured provider
|
||||
"openinfer",
|
||||
"darkbloom",
|
||||
"vispark",
|
||||
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
|
||||
"cognition",
|
||||
"coralbricks",
|
||||
|
|
@ -1173,6 +1175,7 @@ openai_text_completion_compatible_providers: Final[list] = [ # providers that s
|
|||
"poe",
|
||||
"chutes",
|
||||
"v0",
|
||||
"vispark",
|
||||
"lambda_ai",
|
||||
"hyperbolic",
|
||||
"wandb",
|
||||
|
|
|
|||
|
|
@ -523,8 +523,13 @@ def cost_per_token(
|
|||
if _is_anthropic_style:
|
||||
_normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
|
||||
|
||||
billed_prompt_tokens: Final = (
|
||||
max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider))
|
||||
if custom_cost_per_token is not None and _normalized_prompt_tokens > 0
|
||||
else _normalized_prompt_tokens
|
||||
)
|
||||
response_cost: Final = _cost_per_token_custom_pricing_helper(
|
||||
prompt_tokens=_normalized_prompt_tokens,
|
||||
prompt_tokens=billed_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time_ms=response_time_ms,
|
||||
cached_tokens=_cache_read_tokens,
|
||||
|
|
@ -1467,6 +1472,23 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
# Don't fail the main cost calculation if breakdown storage fails
|
||||
|
||||
|
||||
def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | None) -> bool:
|
||||
from litellm.litellm_core_utils.litellm_logging import MODEL_INFO_RATE_KEYS, use_custom_pricing_for_model
|
||||
|
||||
logged_params: Final = getattr(litellm_logging_obj, "litellm_params", None)
|
||||
return (
|
||||
isinstance(logged_params, dict)
|
||||
and use_custom_pricing_for_model(logged_params)
|
||||
and not use_custom_pricing_for_model(logged_params, MODEL_INFO_RATE_KEYS)
|
||||
)
|
||||
|
||||
|
||||
def overrides_provider_stated_cost(
|
||||
custom_pricing: bool | None, litellm_logging_obj: LitellmLoggingObject | None
|
||||
) -> bool:
|
||||
return custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj)
|
||||
|
||||
|
||||
def _without_provider_stated_cost(usage: Usage | None) -> Usage | None:
|
||||
if usage is None or getattr(usage, "cost", None) is None:
|
||||
return usage
|
||||
|
|
@ -1638,8 +1660,9 @@ def completion_cost(
|
|||
cache_read_input_tokens: int | None = None
|
||||
audio_transcription_file_duration: float = 0.0
|
||||
provider_usage_object: Final = get_usage_object(completion_response=completion_response)
|
||||
sets_prices: Final = overrides_provider_stated_cost(custom_pricing, litellm_logging_obj)
|
||||
cost_per_token_usage_object: Final[Usage | None] = (
|
||||
_without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object
|
||||
_without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object
|
||||
)
|
||||
rerank_billed_units: RerankBilledUnits | None = None
|
||||
|
||||
|
|
@ -1654,7 +1677,10 @@ def completion_cost(
|
|||
|
||||
explicit_pricing: Final = custom_pricing is True or base_model is not None
|
||||
caller_set_prices: Final = (
|
||||
explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None
|
||||
sets_prices
|
||||
or base_model is not None
|
||||
or custom_cost_per_token is not None
|
||||
or custom_cost_per_second is not None
|
||||
)
|
||||
custom_priced_request_model: Final = model if custom_pricing is True else None
|
||||
configured_cost_per_query: Final = (
|
||||
|
|
@ -2337,7 +2363,9 @@ def response_cost_calculator(
|
|||
getattr(response_object, HIDDEN_PARAMS_ATTR),
|
||||
)
|
||||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
if provider_response_cost is not None and not overrides_provider_stated_cost(
|
||||
custom_pricing, litellm_logging_obj
|
||||
):
|
||||
return provider_response_cost
|
||||
|
||||
response_cost = completion_cost(
|
||||
|
|
@ -2483,6 +2511,12 @@ def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
|
|||
return raw_entry if isinstance(raw_entry, Mapping) else None
|
||||
|
||||
|
||||
def _min_billed_input_tokens(model: str, custom_llm_provider: str | None) -> int:
|
||||
cost_entry: Final = _raw_cost_map_entry(model) or _raw_cost_map_entry(f"{custom_llm_provider}/{model}") or {}
|
||||
minimum: Final = cost_entry.get("min_billed_input_tokens")
|
||||
return minimum if isinstance(minimum, int) else 0
|
||||
|
||||
|
||||
def pricing_entry_for_cost_calc(
|
||||
model: str | None,
|
||||
completion_response: object | None,
|
||||
|
|
|
|||
|
|
@ -353,6 +353,7 @@ def _get_provider_request_id(original_exception: Exception) -> str | None:
|
|||
_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
|
||||
_METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS
|
||||
MODEL_INFO_RATE_KEYS: Final[frozenset[str]] = _MODEL_INFO_CUSTOM_PRICING_KEYS - {"min_billed_input_tokens"}
|
||||
_UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset(
|
||||
("user_api_key_auth", "user_api_key_budget_reservation")
|
||||
)
|
||||
|
|
@ -6463,13 +6464,16 @@ def _deployment_model_info(litellm_params: dict | None) -> Mapping[str, object]:
|
|||
return MappingProxyType({})
|
||||
|
||||
|
||||
def use_custom_pricing_for_model(litellm_params: dict | None) -> bool:
|
||||
def use_custom_pricing_for_model(
|
||||
litellm_params: dict | None, model_info_pricing_keys: frozenset[str] = _MODEL_INFO_CUSTOM_PRICING_KEYS
|
||||
) -> bool:
|
||||
"""
|
||||
Check if the model uses custom pricing
|
||||
|
||||
Returns True if any custom pricing field is present in `litellm_params`, or if
|
||||
any custom pricing or deployment-scoped pricing field (such as
|
||||
``off_peak_pricing``) is present in the metadata ``model_info``
|
||||
any of `model_info_pricing_keys` (by default every custom pricing or
|
||||
deployment-scoped pricing field, such as ``off_peak_pricing``) is present in
|
||||
the metadata ``model_info``
|
||||
"""
|
||||
if litellm_params is None:
|
||||
return False
|
||||
|
|
@ -6487,7 +6491,7 @@ def use_custom_pricing_for_model(litellm_params: dict | None) -> bool:
|
|||
model_info: dict = metadata.get("model_info", {}) or {}
|
||||
|
||||
if model_info:
|
||||
matching_keys = _MODEL_INFO_CUSTOM_PRICING_KEYS & model_info.keys()
|
||||
matching_keys = model_info_pricing_keys & model_info.keys()
|
||||
for key in matching_keys:
|
||||
if model_info.get(key) is not None:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -659,6 +659,12 @@ def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) -
|
|||
return off_peak
|
||||
|
||||
|
||||
def unbilled_minimum_input_tokens(model_info: ModelInfo, prompt_tokens: int) -> int:
|
||||
if prompt_tokens <= 0:
|
||||
return 0
|
||||
return max((model_info.get("min_billed_input_tokens") or 0) - prompt_tokens, 0)
|
||||
|
||||
|
||||
def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates:
|
||||
"""Swap in off-peak per-token rates when the current UTC time is inside one of the model's
|
||||
off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in
|
||||
|
|
@ -1560,14 +1566,17 @@ def generic_cost_per_token(
|
|||
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
|
||||
)
|
||||
|
||||
prompt_cost = _calculate_input_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=resolved_model_info,
|
||||
prompt_base_cost=base_costs.input_rate,
|
||||
cache_read_cost=effective_cache_read_cost,
|
||||
cache_creation_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
service_tier=service_tier,
|
||||
prompt_cost = (
|
||||
_calculate_input_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=resolved_model_info,
|
||||
prompt_base_cost=base_costs.input_rate,
|
||||
cache_read_cost=effective_cache_read_cost,
|
||||
cache_creation_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
+ unbilled_minimum_input_tokens(resolved_model_info, usage.prompt_tokens) * base_costs.input_rate
|
||||
)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
|
|
|
|||
|
|
@ -2341,7 +2341,7 @@ def migrate_file_to_image_url(
|
|||
image_url_object: Final = ChatCompletionImageObject(
|
||||
type="image_url",
|
||||
image_url=ChatCompletionImageUrlObject(
|
||||
url=cast(str, file_id or file_data),
|
||||
url=cast(str, file_data or file_id),
|
||||
),
|
||||
)
|
||||
if format and isinstance(image_url_object["image_url"], dict):
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
parse_completion_tokens_details,
|
||||
parse_prompt_tokens_details,
|
||||
resolve_explicit_cache_read_cost_rate,
|
||||
unbilled_minimum_input_tokens,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
|
@ -151,4 +152,6 @@ def cost_per_token(
|
|||
rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate)
|
||||
)
|
||||
|
||||
return _bill(breakdown, billed_rates)
|
||||
prompt_cost, completion_cost = _bill(breakdown, billed_rates)
|
||||
minimum_cost: Final = unbilled_minimum_input_tokens(model_info, usage.prompt_tokens) * billed_rates.input_rate
|
||||
return prompt_cost + minimum_cost, completion_cost
|
||||
|
|
|
|||
|
|
@ -2,25 +2,107 @@
|
|||
Dynamic configuration class generator for JSON-based providers.
|
||||
"""
|
||||
|
||||
from collections.abc import Coroutine
|
||||
from collections.abc import Callable, Coroutine, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final, Literal, overload
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
migrate_file_to_image_url,
|
||||
)
|
||||
from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionUserMessage,
|
||||
OpenAIMessageContentListBlock,
|
||||
)
|
||||
|
||||
from .json_loader import SimpleProviderConfig
|
||||
|
||||
_JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
_REQUESTED_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
|
||||
_SCHEMA_MAP_KEYWORDS: Final = frozenset({"properties", "patternProperties", "$defs", "definitions", "dependentSchemas"})
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
|
||||
|
||||
|
||||
def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues:
|
||||
if message["role"] != "user" or not isinstance(message["content"], list):
|
||||
return message
|
||||
content: Final = [
|
||||
migrate_file_to_image_url(part) if part["type"] == "file" else part for part in message["content"]
|
||||
]
|
||||
updated: Final[ChatCompletionUserMessage] = {**message, "content": content}
|
||||
return updated
|
||||
|
||||
|
||||
def _is_unreadable_file_part(part: OpenAIMessageContentListBlock) -> bool:
|
||||
return part["type"] == "file" and not part["file"].get("file_data")
|
||||
|
||||
|
||||
def _without_unreadable_file_parts(message: AllMessageValues) -> AllMessageValues:
|
||||
if message["role"] != "user" or not isinstance(message["content"], list):
|
||||
return message
|
||||
content: Final = [part for part in message["content"] if not _is_unreadable_file_part(part)]
|
||||
updated: Final[ChatCompletionUserMessage] = {**message, "content": content}
|
||||
return updated
|
||||
|
||||
|
||||
def _messages_with_readable_file_parts(
|
||||
messages: Sequence[AllMessageValues], provider_slug: str, drop_params: bool
|
||||
) -> tuple[AllMessageValues, ...]:
|
||||
readable: Final = tuple(_without_unreadable_file_parts(message) for message in messages)
|
||||
if readable == tuple(messages) or drop_params:
|
||||
return readable
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider_slug} cannot read file parts sent by file_id. "
|
||||
"Send the file as file_data, or set `drop_params=True` to drop them"
|
||||
),
|
||||
llm_provider=provider_slug,
|
||||
)
|
||||
|
||||
|
||||
def _without_additional_properties(
|
||||
node: JsonValue, keep: Callable[[JsonValue], bool], is_schema_map: bool = False
|
||||
) -> JsonValue:
|
||||
if isinstance(node, list):
|
||||
return [_without_additional_properties(item, keep) for item in node]
|
||||
if not isinstance(node, dict):
|
||||
return node
|
||||
return {
|
||||
key: _without_additional_properties(item, keep, not is_schema_map and key in _SCHEMA_MAP_KEYWORDS)
|
||||
for key, item in node.items()
|
||||
if is_schema_map or key != "additionalProperties" or keep(item)
|
||||
}
|
||||
|
||||
|
||||
def _response_format_without_additional_properties(
|
||||
response_format: JsonValue, provider_slug: str, model: str, drop_params: bool
|
||||
) -> JsonValue:
|
||||
without_flags: Final = _without_additional_properties(response_format, lambda item: not isinstance(item, bool))
|
||||
without_any: Final = _without_additional_properties(response_format, lambda _: False)
|
||||
if without_flags == without_any or drop_params:
|
||||
return without_any
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider_slug} does not support schema-valued additionalProperties (map fields) in response_format. "
|
||||
"To drop them, set `drop_params=True`"
|
||||
),
|
||||
llm_provider=provider_slug,
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
def create_config_class(provider: SimpleProviderConfig):
|
||||
"""Generate config class dynamically from JSON configuration"""
|
||||
|
||||
|
|
@ -46,14 +128,37 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
|
||||
"""Transform messages based on special_handling config"""
|
||||
|
||||
# Handle content list to string conversion if configured
|
||||
if provider.special_handling.get("convert_content_list_to_string"):
|
||||
messages = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
content_flattened: Final = (
|
||||
handle_messages_with_content_list_to_str_conversion(messages)
|
||||
if provider.special_handling.get("convert_content_list_to_string")
|
||||
else messages
|
||||
)
|
||||
transformed: Final = (
|
||||
[_with_file_parts_as_image_url(message) for message in content_flattened]
|
||||
if provider.special_handling.get("convert_file_parts_to_image_url")
|
||||
else content_flattened
|
||||
)
|
||||
if is_async:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=True)
|
||||
else:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=False)
|
||||
return super()._transform_messages(messages=transformed, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed, model=model, is_async=False)
|
||||
|
||||
def _prompt_cache_ordered_messages(
|
||||
self,
|
||||
messages: list[AllMessageValues], # mutable-ok: overrides OpenAIGPTConfig's list signature
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> list[AllMessageValues]: # mutable-ok: overrides OpenAIGPTConfig's list signature
|
||||
readable: Final = (
|
||||
list(
|
||||
_messages_with_readable_file_parts(
|
||||
messages,
|
||||
provider_slug=provider.slug,
|
||||
drop_params=drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
)
|
||||
)
|
||||
if provider.special_handling.get("convert_file_parts_to_image_url")
|
||||
else messages
|
||||
)
|
||||
return super()._prompt_cache_ordered_messages(readable, litellm_params)
|
||||
|
||||
def _should_preserve_cache_control_for_endpoint(
|
||||
self,
|
||||
|
|
@ -149,12 +254,23 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
|
||||
# Apply supported params
|
||||
for param, value in non_default_params.items():
|
||||
requested_params: Final = _REQUESTED_PARAMS_ADAPTER.validate_python(non_default_params)
|
||||
for param, value in requested_params.items():
|
||||
# Check parameter mappings first
|
||||
if param in provider.param_mappings:
|
||||
optional_params[provider.param_mappings[param]] = value
|
||||
elif param in supported_params:
|
||||
optional_params[param] = value
|
||||
optional_params[param] = (
|
||||
_response_format_without_additional_properties(
|
||||
_JSON_VALUE_ADAPTER.validate_python(value),
|
||||
provider_slug=provider.slug,
|
||||
model=model,
|
||||
drop_params=drop_params or litellm.drop_params is True,
|
||||
)
|
||||
if param == "response_format"
|
||||
and provider.special_handling.get("remove_additional_properties_from_response_format")
|
||||
else value
|
||||
)
|
||||
|
||||
# Apply temperature constraints if present
|
||||
if "temperature" in optional_params:
|
||||
|
|
|
|||
|
|
@ -24,9 +24,9 @@ class SimpleProviderConfig:
|
|||
self.base_url = data["base_url"]
|
||||
self.api_key_env: str = data["api_key_env"]
|
||||
self.api_base_env: str | None = data.get("api_base_env")
|
||||
self.base_class = data.get("base_class", "openai_gpt")
|
||||
self.param_mappings = data.get("param_mappings", {})
|
||||
self.constraints = data.get("constraints", {})
|
||||
self.base_class: str = data.get("base_class", "openai_gpt")
|
||||
self.param_mappings: Mapping[str, str] = data.get("param_mappings", {})
|
||||
self.constraints: Mapping[str, float] = data.get("constraints", {})
|
||||
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
|
||||
data.get("special_handling", {})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -292,5 +292,18 @@
|
|||
"preserve_cache_control": true
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions"]
|
||||
},
|
||||
"vispark": {
|
||||
"base_url": "https://api.lab.vispark.in/v1",
|
||||
"api_key_env": "VISPARK_LAB_API_KEY",
|
||||
"api_base_env": "VISPARK_LAB_API_BASE",
|
||||
"param_mappings": {
|
||||
"max_completion_tokens": "max_tokens"
|
||||
},
|
||||
"special_handling": {
|
||||
"convert_file_parts_to_image_url": true,
|
||||
"remove_additional_properties_from_response_format": true
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/completions"]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,11 @@ Helper util for handling perplexity-specific cost calculation
|
|||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
TokenRates,
|
||||
apply_off_peak_pricing,
|
||||
unbilled_minimum_input_tokens,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -67,7 +71,10 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
output_cost_per_token: Final = rates.output_rate
|
||||
|
||||
## CALCULATE INPUT COST
|
||||
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
|
||||
prompt_tokens: Final = usage.prompt_tokens or 0
|
||||
prompt_cost: float = (
|
||||
prompt_tokens + unbilled_minimum_input_tokens(model_info, prompt_tokens)
|
||||
) * input_cost_per_token
|
||||
|
||||
## ADD CITATION TOKENS COST (if present)
|
||||
citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0
|
||||
|
|
|
|||
|
|
@ -49863,6 +49863,105 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-small": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 1.05e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 65536,
|
||||
"min_billed_input_tokens": 600,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.16e-06,
|
||||
"source": "https://api.lab.vispark.in/v1/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-medium": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 4.21e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 65536,
|
||||
"min_billed_input_tokens": 600,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.263e-05,
|
||||
"source": "https://api.lab.vispark.in/v1/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-large": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 7.37e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 65536,
|
||||
"min_billed_input_tokens": 600,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.211e-05,
|
||||
"source": "https://api.lab.vispark.in/v1/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vercel_ai_gateway/alibaba/qwen-3-14b": {
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
|
|
|
|||
|
|
@ -4228,6 +4228,32 @@
|
|||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"vispark": {
|
||||
"display_name": "Vispark (`vispark`)",
|
||||
"url": "https://lab.vispark.in/#vision",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"vercel_ai_gateway": {
|
||||
"display_name": "Vercel AI Gateway (`vercel_ai_gateway`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import ANTHROPIC_BATCHES_ROUTE
|
||||
from litellm.cost_calculator import overrides_provider_stated_cost
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
native_messages_reported_cost,
|
||||
override_stamped_response_cost,
|
||||
|
|
@ -346,7 +347,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
if isinstance(litellm_model_response, ModelResponse)
|
||||
else None
|
||||
)
|
||||
if reported_cost is not None and not custom_pricing:
|
||||
if reported_cost is not None and not overrides_provider_stated_cost(custom_pricing, logging_obj):
|
||||
return reported_cost
|
||||
return litellm.completion_cost(
|
||||
completion_response=litellm_model_response,
|
||||
|
|
|
|||
|
|
@ -4284,5 +4284,33 @@
|
|||
}
|
||||
],
|
||||
"default_model_placeholder": "bigmodel/glm-5.3"
|
||||
},
|
||||
{
|
||||
"provider": "VISPARK",
|
||||
"provider_display_name": "Vispark",
|
||||
"litellm_provider": "vispark",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://api.lab.vispark.in/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "vispark/vision-medium"
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import asyncio
|
|||
import json
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
|
|
@ -1352,7 +1352,7 @@ def _estimate_request_input_cost_for_model(
|
|||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router, group_billing_minimum=min)
|
||||
]
|
||||
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
|
||||
return max(valid_estimates) if valid_estimates else None
|
||||
|
|
@ -1521,6 +1521,7 @@ def _get_model_cost_info(
|
|||
def _get_model_cost_infos(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
group_billing_minimum: Callable[[Sequence[int]], int] = max,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""Cost-info candidates to estimate a request against for one model group.
|
||||
|
||||
|
|
@ -1528,52 +1529,64 @@ def _get_model_cost_infos(
|
|||
is unknown. Rather than guess, we estimate the cost against every eligible
|
||||
pricing shape in the group (the group's flat rates plus each deployment's
|
||||
tiered table) and let the caller reserve the maximum, so a cheaper sibling
|
||||
deployment can never leave the request under-reserved.
|
||||
deployment can never leave the request under-reserved. The deployments'
|
||||
billed input minimums fold into one through ``group_billing_minimum``, the
|
||||
largest for admission and the smallest for the input cost a disconnect is charged.
|
||||
"""
|
||||
try:
|
||||
base: Final = _get_model_cost_info(model=model, llm_router=llm_router)
|
||||
if base is None:
|
||||
group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router)
|
||||
if group_info is None:
|
||||
return []
|
||||
tiered_tables: Final = _get_deployment_tiered_pricing_tables(model=model, llm_router=llm_router)
|
||||
deployment_infos: Final = _get_deployment_model_infos(model=model, llm_router=llm_router)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"Unable to load model cost info for budget reservation",
|
||||
exc_info=True,
|
||||
)
|
||||
return []
|
||||
if not tiered_tables:
|
||||
return [base]
|
||||
billing_minimums: Final = tuple(
|
||||
_to_int(deployment_info.get("min_billed_input_tokens")) or 0 for deployment_info in deployment_infos
|
||||
)
|
||||
base: Final = (
|
||||
{**group_info, "min_billed_input_tokens": group_billing_minimum(billing_minimums)}
|
||||
if any(billing_minimums)
|
||||
else group_info
|
||||
)
|
||||
tiered_tables: Final = [
|
||||
table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None
|
||||
]
|
||||
return [base, *({**base, "tiered_pricing": table} for table in tiered_tables)]
|
||||
|
||||
|
||||
def _deployment_tiered_pricing_table(
|
||||
deployment: DeploymentTypedDict,
|
||||
llm_router: Router,
|
||||
) -> Sequence[Mapping[str, object]] | None:
|
||||
model_id: Final = _get_value(_get_value(deployment, "model_info"), "id")
|
||||
backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model")
|
||||
if not isinstance(model_id, str) or not isinstance(backend_model, str):
|
||||
return None
|
||||
deployment_model_info: Final = llm_router.registry().deployment_model_info(model_id, backend_model, cached=True)
|
||||
if deployment_model_info is None:
|
||||
return None
|
||||
tiered_pricing: Final = deployment_model_info.get("tiered_pricing")
|
||||
def _tiered_pricing_table(deployment_info: Mapping[str, object]) -> Sequence[Mapping[str, object]] | None:
|
||||
tiered_pricing: Final = deployment_info.get("tiered_pricing")
|
||||
if isinstance(tiered_pricing, list) and tiered_pricing:
|
||||
return tiered_pricing
|
||||
return None
|
||||
|
||||
|
||||
def _get_deployment_tiered_pricing_tables(
|
||||
def _deployment_model_info(
|
||||
deployment: DeploymentTypedDict,
|
||||
llm_router: Router,
|
||||
) -> Mapping[str, object] | None:
|
||||
model_id: Final = _get_value(_get_value(deployment, "model_info"), "id")
|
||||
backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model")
|
||||
if not isinstance(model_id, str) or not isinstance(backend_model, str):
|
||||
return None
|
||||
return llm_router.registry().deployment_model_info(model_id, backend_model, cached=True)
|
||||
|
||||
|
||||
def _get_deployment_model_infos(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
) -> Sequence[Sequence[Mapping[str, object]]]:
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
if llm_router is None:
|
||||
return []
|
||||
deployments: Final = llm_router.registry().deployments(model) or []
|
||||
return [
|
||||
table
|
||||
deployment_info
|
||||
for deployment in deployments
|
||||
if (table := _deployment_tiered_pricing_table(deployment, llm_router)) is not None
|
||||
if (deployment_info := _deployment_model_info(deployment, llm_router)) is not None
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -1627,14 +1640,10 @@ def _estimate_input_tokens(
|
|||
if input_tokens is not None
|
||||
else count_input_tokens_for_model(request_body=request_body, model=model)
|
||||
)
|
||||
if counted is not None:
|
||||
return counted
|
||||
|
||||
max_input_tokens: Final = _to_int(model_info.get("max_input_tokens"))
|
||||
if max_input_tokens is not None:
|
||||
return max_input_tokens
|
||||
|
||||
return None
|
||||
estimated: Final = counted if counted is not None else _to_int(model_info.get("max_input_tokens"))
|
||||
if estimated is None:
|
||||
return None
|
||||
return max(estimated, _to_int(model_info.get("min_billed_input_tokens")) or 0)
|
||||
|
||||
|
||||
DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK: Final = 16384
|
||||
|
|
|
|||
|
|
@ -10673,19 +10673,20 @@ class Router(DeprecatedRouterMembers):
|
|||
model_info[field] = copy.deepcopy(backend_value)
|
||||
|
||||
@staticmethod
|
||||
def _inherit_builtin_base_rates_for_off_peak(
|
||||
model_info: dict, # mutable-ok: cost-map entry filled in place
|
||||
def _inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info: dict[str, object], # mutable-ok: cost-map entry filled in place
|
||||
backend_model: str,
|
||||
custom_llm_provider: str | None,
|
||||
) -> None:
|
||||
"""Fill missing pricing fields on a deployment entry that only sets
|
||||
``off_peak_pricing``, from the backend model's built-in cost map entry.
|
||||
``off_peak_pricing`` or ``min_billed_input_tokens``, from the backend
|
||||
model's built-in cost map entry.
|
||||
|
||||
Cost lookup selects the deployment-scoped entry over the shared backend
|
||||
entry only when the deployment entry carries a base pricing field, and
|
||||
``off_peak_pricing`` is deliberately kept off the shared entry, so a
|
||||
deployment spelling out only its off-peak schedule would otherwise
|
||||
never receive the discount. The backend model's entire canonical cost
|
||||
both fields are deliberately kept off the shared entry, so a deployment
|
||||
spelling out only its off-peak schedule or input minimum would otherwise
|
||||
never have it applied. The backend model's entire canonical cost
|
||||
map entry is copied, field by field, so threshold, tiered,
|
||||
service-tier, cache, character, and per-second rates as well as
|
||||
companion billing fields like ``web_search_billing_unit`` and the
|
||||
|
|
@ -10699,7 +10700,7 @@ class Router(DeprecatedRouterMembers):
|
|||
no-op when any base pricing field is already set or the backend model
|
||||
has no canonical entry.
|
||||
"""
|
||||
if not model_info.get("off_peak_pricing"):
|
||||
if not model_info.get("off_peak_pricing") and model_info.get("min_billed_input_tokens") is None:
|
||||
return
|
||||
if any(
|
||||
model_info.get(field) is not None
|
||||
|
|
@ -11894,7 +11895,7 @@ class Router(DeprecatedRouterMembers):
|
|||
model_info.update(
|
||||
{key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)}
|
||||
)
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider=pricing_provider,
|
||||
|
|
@ -11915,7 +11916,17 @@ class Router(DeprecatedRouterMembers):
|
|||
backend_model=backend_model,
|
||||
custom_llm_provider=pricing_provider,
|
||||
)
|
||||
return model_info
|
||||
backend_info: Final = (
|
||||
lookup_model_info_for_registration(model=backend_model, custom_llm_provider=pricing_provider)
|
||||
if model_info.get("input_cost_per_token") is not None or model_info.get("tiered_pricing") is not None
|
||||
else None
|
||||
)
|
||||
billing_minimum: Final = (
|
||||
None
|
||||
if backend_info is None or isinstance(backend_info, ModelInfoLookupMiss)
|
||||
else backend_info.get("min_billed_input_tokens")
|
||||
)
|
||||
return model_info if billing_minimum is None else {"min_billed_input_tokens": billing_minimum, **model_info}
|
||||
|
||||
@staticmethod
|
||||
def _register_deployment_pricing(deployment: Deployment) -> None:
|
||||
|
|
|
|||
|
|
@ -367,6 +367,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
# Smallest prefix this model will actually cache, whatever caching mechanism its provider uses.
|
||||
# Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT.
|
||||
prompt_cache_min_tokens: int | None
|
||||
min_billed_input_tokens: ReadOnly[int | None]
|
||||
off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates
|
||||
input_cost_per_character: float | None # only for vertex ai models
|
||||
input_cost_per_audio_token: float | None
|
||||
|
|
@ -4145,7 +4146,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
)
|
||||
|
||||
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing", "min_billed_input_tokens"})
|
||||
|
||||
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
|
|
@ -4510,6 +4511,7 @@ class LlmProviders(str, Enum):
|
|||
MOONSHOT = "moonshot"
|
||||
PUBLICAI = "publicai"
|
||||
V0 = "v0"
|
||||
VISPARK = "vispark"
|
||||
MORPH = "morph"
|
||||
OPENCODE_GO = "opencode_go"
|
||||
LAMBDA_AI = "lambda_ai"
|
||||
|
|
|
|||
|
|
@ -7124,6 +7124,7 @@ def get_model_info_helper(
|
|||
),
|
||||
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
min_billed_input_tokens=_model_info.get("min_billed_input_tokens", None),
|
||||
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
"cache_read_input_token_cost_above_200k_tokens", None
|
||||
),
|
||||
|
|
|
|||
|
|
@ -49863,6 +49863,105 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-small": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 1.05e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 65536,
|
||||
"min_billed_input_tokens": 600,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.16e-06,
|
||||
"source": "https://api.lab.vispark.in/v1/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-medium": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 4.21e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 65536,
|
||||
"min_billed_input_tokens": 600,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.263e-05,
|
||||
"source": "https://api.lab.vispark.in/v1/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-large": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 7.37e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 65536,
|
||||
"min_billed_input_tokens": 600,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.211e-05,
|
||||
"source": "https://api.lab.vispark.in/v1/models",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/completions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image",
|
||||
"audio",
|
||||
"video"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"vercel_ai_gateway/alibaba/qwen-3-14b": {
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
|
|
|
|||
|
|
@ -4228,6 +4228,32 @@
|
|||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"vispark": {
|
||||
"display_name": "Vispark (`vispark`)",
|
||||
"url": "https://lab.vispark.in/#vision",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"vercel_ai_gateway": {
|
||||
"display_name": "Vercel AI Gateway (`vercel_ai_gateway`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway",
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ IGNORE_FUNCTIONS = [
|
|||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
"scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
|
||||
"_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did.
|
||||
"_without_additional_properties", # bounded by the nesting depth of a pydantic-validated JsonValue response_format (a finite JSON tree, no cycles possible).
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -99,6 +99,43 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re
|
|||
assert completion_cost == pytest.approx(380 * 9.7e-7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_tokens", "billed_prompt_tokens"),
|
||||
[(0, 0), (10, 600), (600, 600), (900, 900)],
|
||||
ids=[
|
||||
"no_counted_prompt_bills_no_minimum",
|
||||
"below_the_minimum_bills_the_minimum",
|
||||
"at_the_minimum",
|
||||
"above_the_minimum_bills_as_reported",
|
||||
],
|
||||
)
|
||||
def test_generic_cost_per_token_bills_at_least_the_minimum_input_tokens(
|
||||
prompt_tokens: int, billed_prompt_tokens: int
|
||||
) -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"key": "minimum-input-fixture",
|
||||
"max_tokens": None,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": None,
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 5e-6,
|
||||
"min_billed_input_tokens": 600,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"supported_openai_params": None,
|
||||
}
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model="minimum-input-fixture",
|
||||
usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7),
|
||||
custom_llm_provider="openai",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(billed_prompt_tokens * 2e-6)
|
||||
assert completion_cost == pytest.approx(7 * 5e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "expected_prompt_cost"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -994,6 +994,24 @@ class TestDashscopeCostCalculator:
|
|||
assert prompt_cost == 0.0
|
||||
assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10)
|
||||
|
||||
@pytest.mark.parametrize(("prompt_tokens", "billed_prompt_tokens"), [(10, 600), (900, 900)])
|
||||
def test_bills_at_least_the_minimum_input_tokens(self, prompt_tokens: int, billed_prompt_tokens: int):
|
||||
litellm.model_cost["dashscope/qwen-minimum-test"] = {
|
||||
"litellm_provider": "dashscope",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"min_billed_input_tokens": 600,
|
||||
}
|
||||
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-minimum-test",
|
||||
usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7),
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost, billed_prompt_tokens * 1e-06, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, 7 * 2e-06, rel_tol=1e-10)
|
||||
|
||||
OFF_PEAK_WINDOW = "14:00-00:00"
|
||||
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)
|
||||
OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc)
|
||||
|
|
|
|||
480
tests/unit/llms/openai_like/test_vispark_provider.py
Normal file
480
tests/unit/llms/openai_like/test_vispark_provider.py
Normal file
|
|
@ -0,0 +1,480 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
|
||||
_CHAT_COMPLETION: Final = {
|
||||
"id": "chatcmpl_vispark",
|
||||
"object": "chat.completion",
|
||||
"created": 1_790_000_000,
|
||||
"model": "vision-medium",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello from Vispark"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
|
||||
}
|
||||
|
||||
_TEXT_COMPLETION: Final = {
|
||||
"id": "cmpl_vispark",
|
||||
"object": "text_completion",
|
||||
"created": 1_790_000_000,
|
||||
"model": "vision-small",
|
||||
"choices": [{"index": 0, "text": "Hello from Vispark", "finish_reason": "stop", "logprobs": None}],
|
||||
"usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7},
|
||||
}
|
||||
|
||||
|
||||
def test_vispark_provider_resolution(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key")
|
||||
monkeypatch.delenv("VISPARK_LAB_API_BASE", raising=False)
|
||||
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
model="vispark/vision-medium",
|
||||
custom_llm_provider=None,
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
)
|
||||
|
||||
assert model == "vision-medium"
|
||||
assert provider == "vispark"
|
||||
assert api_key == "vispark-env-key"
|
||||
assert api_base == "https://api.lab.vispark.in/v1"
|
||||
|
||||
|
||||
def test_vispark_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key")
|
||||
monkeypatch.setenv("VISPARK_LAB_API_BASE", "https://vispark.env.example/v1")
|
||||
|
||||
_, provider, api_key, api_base = get_llm_provider(
|
||||
model="vispark/vision-medium",
|
||||
custom_llm_provider=None,
|
||||
api_base="https://vispark.internal.example/v1",
|
||||
api_key="vispark-explicit-key",
|
||||
)
|
||||
|
||||
assert provider == "vispark"
|
||||
assert api_key == "vispark-explicit-key"
|
||||
assert api_base == "https://vispark.internal.example/v1"
|
||||
|
||||
|
||||
def test_vispark_api_base_env_overrides_default(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key")
|
||||
monkeypatch.setenv("VISPARK_LAB_API_BASE", "https://vispark.env.example/v1")
|
||||
|
||||
_, provider, _, api_base = get_llm_provider(
|
||||
model="vispark/vision-medium",
|
||||
custom_llm_provider=None,
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
)
|
||||
|
||||
assert provider == "vispark"
|
||||
assert api_base == "https://vispark.env.example/v1"
|
||||
|
||||
|
||||
def test_vispark_api_base_autodetects_provider(monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key")
|
||||
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
model="vision-medium",
|
||||
custom_llm_provider=None,
|
||||
api_base="https://api.lab.vispark.in/v1",
|
||||
api_key=None,
|
||||
)
|
||||
|
||||
assert model == "vision-medium"
|
||||
assert provider == "vispark"
|
||||
assert api_key == "vispark-env-key"
|
||||
assert api_base == "https://api.lab.vispark.in/v1"
|
||||
|
||||
|
||||
def test_vispark_chat_completion_request_maps_max_completion_tokens():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
response: Final = litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
max_completion_tokens=64,
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert request.headers["authorization"] == "Bearer vispark-test-key"
|
||||
assert body["model"] == "vision-medium"
|
||||
assert body["max_tokens"] == 64
|
||||
assert "max_completion_tokens" not in body
|
||||
assert response.choices[0].message.content == "Hello from Vispark"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("additional_properties", [False, True])
|
||||
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(additional_properties: bool):
|
||||
city_schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}},
|
||||
"additionalProperties": additional_properties,
|
||||
}
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"city": city_schema},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
response_format: Final = {"type": "json_schema", "json_schema": {"name": "City", "schema": schema, "strict": True}}
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Capital of France as JSON"}],
|
||||
response_format=response_format,
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
sent_format: Final = json.loads(route.calls.last.request.content)["response_format"]
|
||||
assert "additionalProperties" not in json.dumps(sent_format)
|
||||
assert sent_format["json_schema"]["strict"] is True
|
||||
assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}}
|
||||
assert city_schema["additionalProperties"] == additional_properties
|
||||
|
||||
|
||||
def _map_field_response_format() -> dict[str, object]:
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"facts": {"type": "object", "additionalProperties": {"type": "string"}}},
|
||||
"required": ["facts"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
return {"type": "json_schema", "json_schema": {"name": "Facts", "schema": schema}}
|
||||
|
||||
|
||||
def test_vispark_rejects_a_map_field_response_format_without_drop_params():
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as error:
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Facts about Paris as JSON"}],
|
||||
response_format=_map_field_response_format(),
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
def test_vispark_drops_a_map_field_schema_with_drop_params():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Facts about Paris as JSON"}],
|
||||
response_format=_map_field_response_format(),
|
||||
drop_params=True,
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"]
|
||||
assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]}
|
||||
|
||||
|
||||
def test_vispark_keeps_a_schema_property_named_additional_properties():
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"additionalProperties": {"type": "string", "additionalProperties": False}},
|
||||
"required": ["additionalProperties"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Describe Paris as JSON"}],
|
||||
response_format={"type": "json_schema", "json_schema": {"name": "Notes", "schema": schema}},
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"]
|
||||
assert sent_schema == {
|
||||
"type": "object",
|
||||
"properties": {"additionalProperties": {"type": "string"}},
|
||||
"required": ["additionalProperties"],
|
||||
}
|
||||
|
||||
|
||||
def _file_id_messages() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this file"},
|
||||
{"type": "file", "file": {"file_id": "file-abc123"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_vispark_rejects_a_file_part_sent_by_file_id_without_drop_params():
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
with pytest.raises(litellm.BadRequestError, match="file_id") as error:
|
||||
litellm.completion(model="vispark/vision-medium", messages=_file_id_messages(), api_key="vispark-test-key")
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vispark_drops_a_file_part_sent_by_file_id_with_drop_params(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
await litellm.acompletion(
|
||||
model="vispark/vision-medium",
|
||||
messages=_file_id_messages(),
|
||||
drop_params=True,
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"]
|
||||
assert sent_content == [{"type": "text", "text": "Summarize this file"}]
|
||||
|
||||
|
||||
def test_other_json_providers_keep_additional_properties_and_file_parts():
|
||||
file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}
|
||||
response_format: Final = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "City",
|
||||
"schema": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False},
|
||||
},
|
||||
}
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.scaleway.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION)
|
||||
litellm.completion(
|
||||
model="scaleway/llama-3.3-70b-instruct",
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "Read this"}, file_part]}],
|
||||
response_format=response_format,
|
||||
api_key="scaleway-test-key",
|
||||
)
|
||||
|
||||
body: Final = json.loads(route.calls.last.request.content)
|
||||
assert body["response_format"] == response_format
|
||||
assert body["messages"][0]["content"][1] == file_part
|
||||
|
||||
|
||||
_PDF_DATA_URL: Final = "data:application/pdf;base64,JVBERi0xLjQK"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file",
|
||||
[{"file_data": _PDF_DATA_URL}, {"file_id": "file-abc123", "file_data": _PDF_DATA_URL}],
|
||||
ids=["file_data", "file_id_and_file_data"],
|
||||
)
|
||||
def test_vispark_sends_a_pdf_file_part_as_an_image_url(file: dict[str, str]):
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What code is in this PDF?"},
|
||||
{"type": "file", "file": file},
|
||||
],
|
||||
}
|
||||
],
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"]
|
||||
assert sent_content[0] == {"type": "text", "text": "What code is in this PDF?"}
|
||||
assert sent_content[1]["type"] == "image_url"
|
||||
assert sent_content[1]["image_url"]["url"] == _PDF_DATA_URL
|
||||
assert all(part["type"] != "file" for part in sent_content)
|
||||
|
||||
|
||||
def test_vispark_text_completion_request():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/completions").respond(200, json=_TEXT_COMPLETION)
|
||||
response: Final = litellm.text_completion(
|
||||
model="vispark/vision-small",
|
||||
prompt="Say hello",
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
body: Final = json.loads(route.calls.last.request.content)
|
||||
assert route.call_count == 1
|
||||
assert body["model"] == "vision-small"
|
||||
assert body["prompt"] == "Say hello"
|
||||
assert response.choices[0].text == "Hello from Vispark"
|
||||
|
||||
|
||||
def test_vispark_cost_follows_the_cost_map():
|
||||
cost_entry: Final = litellm.model_cost["vispark/vision-large"]
|
||||
prompt_tokens: Final = cost_entry["min_billed_input_tokens"] + 1_000
|
||||
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="vision-large",
|
||||
custom_llm_provider="vispark",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=500,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(prompt_tokens * cost_entry["input_cost_per_token"])
|
||||
assert completion_cost == pytest.approx(500 * cost_entry["output_cost_per_token"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["vision-small", "vision-medium", "vision-large"])
|
||||
def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str):
|
||||
cost_entry: Final = litellm.model_cost[f"vispark/{model}"]
|
||||
minimum: Final = cost_entry["min_billed_input_tokens"]
|
||||
|
||||
prompt_cost, _ = litellm.cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider="vispark",
|
||||
prompt_tokens=minimum // 2,
|
||||
completion_tokens=1,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"custom_pricing",
|
||||
[
|
||||
{"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6},
|
||||
{"tiered_pricing": [{"range": [0, 1_000_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}]},
|
||||
],
|
||||
ids=["flat", "tiered"],
|
||||
)
|
||||
def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens(
|
||||
custom_pricing: dict[str, object],
|
||||
):
|
||||
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vispark-custom-priced",
|
||||
"litellm_params": {
|
||||
"model": "vispark/vision-small",
|
||||
"api_key": "vispark-test-key",
|
||||
**custom_pricing,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(200, json=_CHAT_COMPLETION)
|
||||
response: Final = router.completion(
|
||||
model="vispark-custom-priced", messages=[{"role": "user", "content": "Say hello"}]
|
||||
)
|
||||
|
||||
completion_tokens: Final = _CHAT_COMPLETION["usage"]["completion_tokens"]
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6)
|
||||
|
||||
|
||||
def test_vispark_deployment_minimum_override_stays_on_its_own_deployment():
|
||||
builtin: Final = litellm.model_cost["vispark/vision-small"]
|
||||
minimum: Final = builtin["min_billed_input_tokens"]
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vispark-no-minimum",
|
||||
"litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"},
|
||||
"model_info": {"min_billed_input_tokens": 0},
|
||||
},
|
||||
{
|
||||
"model_name": "vispark-default",
|
||||
"litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"},
|
||||
},
|
||||
]
|
||||
)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json={**_CHAT_COMPLETION, "model": "vision-small"}
|
||||
)
|
||||
no_minimum: Final = router.completion(
|
||||
model="vispark-no-minimum", messages=[{"role": "user", "content": "Say hello"}]
|
||||
)
|
||||
default: Final = router.completion(model="vispark-default", messages=[{"role": "user", "content": "Say hello"}])
|
||||
|
||||
usage: Final = _CHAT_COMPLETION["usage"]
|
||||
completion_cost: Final = usage["completion_tokens"] * builtin["output_cost_per_token"]
|
||||
assert no_minimum._hidden_params["response_cost"] == pytest.approx(
|
||||
usage["prompt_tokens"] * builtin["input_cost_per_token"] + completion_cost
|
||||
)
|
||||
assert default._hidden_params["response_cost"] == pytest.approx(
|
||||
minimum * builtin["input_cost_per_token"] + completion_cost
|
||||
)
|
||||
assert litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] == minimum
|
||||
|
||||
|
||||
def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens():
|
||||
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
|
||||
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="vision-small",
|
||||
custom_llm_provider="vispark",
|
||||
prompt_tokens=minimum // 2,
|
||||
completion_tokens=3,
|
||||
custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6},
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(minimum * 2e-6)
|
||||
assert completion_cost == pytest.approx(3 * 5e-6)
|
||||
|
||||
|
||||
def test_vispark_explicit_custom_cost_per_token_bills_no_minimum_without_a_counted_prompt():
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="vision-small",
|
||||
custom_llm_provider="vispark",
|
||||
prompt_tokens=0,
|
||||
completion_tokens=3,
|
||||
custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6},
|
||||
)
|
||||
|
||||
assert prompt_cost == 0.0
|
||||
assert completion_cost == pytest.approx(3 * 5e-6)
|
||||
|
||||
|
||||
def test_vispark_models_are_listed_for_the_provider():
|
||||
from litellm.utils import get_valid_models
|
||||
|
||||
expected: Final = {name for name, entry in litellm.model_cost.items() if entry.get("litellm_provider") == "vispark"}
|
||||
|
||||
assert expected
|
||||
assert set(litellm.models_by_provider["vispark"]) == expected
|
||||
assert expected <= set(get_valid_models(custom_llm_provider="vispark"))
|
||||
|
|
@ -140,6 +140,26 @@ class TestPerplexityCostCalculator:
|
|||
assert prompt_cost == 0.0
|
||||
assert completion_cost == 0.008
|
||||
|
||||
@pytest.mark.parametrize(("prompt_tokens", "billed_prompt_tokens"), [(10, 600), (900, 900)])
|
||||
def test_manual_pricing_bills_at_least_the_minimum_input_tokens(
|
||||
self, prompt_tokens: int, billed_prompt_tokens: int
|
||||
):
|
||||
litellm.model_cost["perplexity/sonar-minimum-test"] = {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"min_billed_input_tokens": 600,
|
||||
}
|
||||
|
||||
prompt_cost, completion_cost = perplexity_cost_per_token(
|
||||
model="sonar-minimum-test",
|
||||
usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7),
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost, billed_prompt_tokens * 1e-06, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, 7 * 2e-06, rel_tol=1e-10)
|
||||
|
||||
OFF_PEAK_MODEL = "sonar-off-peak-test"
|
||||
OFF_PEAK_WINDOW = "14:00-00:00"
|
||||
INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)
|
||||
|
|
|
|||
|
|
@ -2951,6 +2951,24 @@ def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delt
|
|||
assert result["result"]._hidden_params["response_cost"] == 0.00412
|
||||
|
||||
|
||||
def test_streamed_messages_from_a_deployment_that_only_sets_a_billing_minimum_bill_the_reported_cost():
|
||||
logging_obj = _go_logging_obj()
|
||||
logging_obj.litellm_params = {"metadata": {"model_info": {"min_billed_input_tokens": 100000}}}
|
||||
|
||||
result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
|
||||
litellm_logging_obj=logging_obj,
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True},
|
||||
endpoint_type="messages",
|
||||
start_time=datetime.now(),
|
||||
all_chunks=_go_stream_chunks(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
|
||||
assert result["kwargs"]["response_cost"] == 0.00412
|
||||
|
||||
|
||||
def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field():
|
||||
logging_obj = _go_logging_obj()
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
_get_team_member_budget_counter,
|
||||
estimate_request_input_cost,
|
||||
estimate_request_max_cost,
|
||||
get_budget_window_start,
|
||||
release_unbound_budget_reservation,
|
||||
|
|
@ -221,6 +222,73 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None:
|
|||
assert math.isclose(after, before * 1000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_minimum", [None, 0])
|
||||
def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_minimum: int | None) -> None:
|
||||
backend_model: Final = "vispark/vision-small"
|
||||
backend_info: Final = litellm.get_model_info(model=backend_model)
|
||||
built_in_minimum: Final = backend_info.get("min_billed_input_tokens")
|
||||
input_cost_per_token: Final = backend_info.get("input_cost_per_token")
|
||||
assert built_in_minimum is not None and built_in_minimum > 0
|
||||
assert input_cost_per_token is not None
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
Deployment(
|
||||
model_name="minimum-group",
|
||||
litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"),
|
||||
model_info=ModelInfo(id="minimum-deployment", min_billed_input_tokens=deployment_minimum),
|
||||
).model_dump()
|
||||
]
|
||||
)
|
||||
prompt_tokens: Final = 12
|
||||
|
||||
reserved: Final = estimate_request_input_cost(
|
||||
request_body={"model": "minimum-group", "messages": [{"role": "user", "content": "hi"}]},
|
||||
route="/chat/completions",
|
||||
llm_router=router,
|
||||
input_token_counts={"minimum-group": prompt_tokens},
|
||||
)
|
||||
|
||||
billed_tokens: Final = built_in_minimum if deployment_minimum is None else max(prompt_tokens, deployment_minimum)
|
||||
assert reserved is not None
|
||||
assert math.isclose(reserved, billed_tokens * input_cost_per_token)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sibling_minimum", [0, None])
|
||||
def test_disconnect_charge_is_only_the_minimum_every_deployment_in_the_group_bills(sibling_minimum: int | None) -> None:
|
||||
backend_model: Final = "vispark/vision-small"
|
||||
backend_info: Final = litellm.get_model_info(model=backend_model)
|
||||
built_in_minimum: Final = backend_info.get("min_billed_input_tokens")
|
||||
input_cost_per_token: Final = backend_info.get("input_cost_per_token")
|
||||
assert built_in_minimum is not None and built_in_minimum > 0
|
||||
assert input_cost_per_token is not None
|
||||
largest_minimum: Final = built_in_minimum * 100
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
Deployment(
|
||||
model_name="mixed-minimum-group",
|
||||
litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"),
|
||||
model_info=ModelInfo(id=deployment_id, min_billed_input_tokens=minimum),
|
||||
).model_dump()
|
||||
for deployment_id, minimum in (("largest-minimum", largest_minimum), ("sibling", sibling_minimum))
|
||||
]
|
||||
)
|
||||
prompt_tokens: Final = 12
|
||||
request_body: Final = {"model": "mixed-minimum-group", "messages": [{"role": "user", "content": "hi"}]}
|
||||
token_counts: Final = {"mixed-minimum-group": prompt_tokens}
|
||||
|
||||
disconnect_charge: Final = estimate_request_input_cost(
|
||||
request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts
|
||||
)
|
||||
admission_estimate: Final = estimate_request_max_cost(
|
||||
request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts
|
||||
)
|
||||
|
||||
shared_minimum: Final = built_in_minimum if sibling_minimum is None else sibling_minimum
|
||||
assert disconnect_charge is not None and admission_estimate is not None
|
||||
assert math.isclose(disconnect_charge, max(prompt_tokens, shared_minimum) * input_cost_per_token)
|
||||
assert admission_estimate >= largest_minimum * input_cost_per_token
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"expiry_offset, expected_max_budget",
|
||||
|
|
|
|||
|
|
@ -1343,8 +1343,8 @@ def test_reservation_uses_most_expensive_deployment_in_group():
|
|||
return_value={"max_output_tokens": 200000},
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.spend_tracking.budget_reservation._get_deployment_tiered_pricing_tables",
|
||||
return_value=[cheap, expensive],
|
||||
"litellm.proxy.spend_tracking.budget_reservation._get_deployment_model_infos",
|
||||
return_value=[{"tiered_pricing": cheap}, {"tiered_pricing": expensive}],
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.spend_tracking.budget_reservation._estimate_input_tokens",
|
||||
|
|
|
|||
|
|
@ -10589,3 +10589,50 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
|
|||
assert entry is not None
|
||||
assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03
|
||||
assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "chat_url", "usage_cost", "stated_cost"),
|
||||
[
|
||||
(
|
||||
"perplexity/sonar",
|
||||
"https://api.perplexity.ai/chat/completions",
|
||||
{"request_cost": 0.0025, "total_cost": 0.00266},
|
||||
0.00266,
|
||||
),
|
||||
("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721),
|
||||
("openrouter/openai/gpt-5.4-nano", "https://openrouter.ai/api/v1/chat/completions", 0.00721, 0.00721),
|
||||
],
|
||||
ids=["perplexity", "kyma", "openrouter"],
|
||||
)
|
||||
def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost(
|
||||
_local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float
|
||||
):
|
||||
from litellm import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "deployment-with-minimum",
|
||||
"litellm_params": {"model": model, "api_key": "provider-test-key"},
|
||||
"model_info": {"min_billed_input_tokens": 1000},
|
||||
}
|
||||
]
|
||||
)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post(chat_url).respond(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl_minimum",
|
||||
"object": "chat.completion",
|
||||
"created": 1_790_000_000,
|
||||
"model": "sonar",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Paris"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8, "cost": usage_cost},
|
||||
},
|
||||
)
|
||||
response: Final = router.completion(
|
||||
model="deployment-with-minimum", messages=[{"role": "user", "content": "Capital of France?"}]
|
||||
)
|
||||
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(stated_cost)
|
||||
|
|
|
|||
|
|
@ -1694,7 +1694,7 @@ def test_cost_map_backend_model_uses_canonical_model_precedence(
|
|||
assert Router._cost_map_backend_model(deployment) == expected
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_fills_missing_rates():
|
||||
"""Direct unit test of the helper: an entry carrying only an
|
||||
off_peak_pricing block inherits the backend model's built-in base token
|
||||
rates, so cost lookup via the deployment id can bill standard rates
|
||||
|
|
@ -1709,7 +1709,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
|||
}
|
||||
model_info = {"off_peak_pricing": off_peak_block}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -1720,7 +1720,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
|||
assert model_info["off_peak_pricing"] == off_peak_block
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_carries_threshold_rates():
|
||||
"""A backend with above-threshold pricing hands the whole rate structure to
|
||||
the deployment entry, so peak-hour billing of large prompts through that
|
||||
entry matches the shared backend entry instead of flattening to the base
|
||||
|
|
@ -1734,7 +1734,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
|||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
@ -1749,7 +1749,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
|||
)
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_carries_companion_billing_fields():
|
||||
"""Billing rules that are not literal cost rates, like the web search
|
||||
billing unit, must ride along, or grounding and regional uplifts would
|
||||
bill differently through the deployment entry than through the shared
|
||||
|
|
@ -1763,7 +1763,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field
|
|||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider=None,
|
||||
|
|
@ -1773,7 +1773,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field
|
|||
assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"]
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_tiered_only_backend_stores_no_zero():
|
||||
"""A tiered-only backend has no flat token rates; get_model_info synthesizes
|
||||
zeros for them, and storing those would mark the deployment explicitly
|
||||
priced free. The tier table itself must carry over as an isolated copy so
|
||||
|
|
@ -1786,7 +1786,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z
|
|||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="dashscope",
|
||||
|
|
@ -1803,7 +1803,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z
|
|||
assert raw_tiers[0] == original_first_tier
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_leaves_explicit_rates_alone():
|
||||
"""An entry that sets its own base rate beside the block already counts as
|
||||
a full custom pricing entry; the helper must not mix builtin rates into it.
|
||||
"""
|
||||
|
|
@ -1812,7 +1812,7 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
|||
"input_cost_per_token": 3e-06,
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model="gpt-4o-mini",
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -1822,12 +1822,12 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
|||
assert "output_cost_per_token" not in model_info
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_noop_without_block_or_backend():
|
||||
"""Nothing happens without an off_peak_pricing block, and an unmapped
|
||||
backend model leaves the entry unchanged rather than raising.
|
||||
"""
|
||||
plain_info = {"id": "dep-1"}
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=plain_info,
|
||||
backend_model="gpt-4o-mini",
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -1835,7 +1835,7 @@ def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend()
|
|||
assert plain_info == {"id": "dep-1"}
|
||||
|
||||
off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}}
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=off_peak_info,
|
||||
backend_model="this-backend-model-does-not-exist-x9y8z7",
|
||||
custom_llm_provider=None,
|
||||
|
|
|
|||
|
|
@ -1166,6 +1166,7 @@ _COST_MAP_ROW_SCHEMA: Final = {
|
|||
"supports_parallel_tool_use_config": {"type": "boolean"},
|
||||
"supports_pdf_input": {"type": "boolean"},
|
||||
"prompt_cache_min_tokens": {"type": "number"},
|
||||
"min_billed_input_tokens": {"type": "number"},
|
||||
"supports_prompt_cache_breakpoint": {"type": "boolean"},
|
||||
"supports_thinking_cache_preservation": {"type": "boolean"},
|
||||
"supports_prompt_caching": {"type": "boolean"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue