diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index ebee2c64140..3597795ba35 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -294,6 +294,9 @@ pub struct ModelInfo { /// Free-form notes about the entry (e.g. pricing derivation). #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option>, + /// Input tokens billed per request even when the prompt is shorter. + #[serde(skip_serializing_if = "Option::is_none")] + pub min_billed_input_tokens: Option, /// Primary API surface / task type of the model. #[serde(skip_serializing_if = "Option::is_none")] pub mode: Option, diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 7c283a9d7d4..04b93a94df6 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -2,12 +2,15 @@ Dynamic configuration class generator for JSON-based providers. """ -from collections.abc import Coroutine, Mapping +import json +from collections.abc import Callable, Coroutine, Mapping from typing import TYPE_CHECKING, Final, Literal, overload from pydantic import JsonValue, TypeAdapter +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, migrate_file_to_image_url, @@ -37,14 +40,35 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues return updated -def _without_additional_properties(value: JsonValue) -> JsonValue: - if isinstance(value, dict): - return { - key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties" - } - if isinstance(value, list): - return [_without_additional_properties(item) for item in value] - return value +def _without_additional_properties_flags(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + return {key: item for key, item in node.items() if key != "additionalProperties" or not isinstance(item, bool)} + + +def _without_additional_properties_key(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + return {key: item for key, item in node.items() if key != "additionalProperties"} + + +def _rebuilt_json( + value: JsonValue, object_hook: Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]] +) -> JsonValue: + return _JSON_VALUE_ADAPTER.validate_python(json.loads(json.dumps(value), object_hook=object_hook)) + + +def _response_format_without_additional_properties( + response_format: JsonValue, provider_slug: str, model: str, drop_params: bool +) -> JsonValue: + without_flags: Final = _rebuilt_json(response_format, _without_additional_properties_flags) + without_any: Final = _rebuilt_json(response_format, _without_additional_properties_key) + if without_flags == without_any or drop_params: + return without_any + raise UnsupportedParamsError( + message=( + f"{provider_slug} does not support schema-valued additionalProperties (map fields) in response_format. " + "To drop them, set `drop_params=True`" + ), + llm_provider=provider_slug, + model=model, + ) def create_config_class(provider: SimpleProviderConfig): @@ -178,7 +202,12 @@ def create_config_class(provider: SimpleProviderConfig): optional_params[provider.param_mappings[param]] = value elif param in supported_params: optional_params[param] = ( - _without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) + _response_format_without_additional_properties( + _JSON_VALUE_ADAPTER.validate_python(value), + provider_slug=provider.slug, + model=model, + drop_params=drop_params or litellm.drop_params is True, + ) if param == "response_format" and provider.special_handling.get("remove_additional_properties_from_response_format") else value diff --git a/litellm/router.py b/litellm/router.py index c637a739d4b..6f2b572c77c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10331,19 +10331,20 @@ class Router: model_info[field] = copy.deepcopy(backend_value) @staticmethod - def _inherit_builtin_base_rates_for_off_peak( + def _inherit_builtin_base_rates_for_scoped_pricing( model_info: dict, # mutable-ok: cost-map entry filled in place backend_model: str, custom_llm_provider: str | None, ) -> None: """Fill missing pricing fields on a deployment entry that only sets - ``off_peak_pricing``, from the backend model's built-in cost map entry. + ``off_peak_pricing`` or ``min_billed_input_tokens``, from the backend + model's built-in cost map entry. Cost lookup selects the deployment-scoped entry over the shared backend entry only when the deployment entry carries a base pricing field, and - ``off_peak_pricing`` is deliberately kept off the shared entry, so a - deployment spelling out only its off-peak schedule would otherwise - never receive the discount. The backend model's entire canonical cost + both fields are deliberately kept off the shared entry, so a deployment + spelling out only its off-peak schedule or input minimum would otherwise + never have it applied. The backend model's entire canonical cost map entry is copied, field by field, so threshold, tiered, service-tier, cache, character, and per-second rates as well as companion billing fields like ``web_search_billing_unit`` and the @@ -10357,7 +10358,7 @@ class Router: no-op when any base pricing field is already set or the backend model has no canonical entry. """ - if not model_info.get("off_peak_pricing"): + if not model_info.get("off_peak_pricing") and model_info.get("min_billed_input_tokens") is None: return if any( model_info.get(field) is not None @@ -11548,7 +11549,7 @@ class Router: model_info.update( {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} ) - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cac11598628..025424d8810 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4130,7 +4130,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): return {k: v for k, v in model_info.items() if k not in cls.model_fields} -DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) +DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing", "min_billed_input_tokens"}) DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( {"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"} diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 5d2a4764f25..45636725b83 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -126,10 +126,8 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens(): assert response.choices[0].message.content == "Hello from Vispark" -@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}]) -def test_vispark_strips_additional_properties_from_a_json_schema_response_format( - additional_properties: bool | dict[str, str], -): +@pytest.mark.parametrize("additional_properties", [False, True]) +def test_vispark_strips_additional_properties_from_a_json_schema_response_format(additional_properties: bool): city_schema: Final = { "type": "object", "properties": {"name": {"type": "string"}}, @@ -160,6 +158,50 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format assert city_schema["additionalProperties"] == additional_properties +def _map_field_response_format() -> dict[str, object]: + schema: Final = { + "type": "object", + "properties": {"facts": {"type": "object", "additionalProperties": {"type": "string"}}}, + "required": ["facts"], + "additionalProperties": False, + } + return {"type": "json_schema", "json_schema": {"name": "Facts", "schema": schema}} + + +def test_vispark_rejects_a_map_field_response_format_without_drop_params(): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + with pytest.raises(litellm.UnsupportedParamsError) as error: + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Facts about Paris as JSON"}], + response_format=_map_field_response_format(), + api_key="vispark-test-key", + ) + + assert error.value.status_code == 400 + assert route.call_count == 0 + + +def test_vispark_drops_a_map_field_schema_with_drop_params(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Facts about Paris as JSON"}], + response_format=_map_field_response_format(), + drop_params=True, + api_key="vispark-test-key", + ) + + sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"] + assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]} + + def test_other_json_providers_keep_additional_properties_and_file_parts(): file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}} response_format: Final = { @@ -281,6 +323,42 @@ def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_i assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6) +def test_vispark_deployment_minimum_override_stays_on_its_own_deployment(): + builtin: Final = litellm.model_cost["vispark/vision-small"] + minimum: Final = builtin["min_billed_input_tokens"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vispark-no-minimum", + "litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"}, + "model_info": {"min_billed_input_tokens": 0}, + }, + { + "model_name": "vispark-default", + "litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"}, + }, + ] + ) + with respx.mock() as upstream: + upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json={**_CHAT_COMPLETION, "model": "vision-small"} + ) + no_minimum: Final = router.completion( + model="vispark-no-minimum", messages=[{"role": "user", "content": "Say hello"}] + ) + default: Final = router.completion(model="vispark-default", messages=[{"role": "user", "content": "Say hello"}]) + + usage: Final = _CHAT_COMPLETION["usage"] + completion_cost: Final = usage["completion_tokens"] * builtin["output_cost_per_token"] + assert no_minimum._hidden_params["response_cost"] == pytest.approx( + usage["prompt_tokens"] * builtin["input_cost_per_token"] + completion_cost + ) + assert default._hidden_params["response_cost"] == pytest.approx( + minimum * builtin["input_cost_per_token"] + completion_cost + ) + assert litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] == minimum + + def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens(): minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..16ed772d3df 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -1637,7 +1637,7 @@ def test_cost_map_backend_model_uses_canonical_model_precedence( assert Router._cost_map_backend_model(deployment) == expected -def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): +def test_inherit_builtin_base_rates_for_scoped_pricing_fills_missing_rates(): """Direct unit test of the helper: an entry carrying only an off_peak_pricing block inherits the backend model's built-in base token rates, so cost lookup via the deployment id can bill standard rates @@ -1652,7 +1652,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): } model_info = {"off_peak_pricing": off_peak_block} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="openai", @@ -1663,7 +1663,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): assert model_info["off_peak_pricing"] == off_peak_block -def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): +def test_inherit_builtin_base_rates_for_scoped_pricing_carries_threshold_rates(): """A backend with above-threshold pricing hands the whole rate structure to the deployment entry, so peak-hour billing of large prompts through that entry matches the shared backend entry instead of flattening to the base @@ -1677,7 +1677,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="gemini", @@ -1692,7 +1692,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): ) -def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields(): +def test_inherit_builtin_base_rates_for_scoped_pricing_carries_companion_billing_fields(): """Billing rules that are not literal cost rates, like the web search billing unit, must ride along, or grounding and regional uplifts would bill differently through the deployment entry than through the shared @@ -1706,7 +1706,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=None, @@ -1716,7 +1716,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"] -def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero(): +def test_inherit_builtin_base_rates_for_scoped_pricing_tiered_only_backend_stores_no_zero(): """A tiered-only backend has no flat token rates; get_model_info synthesizes zeros for them, and storing those would mark the deployment explicitly priced free. The tier table itself must carry over as an isolated copy so @@ -1729,7 +1729,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="dashscope", @@ -1746,7 +1746,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z assert raw_tiers[0] == original_first_tier -def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): +def test_inherit_builtin_base_rates_for_scoped_pricing_leaves_explicit_rates_alone(): """An entry that sets its own base rate beside the block already counts as a full custom pricing entry; the helper must not mix builtin rates into it. """ @@ -1755,7 +1755,7 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): "input_cost_per_token": 3e-06, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model="gpt-4o-mini", custom_llm_provider="openai", @@ -1765,12 +1765,12 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): assert "output_cost_per_token" not in model_info -def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend(): +def test_inherit_builtin_base_rates_for_scoped_pricing_noop_without_block_or_backend(): """Nothing happens without an off_peak_pricing block, and an unmapped backend model leaves the entry unchanged rather than raising. """ plain_info = {"id": "dep-1"} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=plain_info, backend_model="gpt-4o-mini", custom_llm_provider="openai", @@ -1778,7 +1778,7 @@ def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend() assert plain_info == {"id": "dep-1"} off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=off_peak_info, backend_model="this-backend-model-does-not-exist-x9y8z7", custom_llm_provider=None,