diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 86b22069a39..055af080a9f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -522,8 +522,13 @@ def cost_per_token( if _is_anthropic_style: _normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens + billed_prompt_tokens: Final = ( + max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider)) + if custom_cost_per_token is not None + else _normalized_prompt_tokens + ) response_cost: Final = _cost_per_token_custom_pricing_helper( - prompt_tokens=_normalized_prompt_tokens, + prompt_tokens=billed_prompt_tokens, completion_tokens=completion_tokens, response_time_ms=response_time_ms, cached_tokens=_cache_read_tokens, @@ -2436,6 +2441,12 @@ def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: return raw_entry if isinstance(raw_entry, Mapping) else None +def _min_billed_input_tokens(model: str, custom_llm_provider: str | None) -> int: + cost_entry: Final = _raw_cost_map_entry(model) or _raw_cost_map_entry(f"{custom_llm_provider}/{model}") or {} + minimum: Final = cost_entry.get("min_billed_input_tokens") + return minimum if isinstance(minimum, int) else 0 + + def pricing_entry_for_cost_calc( model: str | None, completion_response: object | None, diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 1b2d54dde35..7c283a9d7d4 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -17,7 +17,6 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage -from litellm.utils import remove_additional_properties from .json_loader import SimpleProviderConfig @@ -38,6 +37,16 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues return updated +def _without_additional_properties(value: JsonValue) -> JsonValue: + if isinstance(value, dict): + return { + key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties" + } + if isinstance(value, list): + return [_without_additional_properties(item) for item in value] + return value + + def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" @@ -169,7 +178,7 @@ def create_config_class(provider: SimpleProviderConfig): optional_params[provider.param_mappings[param]] = value elif param in supported_params: optional_params[param] = ( - remove_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) + _without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) if param == "response_format" and provider.special_handling.get("remove_additional_properties_from_response_format") else value diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 11eca48f3be..509ba98e5c6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48595,6 +48595,7 @@ "supports_vision": true }, "vispark/vision-small": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 1.05e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48627,6 +48628,7 @@ "supports_vision": true }, "vispark/vision-medium": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 4.21e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48659,6 +48661,7 @@ "supports_vision": true }, "vispark/vision-large": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 7.37e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..1a573816579 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11569,7 +11569,22 @@ class Router: backend_model=backend_model, custom_llm_provider=pricing_provider, ) - return model_info + billing_minimum: Final[Mapping[str, int]] = ( + Router._builtin_billing_minimum(backend_model=backend_model, custom_llm_provider=pricing_provider) + if model_info.get("input_cost_per_token") is not None + else MappingProxyType({}) + ) + return {**billing_minimum, **model_info} + + @staticmethod + def _builtin_billing_minimum(backend_model: str, custom_llm_provider: str | None) -> Mapping[str, int]: + backend_info: Final = lookup_model_info_for_registration( + model=backend_model, custom_llm_provider=custom_llm_provider + ) + minimum: Final = ( + None if isinstance(backend_info, ModelInfoLookupMiss) else backend_info.get("min_billed_input_tokens") + ) + return MappingProxyType({} if minimum is None else {"min_billed_input_tokens": minimum}) @staticmethod def _register_deployment_pricing(deployment: Deployment) -> None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 11eca48f3be..509ba98e5c6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48595,6 +48595,7 @@ "supports_vision": true }, "vispark/vision-small": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 1.05e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48627,6 +48628,7 @@ "supports_vision": true }, "vispark/vision-medium": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 4.21e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48659,6 +48661,7 @@ "supports_vision": true }, "vispark/vision-large": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 7.37e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 3b574e46950..5d2a4764f25 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -126,10 +126,18 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens(): assert response.choices[0].message.content == "Hello from Vispark" -def test_vispark_strips_additional_properties_from_a_json_schema_response_format(): +@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}]) +def test_vispark_strips_additional_properties_from_a_json_schema_response_format( + additional_properties: bool | dict[str, str], +): + city_schema: Final = { + "type": "object", + "properties": {"name": {"type": "string"}}, + "additionalProperties": additional_properties, + } schema: Final = { "type": "object", - "properties": {"city": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}}, + "properties": {"city": city_schema}, "required": ["city"], "additionalProperties": False, } @@ -149,7 +157,30 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format assert "additionalProperties" not in json.dumps(sent_format) assert sent_format["json_schema"]["strict"] is True assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}} - assert schema["additionalProperties"] is False + assert city_schema["additionalProperties"] == additional_properties + + +def test_other_json_providers_keep_additional_properties_and_file_parts(): + file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}} + response_format: Final = { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}, + }, + } + with respx.mock() as upstream: + route: Final = upstream.post("https://api.scaleway.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) + litellm.completion( + model="scaleway/llama-3.3-70b-instruct", + messages=[{"role": "user", "content": [{"type": "text", "text": "Read this"}, file_part]}], + response_format=response_format, + api_key="scaleway-test-key", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == response_format + assert body["messages"][0]["content"][1] == file_part def test_vispark_sends_a_pdf_file_part_as_an_image_url(): @@ -225,6 +256,46 @@ def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str): assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"]) +def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens(): + minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vispark-custom-priced", + "litellm_params": { + "model": "vispark/vision-small", + "api_key": "vispark-test-key", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 5e-6, + }, + } + ] + ) + with respx.mock() as upstream: + upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) + response: Final = router.completion( + model="vispark-custom-priced", messages=[{"role": "user", "content": "Say hello"}] + ) + + completion_tokens: Final = _CHAT_COMPLETION["usage"]["completion_tokens"] + assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6) + + +def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens(): + minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] + + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-small", + custom_llm_provider="vispark", + prompt_tokens=minimum // 2, + completion_tokens=3, + custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + ) + + assert prompt_cost == pytest.approx(minimum * 2e-6) + assert completion_cost == pytest.approx(3 * 5e-6) + + def test_vispark_models_are_listed_for_the_provider(): from litellm.utils import get_valid_models