fix(vispark): bill the input token minimum under custom pricing and strip every additionalProperties value

This commit is contained in:
mateo-berri 2026-10-28 16:36:29 -07:00
parent 9ad9e425e9
commit c92f96e566
6 changed files with 119 additions and 7 deletions

View file

@ -522,8 +522,13 @@ def cost_per_token(
if _is_anthropic_style:
_normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
billed_prompt_tokens: Final = (
max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider))
if custom_cost_per_token is not None
else _normalized_prompt_tokens
)
response_cost: Final = _cost_per_token_custom_pricing_helper(
prompt_tokens=_normalized_prompt_tokens,
prompt_tokens=billed_prompt_tokens,
completion_tokens=completion_tokens,
response_time_ms=response_time_ms,
cached_tokens=_cache_read_tokens,
@ -2436,6 +2441,12 @@ def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
return raw_entry if isinstance(raw_entry, Mapping) else None
def _min_billed_input_tokens(model: str, custom_llm_provider: str | None) -> int:
cost_entry: Final = _raw_cost_map_entry(model) or _raw_cost_map_entry(f"{custom_llm_provider}/{model}") or {}
minimum: Final = cost_entry.get("min_billed_input_tokens")
return minimum if isinstance(minimum, int) else 0
def pricing_entry_for_cost_calc(
model: str | None,
completion_response: object | None,

View file

@ -17,7 +17,6 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
from litellm.utils import remove_additional_properties
from .json_loader import SimpleProviderConfig
@ -38,6 +37,16 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues
return updated
def _without_additional_properties(value: JsonValue) -> JsonValue:
if isinstance(value, dict):
return {
key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties"
}
if isinstance(value, list):
return [_without_additional_properties(item) for item in value]
return value
def create_config_class(provider: SimpleProviderConfig):
"""Generate config class dynamically from JSON configuration"""
@ -169,7 +178,7 @@ def create_config_class(provider: SimpleProviderConfig):
optional_params[provider.param_mappings[param]] = value
elif param in supported_params:
optional_params[param] = (
remove_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value))
_without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value))
if param == "response_format"
and provider.special_handling.get("remove_additional_properties_from_response_format")
else value

View file

@ -48595,6 +48595,7 @@
"supports_vision": true
},
"vispark/vision-small": {
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 1.05e-06,
"litellm_provider": "vispark",
"max_input_tokens": 1000000,
@ -48627,6 +48628,7 @@
"supports_vision": true
},
"vispark/vision-medium": {
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 4.21e-06,
"litellm_provider": "vispark",
"max_input_tokens": 1000000,
@ -48659,6 +48661,7 @@
"supports_vision": true
},
"vispark/vision-large": {
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 7.37e-06,
"litellm_provider": "vispark",
"max_input_tokens": 1000000,

View file

@ -11569,7 +11569,22 @@ class Router:
backend_model=backend_model,
custom_llm_provider=pricing_provider,
)
return model_info
billing_minimum: Final[Mapping[str, int]] = (
Router._builtin_billing_minimum(backend_model=backend_model, custom_llm_provider=pricing_provider)
if model_info.get("input_cost_per_token") is not None
else MappingProxyType({})
)
return {**billing_minimum, **model_info}
@staticmethod
def _builtin_billing_minimum(backend_model: str, custom_llm_provider: str | None) -> Mapping[str, int]:
backend_info: Final = lookup_model_info_for_registration(
model=backend_model, custom_llm_provider=custom_llm_provider
)
minimum: Final = (
None if isinstance(backend_info, ModelInfoLookupMiss) else backend_info.get("min_billed_input_tokens")
)
return MappingProxyType({} if minimum is None else {"min_billed_input_tokens": minimum})
@staticmethod
def _register_deployment_pricing(deployment: Deployment) -> None:

View file

@ -48595,6 +48595,7 @@
"supports_vision": true
},
"vispark/vision-small": {
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 1.05e-06,
"litellm_provider": "vispark",
"max_input_tokens": 1000000,
@ -48627,6 +48628,7 @@
"supports_vision": true
},
"vispark/vision-medium": {
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 4.21e-06,
"litellm_provider": "vispark",
"max_input_tokens": 1000000,
@ -48659,6 +48661,7 @@
"supports_vision": true
},
"vispark/vision-large": {
"cache_read_input_token_cost": 0.0,
"input_cost_per_token": 7.37e-06,
"litellm_provider": "vispark",
"max_input_tokens": 1000000,

View file

@ -126,10 +126,18 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens():
assert response.choices[0].message.content == "Hello from Vispark"
def test_vispark_strips_additional_properties_from_a_json_schema_response_format():
@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}])
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(
additional_properties: bool | dict[str, str],
):
city_schema: Final = {
"type": "object",
"properties": {"name": {"type": "string"}},
"additionalProperties": additional_properties,
}
schema: Final = {
"type": "object",
"properties": {"city": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}},
"properties": {"city": city_schema},
"required": ["city"],
"additionalProperties": False,
}
@ -149,7 +157,30 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format
assert "additionalProperties" not in json.dumps(sent_format)
assert sent_format["json_schema"]["strict"] is True
assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}}
assert schema["additionalProperties"] is False
assert city_schema["additionalProperties"] == additional_properties
def test_other_json_providers_keep_additional_properties_and_file_parts():
file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}
response_format: Final = {
"type": "json_schema",
"json_schema": {
"name": "City",
"schema": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False},
},
}
with respx.mock() as upstream:
route: Final = upstream.post("https://api.scaleway.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION)
litellm.completion(
model="scaleway/llama-3.3-70b-instruct",
messages=[{"role": "user", "content": [{"type": "text", "text": "Read this"}, file_part]}],
response_format=response_format,
api_key="scaleway-test-key",
)
body: Final = json.loads(route.calls.last.request.content)
assert body["response_format"] == response_format
assert body["messages"][0]["content"][1] == file_part
def test_vispark_sends_a_pdf_file_part_as_an_image_url():
@ -225,6 +256,46 @@ def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str):
assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"])
def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens():
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
router: Final = litellm.Router(
model_list=[
{
"model_name": "vispark-custom-priced",
"litellm_params": {
"model": "vispark/vision-small",
"api_key": "vispark-test-key",
"input_cost_per_token": 2e-6,
"output_cost_per_token": 5e-6,
},
}
]
)
with respx.mock() as upstream:
upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(200, json=_CHAT_COMPLETION)
response: Final = router.completion(
model="vispark-custom-priced", messages=[{"role": "user", "content": "Say hello"}]
)
completion_tokens: Final = _CHAT_COMPLETION["usage"]["completion_tokens"]
assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6)
def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens():
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
prompt_cost, completion_cost = litellm.cost_per_token(
model="vision-small",
custom_llm_provider="vispark",
prompt_tokens=minimum // 2,
completion_tokens=3,
custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6},
)
assert prompt_cost == pytest.approx(minimum * 2e-6)
assert completion_cost == pytest.approx(3 * 5e-6)
def test_vispark_models_are_listed_for_the_provider():
from litellm.utils import get_valid_models