mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(vispark): bill the input token minimum under custom pricing and strip every additionalProperties value
This commit is contained in:
parent
9ad9e425e9
commit
c92f96e566
6 changed files with 119 additions and 7 deletions
|
|
@ -522,8 +522,13 @@ def cost_per_token(
|
|||
if _is_anthropic_style:
|
||||
_normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens
|
||||
|
||||
billed_prompt_tokens: Final = (
|
||||
max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider))
|
||||
if custom_cost_per_token is not None
|
||||
else _normalized_prompt_tokens
|
||||
)
|
||||
response_cost: Final = _cost_per_token_custom_pricing_helper(
|
||||
prompt_tokens=_normalized_prompt_tokens,
|
||||
prompt_tokens=billed_prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time_ms=response_time_ms,
|
||||
cached_tokens=_cache_read_tokens,
|
||||
|
|
@ -2436,6 +2441,12 @@ def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
|
|||
return raw_entry if isinstance(raw_entry, Mapping) else None
|
||||
|
||||
|
||||
def _min_billed_input_tokens(model: str, custom_llm_provider: str | None) -> int:
|
||||
cost_entry: Final = _raw_cost_map_entry(model) or _raw_cost_map_entry(f"{custom_llm_provider}/{model}") or {}
|
||||
minimum: Final = cost_entry.get("min_billed_input_tokens")
|
||||
return minimum if isinstance(minimum, int) else 0
|
||||
|
||||
|
||||
def pricing_entry_for_cost_calc(
|
||||
model: str | None,
|
||||
completion_response: object | None,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
|||
from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
|
||||
from litellm.utils import remove_additional_properties
|
||||
|
||||
from .json_loader import SimpleProviderConfig
|
||||
|
||||
|
|
@ -38,6 +37,16 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues
|
|||
return updated
|
||||
|
||||
|
||||
def _without_additional_properties(value: JsonValue) -> JsonValue:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties"
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_without_additional_properties(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def create_config_class(provider: SimpleProviderConfig):
|
||||
"""Generate config class dynamically from JSON configuration"""
|
||||
|
||||
|
|
@ -169,7 +178,7 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
optional_params[provider.param_mappings[param]] = value
|
||||
elif param in supported_params:
|
||||
optional_params[param] = (
|
||||
remove_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value))
|
||||
_without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value))
|
||||
if param == "response_format"
|
||||
and provider.special_handling.get("remove_additional_properties_from_response_format")
|
||||
else value
|
||||
|
|
|
|||
|
|
@ -48595,6 +48595,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-small": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 1.05e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
|
|
@ -48627,6 +48628,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-medium": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 4.21e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
|
|
@ -48659,6 +48661,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-large": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 7.37e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
|
|
|
|||
|
|
@ -11569,7 +11569,22 @@ class Router:
|
|||
backend_model=backend_model,
|
||||
custom_llm_provider=pricing_provider,
|
||||
)
|
||||
return model_info
|
||||
billing_minimum: Final[Mapping[str, int]] = (
|
||||
Router._builtin_billing_minimum(backend_model=backend_model, custom_llm_provider=pricing_provider)
|
||||
if model_info.get("input_cost_per_token") is not None
|
||||
else MappingProxyType({})
|
||||
)
|
||||
return {**billing_minimum, **model_info}
|
||||
|
||||
@staticmethod
|
||||
def _builtin_billing_minimum(backend_model: str, custom_llm_provider: str | None) -> Mapping[str, int]:
|
||||
backend_info: Final = lookup_model_info_for_registration(
|
||||
model=backend_model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
minimum: Final = (
|
||||
None if isinstance(backend_info, ModelInfoLookupMiss) else backend_info.get("min_billed_input_tokens")
|
||||
)
|
||||
return MappingProxyType({} if minimum is None else {"min_billed_input_tokens": minimum})
|
||||
|
||||
@staticmethod
|
||||
def _register_deployment_pricing(deployment: Deployment) -> None:
|
||||
|
|
|
|||
|
|
@ -48595,6 +48595,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-small": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 1.05e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
|
|
@ -48627,6 +48628,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-medium": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 4.21e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
|
|
@ -48659,6 +48661,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"vispark/vision-large": {
|
||||
"cache_read_input_token_cost": 0.0,
|
||||
"input_cost_per_token": 7.37e-06,
|
||||
"litellm_provider": "vispark",
|
||||
"max_input_tokens": 1000000,
|
||||
|
|
|
|||
|
|
@ -126,10 +126,18 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens():
|
|||
assert response.choices[0].message.content == "Hello from Vispark"
|
||||
|
||||
|
||||
def test_vispark_strips_additional_properties_from_a_json_schema_response_format():
|
||||
@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}])
|
||||
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(
|
||||
additional_properties: bool | dict[str, str],
|
||||
):
|
||||
city_schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}},
|
||||
"additionalProperties": additional_properties,
|
||||
}
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}},
|
||||
"properties": {"city": city_schema},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
|
@ -149,7 +157,30 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format
|
|||
assert "additionalProperties" not in json.dumps(sent_format)
|
||||
assert sent_format["json_schema"]["strict"] is True
|
||||
assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}}
|
||||
assert schema["additionalProperties"] is False
|
||||
assert city_schema["additionalProperties"] == additional_properties
|
||||
|
||||
|
||||
def test_other_json_providers_keep_additional_properties_and_file_parts():
|
||||
file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}
|
||||
response_format: Final = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "City",
|
||||
"schema": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False},
|
||||
},
|
||||
}
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.scaleway.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION)
|
||||
litellm.completion(
|
||||
model="scaleway/llama-3.3-70b-instruct",
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "Read this"}, file_part]}],
|
||||
response_format=response_format,
|
||||
api_key="scaleway-test-key",
|
||||
)
|
||||
|
||||
body: Final = json.loads(route.calls.last.request.content)
|
||||
assert body["response_format"] == response_format
|
||||
assert body["messages"][0]["content"][1] == file_part
|
||||
|
||||
|
||||
def test_vispark_sends_a_pdf_file_part_as_an_image_url():
|
||||
|
|
@ -225,6 +256,46 @@ def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str):
|
|||
assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"])
|
||||
|
||||
|
||||
def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens():
|
||||
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vispark-custom-priced",
|
||||
"litellm_params": {
|
||||
"model": "vispark/vision-small",
|
||||
"api_key": "vispark-test-key",
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 5e-6,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(200, json=_CHAT_COMPLETION)
|
||||
response: Final = router.completion(
|
||||
model="vispark-custom-priced", messages=[{"role": "user", "content": "Say hello"}]
|
||||
)
|
||||
|
||||
completion_tokens: Final = _CHAT_COMPLETION["usage"]["completion_tokens"]
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6)
|
||||
|
||||
|
||||
def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens():
|
||||
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
|
||||
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="vision-small",
|
||||
custom_llm_provider="vispark",
|
||||
prompt_tokens=minimum // 2,
|
||||
completion_tokens=3,
|
||||
custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6},
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(minimum * 2e-6)
|
||||
assert completion_cost == pytest.approx(3 * 5e-6)
|
||||
|
||||
|
||||
def test_vispark_models_are_listed_for_the_provider():
|
||||
from litellm.utils import get_valid_models
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue