fix(vispark): scope the input minimum per deployment and reject map-field schemas

Keep a deployment's min_billed_input_tokens on its own cost-map entry so it
no longer changes billing for other deployments of the same backend model.
Strip boolean additionalProperties for Vispark, answer 400 for schema-valued
ones unless drop_params is set, and do it without recursion.
This commit is contained in:
mateo-berri 2026-10-28 18:42:43 -07:00
parent b6be3afe70
commit 2105d7ac6d
6 changed files with 146 additions and 35 deletions

View file

@ -294,6 +294,9 @@ pub struct ModelInfo {
/// Free-form notes about the entry (e.g. pricing derivation).
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<BTreeMap<String, Value>>,
/// Input tokens billed per request even when the prompt is shorter.
#[serde(skip_serializing_if = "Option::is_none")]
pub min_billed_input_tokens: Option<u64>,
/// Primary API surface / task type of the model.
#[serde(skip_serializing_if = "Option::is_none")]
pub mode: Option<Mode>,

View file

@ -2,12 +2,15 @@
Dynamic configuration class generator for JSON-based providers.
"""
from collections.abc import Coroutine, Mapping
import json
from collections.abc import Callable, Coroutine, Mapping
from typing import TYPE_CHECKING, Final, Literal, overload
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.prompt_templates.common_utils import (
handle_messages_with_content_list_to_str_conversion,
migrate_file_to_image_url,
@ -37,14 +40,35 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues
return updated
def _without_additional_properties(value: JsonValue) -> JsonValue:
if isinstance(value, dict):
return {
key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties"
}
if isinstance(value, list):
return [_without_additional_properties(item) for item in value]
return value
def _without_additional_properties_flags(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
return {key: item for key, item in node.items() if key != "additionalProperties" or not isinstance(item, bool)}
def _without_additional_properties_key(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
return {key: item for key, item in node.items() if key != "additionalProperties"}
def _rebuilt_json(
value: JsonValue, object_hook: Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]]
) -> JsonValue:
return _JSON_VALUE_ADAPTER.validate_python(json.loads(json.dumps(value), object_hook=object_hook))
def _response_format_without_additional_properties(
response_format: JsonValue, provider_slug: str, model: str, drop_params: bool
) -> JsonValue:
without_flags: Final = _rebuilt_json(response_format, _without_additional_properties_flags)
without_any: Final = _rebuilt_json(response_format, _without_additional_properties_key)
if without_flags == without_any or drop_params:
return without_any
raise UnsupportedParamsError(
message=(
f"{provider_slug} does not support schema-valued additionalProperties (map fields) in response_format. "
"To drop them, set `drop_params=True`"
),
llm_provider=provider_slug,
model=model,
)
def create_config_class(provider: SimpleProviderConfig):
@ -178,7 +202,12 @@ def create_config_class(provider: SimpleProviderConfig):
optional_params[provider.param_mappings[param]] = value
elif param in supported_params:
optional_params[param] = (
_without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value))
_response_format_without_additional_properties(
_JSON_VALUE_ADAPTER.validate_python(value),
provider_slug=provider.slug,
model=model,
drop_params=drop_params or litellm.drop_params is True,
)
if param == "response_format"
and provider.special_handling.get("remove_additional_properties_from_response_format")
else value

View file

@ -10331,19 +10331,20 @@ class Router:
model_info[field] = copy.deepcopy(backend_value)
@staticmethod
def _inherit_builtin_base_rates_for_off_peak(
def _inherit_builtin_base_rates_for_scoped_pricing(
model_info: dict, # mutable-ok: cost-map entry filled in place
backend_model: str,
custom_llm_provider: str | None,
) -> None:
"""Fill missing pricing fields on a deployment entry that only sets
``off_peak_pricing``, from the backend model's built-in cost map entry.
``off_peak_pricing`` or ``min_billed_input_tokens``, from the backend
model's built-in cost map entry.
Cost lookup selects the deployment-scoped entry over the shared backend
entry only when the deployment entry carries a base pricing field, and
``off_peak_pricing`` is deliberately kept off the shared entry, so a
deployment spelling out only its off-peak schedule would otherwise
never receive the discount. The backend model's entire canonical cost
both fields are deliberately kept off the shared entry, so a deployment
spelling out only its off-peak schedule or input minimum would otherwise
never have it applied. The backend model's entire canonical cost
map entry is copied, field by field, so threshold, tiered,
service-tier, cache, character, and per-second rates as well as
companion billing fields like ``web_search_billing_unit`` and the
@ -10357,7 +10358,7 @@ class Router:
no-op when any base pricing field is already set or the backend model
has no canonical entry.
"""
if not model_info.get("off_peak_pricing"):
if not model_info.get("off_peak_pricing") and model_info.get("min_billed_input_tokens") is None:
return
if any(
model_info.get(field) is not None
@ -11548,7 +11549,7 @@ class Router:
model_info.update(
{key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)}
)
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider=pricing_provider,

View file

@ -4130,7 +4130,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
return {k: v for k, v in model_info.items() if k not in cls.model_fields}
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing", "min_billed_input_tokens"})
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
{"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"}

View file

@ -126,10 +126,8 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens():
assert response.choices[0].message.content == "Hello from Vispark"
@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}])
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(
additional_properties: bool | dict[str, str],
):
@pytest.mark.parametrize("additional_properties", [False, True])
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(additional_properties: bool):
city_schema: Final = {
"type": "object",
"properties": {"name": {"type": "string"}},
@ -160,6 +158,50 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format
assert city_schema["additionalProperties"] == additional_properties
def _map_field_response_format() -> dict[str, object]:
schema: Final = {
"type": "object",
"properties": {"facts": {"type": "object", "additionalProperties": {"type": "string"}}},
"required": ["facts"],
"additionalProperties": False,
}
return {"type": "json_schema", "json_schema": {"name": "Facts", "schema": schema}}
def test_vispark_rejects_a_map_field_response_format_without_drop_params():
with respx.mock(assert_all_called=False) as upstream:
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
200, json=_CHAT_COMPLETION
)
with pytest.raises(litellm.UnsupportedParamsError) as error:
litellm.completion(
model="vispark/vision-medium",
messages=[{"role": "user", "content": "Facts about Paris as JSON"}],
response_format=_map_field_response_format(),
api_key="vispark-test-key",
)
assert error.value.status_code == 400
assert route.call_count == 0
def test_vispark_drops_a_map_field_schema_with_drop_params():
with respx.mock() as upstream:
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
200, json=_CHAT_COMPLETION
)
litellm.completion(
model="vispark/vision-medium",
messages=[{"role": "user", "content": "Facts about Paris as JSON"}],
response_format=_map_field_response_format(),
drop_params=True,
api_key="vispark-test-key",
)
sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"]
assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]}
def test_other_json_providers_keep_additional_properties_and_file_parts():
file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}
response_format: Final = {
@ -281,6 +323,42 @@ def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_i
assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6)
def test_vispark_deployment_minimum_override_stays_on_its_own_deployment():
builtin: Final = litellm.model_cost["vispark/vision-small"]
minimum: Final = builtin["min_billed_input_tokens"]
router: Final = litellm.Router(
model_list=[
{
"model_name": "vispark-no-minimum",
"litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"},
"model_info": {"min_billed_input_tokens": 0},
},
{
"model_name": "vispark-default",
"litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"},
},
]
)
with respx.mock() as upstream:
upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
200, json={**_CHAT_COMPLETION, "model": "vision-small"}
)
no_minimum: Final = router.completion(
model="vispark-no-minimum", messages=[{"role": "user", "content": "Say hello"}]
)
default: Final = router.completion(model="vispark-default", messages=[{"role": "user", "content": "Say hello"}])
usage: Final = _CHAT_COMPLETION["usage"]
completion_cost: Final = usage["completion_tokens"] * builtin["output_cost_per_token"]
assert no_minimum._hidden_params["response_cost"] == pytest.approx(
usage["prompt_tokens"] * builtin["input_cost_per_token"] + completion_cost
)
assert default._hidden_params["response_cost"] == pytest.approx(
minimum * builtin["input_cost_per_token"] + completion_cost
)
assert litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] == minimum
def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens():
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]

View file

@ -1637,7 +1637,7 @@ def test_cost_map_backend_model_uses_canonical_model_precedence(
assert Router._cost_map_backend_model(deployment) == expected
def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
def test_inherit_builtin_base_rates_for_scoped_pricing_fills_missing_rates():
"""Direct unit test of the helper: an entry carrying only an
off_peak_pricing block inherits the backend model's built-in base token
rates, so cost lookup via the deployment id can bill standard rates
@ -1652,7 +1652,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
}
model_info = {"off_peak_pricing": off_peak_block}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider="openai",
@ -1663,7 +1663,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
assert model_info["off_peak_pricing"] == off_peak_block
def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
def test_inherit_builtin_base_rates_for_scoped_pricing_carries_threshold_rates():
"""A backend with above-threshold pricing hands the whole rate structure to
the deployment entry, so peak-hour billing of large prompts through that
entry matches the shared backend entry instead of flattening to the base
@ -1677,7 +1677,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider="gemini",
@ -1692,7 +1692,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
)
def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields():
def test_inherit_builtin_base_rates_for_scoped_pricing_carries_companion_billing_fields():
"""Billing rules that are not literal cost rates, like the web search
billing unit, must ride along, or grounding and regional uplifts would
bill differently through the deployment entry than through the shared
@ -1706,7 +1706,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider=None,
@ -1716,7 +1716,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field
assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"]
def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero():
def test_inherit_builtin_base_rates_for_scoped_pricing_tiered_only_backend_stores_no_zero():
"""A tiered-only backend has no flat token rates; get_model_info synthesizes
zeros for them, and storing those would mark the deployment explicitly
priced free. The tier table itself must carry over as an isolated copy so
@ -1729,7 +1729,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=model_info,
backend_model=backend_model,
custom_llm_provider="dashscope",
@ -1746,7 +1746,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z
assert raw_tiers[0] == original_first_tier
def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
def test_inherit_builtin_base_rates_for_scoped_pricing_leaves_explicit_rates_alone():
"""An entry that sets its own base rate beside the block already counts as
a full custom pricing entry; the helper must not mix builtin rates into it.
"""
@ -1755,7 +1755,7 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
"input_cost_per_token": 3e-06,
}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=model_info,
backend_model="gpt-4o-mini",
custom_llm_provider="openai",
@ -1765,12 +1765,12 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
assert "output_cost_per_token" not in model_info
def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend():
def test_inherit_builtin_base_rates_for_scoped_pricing_noop_without_block_or_backend():
"""Nothing happens without an off_peak_pricing block, and an unmapped
backend model leaves the entry unchanged rather than raising.
"""
plain_info = {"id": "dep-1"}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=plain_info,
backend_model="gpt-4o-mini",
custom_llm_provider="openai",
@ -1778,7 +1778,7 @@ def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend()
assert plain_info == {"id": "dep-1"}
off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}}
Router._inherit_builtin_base_rates_for_off_peak(
Router._inherit_builtin_base_rates_for_scoped_pricing(
model_info=off_peak_info,
backend_model="this-backend-model-does-not-exist-x9y8z7",
custom_llm_provider=None,