mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(vispark): scope the input minimum per deployment and reject map-field schemas
Keep a deployment's min_billed_input_tokens on its own cost-map entry so it no longer changes billing for other deployments of the same backend model. Strip boolean additionalProperties for Vispark, answer 400 for schema-valued ones unless drop_params is set, and do it without recursion.
This commit is contained in:
parent
b6be3afe70
commit
2105d7ac6d
6 changed files with 146 additions and 35 deletions
|
|
@ -294,6 +294,9 @@ pub struct ModelInfo {
|
|||
/// Free-form notes about the entry (e.g. pricing derivation).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<BTreeMap<String, Value>>,
|
||||
/// Input tokens billed per request even when the prompt is shorter.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_billed_input_tokens: Option<u64>,
|
||||
/// Primary API surface / task type of the model.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mode: Option<Mode>,
|
||||
|
|
|
|||
|
|
@ -2,12 +2,15 @@
|
|||
Dynamic configuration class generator for JSON-based providers.
|
||||
"""
|
||||
|
||||
from collections.abc import Coroutine, Mapping
|
||||
import json
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from typing import TYPE_CHECKING, Final, Literal, overload
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
migrate_file_to_image_url,
|
||||
|
|
@ -37,14 +40,35 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues
|
|||
return updated
|
||||
|
||||
|
||||
def _without_additional_properties(value: JsonValue) -> JsonValue:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties"
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [_without_additional_properties(item) for item in value]
|
||||
return value
|
||||
def _without_additional_properties_flags(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
|
||||
return {key: item for key, item in node.items() if key != "additionalProperties" or not isinstance(item, bool)}
|
||||
|
||||
|
||||
def _without_additional_properties_key(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]:
|
||||
return {key: item for key, item in node.items() if key != "additionalProperties"}
|
||||
|
||||
|
||||
def _rebuilt_json(
|
||||
value: JsonValue, object_hook: Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]]
|
||||
) -> JsonValue:
|
||||
return _JSON_VALUE_ADAPTER.validate_python(json.loads(json.dumps(value), object_hook=object_hook))
|
||||
|
||||
|
||||
def _response_format_without_additional_properties(
|
||||
response_format: JsonValue, provider_slug: str, model: str, drop_params: bool
|
||||
) -> JsonValue:
|
||||
without_flags: Final = _rebuilt_json(response_format, _without_additional_properties_flags)
|
||||
without_any: Final = _rebuilt_json(response_format, _without_additional_properties_key)
|
||||
if without_flags == without_any or drop_params:
|
||||
return without_any
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"{provider_slug} does not support schema-valued additionalProperties (map fields) in response_format. "
|
||||
"To drop them, set `drop_params=True`"
|
||||
),
|
||||
llm_provider=provider_slug,
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
def create_config_class(provider: SimpleProviderConfig):
|
||||
|
|
@ -178,7 +202,12 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
optional_params[provider.param_mappings[param]] = value
|
||||
elif param in supported_params:
|
||||
optional_params[param] = (
|
||||
_without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value))
|
||||
_response_format_without_additional_properties(
|
||||
_JSON_VALUE_ADAPTER.validate_python(value),
|
||||
provider_slug=provider.slug,
|
||||
model=model,
|
||||
drop_params=drop_params or litellm.drop_params is True,
|
||||
)
|
||||
if param == "response_format"
|
||||
and provider.special_handling.get("remove_additional_properties_from_response_format")
|
||||
else value
|
||||
|
|
|
|||
|
|
@ -10331,19 +10331,20 @@ class Router:
|
|||
model_info[field] = copy.deepcopy(backend_value)
|
||||
|
||||
@staticmethod
|
||||
def _inherit_builtin_base_rates_for_off_peak(
|
||||
def _inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info: dict, # mutable-ok: cost-map entry filled in place
|
||||
backend_model: str,
|
||||
custom_llm_provider: str | None,
|
||||
) -> None:
|
||||
"""Fill missing pricing fields on a deployment entry that only sets
|
||||
``off_peak_pricing``, from the backend model's built-in cost map entry.
|
||||
``off_peak_pricing`` or ``min_billed_input_tokens``, from the backend
|
||||
model's built-in cost map entry.
|
||||
|
||||
Cost lookup selects the deployment-scoped entry over the shared backend
|
||||
entry only when the deployment entry carries a base pricing field, and
|
||||
``off_peak_pricing`` is deliberately kept off the shared entry, so a
|
||||
deployment spelling out only its off-peak schedule would otherwise
|
||||
never receive the discount. The backend model's entire canonical cost
|
||||
both fields are deliberately kept off the shared entry, so a deployment
|
||||
spelling out only its off-peak schedule or input minimum would otherwise
|
||||
never have it applied. The backend model's entire canonical cost
|
||||
map entry is copied, field by field, so threshold, tiered,
|
||||
service-tier, cache, character, and per-second rates as well as
|
||||
companion billing fields like ``web_search_billing_unit`` and the
|
||||
|
|
@ -10357,7 +10358,7 @@ class Router:
|
|||
no-op when any base pricing field is already set or the backend model
|
||||
has no canonical entry.
|
||||
"""
|
||||
if not model_info.get("off_peak_pricing"):
|
||||
if not model_info.get("off_peak_pricing") and model_info.get("min_billed_input_tokens") is None:
|
||||
return
|
||||
if any(
|
||||
model_info.get(field) is not None
|
||||
|
|
@ -11548,7 +11549,7 @@ class Router:
|
|||
model_info.update(
|
||||
{key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)}
|
||||
)
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider=pricing_provider,
|
||||
|
|
|
|||
|
|
@ -4130,7 +4130,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
return {k: v for k, v in model_info.items() if k not in cls.model_fields}
|
||||
|
||||
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
|
||||
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing", "min_billed_input_tokens"})
|
||||
|
||||
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"}
|
||||
|
|
|
|||
|
|
@ -126,10 +126,8 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens():
|
|||
assert response.choices[0].message.content == "Hello from Vispark"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}])
|
||||
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(
|
||||
additional_properties: bool | dict[str, str],
|
||||
):
|
||||
@pytest.mark.parametrize("additional_properties", [False, True])
|
||||
def test_vispark_strips_additional_properties_from_a_json_schema_response_format(additional_properties: bool):
|
||||
city_schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}},
|
||||
|
|
@ -160,6 +158,50 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format
|
|||
assert city_schema["additionalProperties"] == additional_properties
|
||||
|
||||
|
||||
def _map_field_response_format() -> dict[str, object]:
|
||||
schema: Final = {
|
||||
"type": "object",
|
||||
"properties": {"facts": {"type": "object", "additionalProperties": {"type": "string"}}},
|
||||
"required": ["facts"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
return {"type": "json_schema", "json_schema": {"name": "Facts", "schema": schema}}
|
||||
|
||||
|
||||
def test_vispark_rejects_a_map_field_response_format_without_drop_params():
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as error:
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Facts about Paris as JSON"}],
|
||||
response_format=_map_field_response_format(),
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert route.call_count == 0
|
||||
|
||||
|
||||
def test_vispark_drops_a_map_field_schema_with_drop_params():
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json=_CHAT_COMPLETION
|
||||
)
|
||||
litellm.completion(
|
||||
model="vispark/vision-medium",
|
||||
messages=[{"role": "user", "content": "Facts about Paris as JSON"}],
|
||||
response_format=_map_field_response_format(),
|
||||
drop_params=True,
|
||||
api_key="vispark-test-key",
|
||||
)
|
||||
|
||||
sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"]
|
||||
assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]}
|
||||
|
||||
|
||||
def test_other_json_providers_keep_additional_properties_and_file_parts():
|
||||
file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}
|
||||
response_format: Final = {
|
||||
|
|
@ -281,6 +323,42 @@ def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_i
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6)
|
||||
|
||||
|
||||
def test_vispark_deployment_minimum_override_stays_on_its_own_deployment():
|
||||
builtin: Final = litellm.model_cost["vispark/vision-small"]
|
||||
minimum: Final = builtin["min_billed_input_tokens"]
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vispark-no-minimum",
|
||||
"litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"},
|
||||
"model_info": {"min_billed_input_tokens": 0},
|
||||
},
|
||||
{
|
||||
"model_name": "vispark-default",
|
||||
"litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"},
|
||||
},
|
||||
]
|
||||
)
|
||||
with respx.mock() as upstream:
|
||||
upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(
|
||||
200, json={**_CHAT_COMPLETION, "model": "vision-small"}
|
||||
)
|
||||
no_minimum: Final = router.completion(
|
||||
model="vispark-no-minimum", messages=[{"role": "user", "content": "Say hello"}]
|
||||
)
|
||||
default: Final = router.completion(model="vispark-default", messages=[{"role": "user", "content": "Say hello"}])
|
||||
|
||||
usage: Final = _CHAT_COMPLETION["usage"]
|
||||
completion_cost: Final = usage["completion_tokens"] * builtin["output_cost_per_token"]
|
||||
assert no_minimum._hidden_params["response_cost"] == pytest.approx(
|
||||
usage["prompt_tokens"] * builtin["input_cost_per_token"] + completion_cost
|
||||
)
|
||||
assert default._hidden_params["response_cost"] == pytest.approx(
|
||||
minimum * builtin["input_cost_per_token"] + completion_cost
|
||||
)
|
||||
assert litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] == minimum
|
||||
|
||||
|
||||
def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens():
|
||||
minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"]
|
||||
|
||||
|
|
|
|||
|
|
@ -1637,7 +1637,7 @@ def test_cost_map_backend_model_uses_canonical_model_precedence(
|
|||
assert Router._cost_map_backend_model(deployment) == expected
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_fills_missing_rates():
|
||||
"""Direct unit test of the helper: an entry carrying only an
|
||||
off_peak_pricing block inherits the backend model's built-in base token
|
||||
rates, so cost lookup via the deployment id can bill standard rates
|
||||
|
|
@ -1652,7 +1652,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
|||
}
|
||||
model_info = {"off_peak_pricing": off_peak_block}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -1663,7 +1663,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates():
|
|||
assert model_info["off_peak_pricing"] == off_peak_block
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_carries_threshold_rates():
|
||||
"""A backend with above-threshold pricing hands the whole rate structure to
|
||||
the deployment entry, so peak-hour billing of large prompts through that
|
||||
entry matches the shared backend entry instead of flattening to the base
|
||||
|
|
@ -1677,7 +1677,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
|||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="gemini",
|
||||
|
|
@ -1692,7 +1692,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates():
|
|||
)
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_carries_companion_billing_fields():
|
||||
"""Billing rules that are not literal cost rates, like the web search
|
||||
billing unit, must ride along, or grounding and regional uplifts would
|
||||
bill differently through the deployment entry than through the shared
|
||||
|
|
@ -1706,7 +1706,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field
|
|||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider=None,
|
||||
|
|
@ -1716,7 +1716,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field
|
|||
assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"]
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_tiered_only_backend_stores_no_zero():
|
||||
"""A tiered-only backend has no flat token rates; get_model_info synthesizes
|
||||
zeros for them, and storing those would mark the deployment explicitly
|
||||
priced free. The tier table itself must carry over as an isolated copy so
|
||||
|
|
@ -1729,7 +1729,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z
|
|||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07},
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model=backend_model,
|
||||
custom_llm_provider="dashscope",
|
||||
|
|
@ -1746,7 +1746,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z
|
|||
assert raw_tiers[0] == original_first_tier
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_leaves_explicit_rates_alone():
|
||||
"""An entry that sets its own base rate beside the block already counts as
|
||||
a full custom pricing entry; the helper must not mix builtin rates into it.
|
||||
"""
|
||||
|
|
@ -1755,7 +1755,7 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
|||
"input_cost_per_token": 3e-06,
|
||||
}
|
||||
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=model_info,
|
||||
backend_model="gpt-4o-mini",
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -1765,12 +1765,12 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone():
|
|||
assert "output_cost_per_token" not in model_info
|
||||
|
||||
|
||||
def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend():
|
||||
def test_inherit_builtin_base_rates_for_scoped_pricing_noop_without_block_or_backend():
|
||||
"""Nothing happens without an off_peak_pricing block, and an unmapped
|
||||
backend model leaves the entry unchanged rather than raising.
|
||||
"""
|
||||
plain_info = {"id": "dep-1"}
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=plain_info,
|
||||
backend_model="gpt-4o-mini",
|
||||
custom_llm_provider="openai",
|
||||
|
|
@ -1778,7 +1778,7 @@ def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend()
|
|||
assert plain_info == {"id": "dep-1"}
|
||||
|
||||
off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}}
|
||||
Router._inherit_builtin_base_rates_for_off_peak(
|
||||
Router._inherit_builtin_base_rates_for_scoped_pricing(
|
||||
model_info=off_peak_info,
|
||||
backend_model="this-backend-model-does-not-exist-x9y8z7",
|
||||
custom_llm_provider=None,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue