From b00ca54e3f6af592bf3656d01ccf607564d274f0 Mon Sep 17 00:00:00 2001 From: Vansh Date: Fri, 2 Oct 2026 13:04:52 +0530 Subject: [PATCH 01/18] feat(litellm): add Vispark provider with Vision Small/Medium/Large Rebased onto latest main: re-applied vispark registrations (model sets, provider enum, JSON provider registry, pricing catalogs, dashboard fields, endpoints matrix) and moved provider test to tests/unit/llms/openai_like/ per current layout. --- litellm/__init__.py | 5 + litellm/constants.py | 3 + litellm/llms/openai_like/providers.json | 9 + ...odel_prices_and_context_window_backup.json | 96 ++++++++++ .../provider_create_fields.json | 28 +++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 96 ++++++++++ provider_endpoints_support.json | 17 ++ .../llms/openai_like/test_vispark_provider.py | 180 ++++++++++++++++++ 9 files changed, 435 insertions(+) create mode 100644 tests/unit/llms/openai_like/test_vispark_provider.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 9a4f4605519..b2dd33e192a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -688,6 +688,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() v0_models: Set = set() +vispark_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() inception_models: Set = set() @@ -954,6 +955,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: darkbloom_models.add(key) elif value.get("litellm_provider") == "v0": v0_models.add(key) + elif value.get("litellm_provider") == "vispark": + vispark_models.add(key) elif value.get("litellm_provider") == "morph": morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": @@ -1116,6 +1119,7 @@ model_list = list( | publicai_models | darkbloom_models | v0_models + | vispark_models | morph_models | lambda_ai_models | inception_models @@ -1228,6 +1232,7 @@ def _build_models_by_provider() -> dict: "publicai": publicai_models, "darkbloom": darkbloom_models, "v0": v0_models, + "vispark": vispark_models, "morph": morph_models, "lambda_ai": lambda_ai_models, "inception": inception_models, diff --git a/litellm/constants.py b/litellm/constants.py index af4d1268c03..b9433a70147 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -963,6 +963,7 @@ openai_compatible_endpoints: Final[list] = [ "https://api.cognition.ai/v1", "https://api.cortecs.ai/v1", "https://api.scx.ai/v1", + "https://api.lab.vispark.in/v1", "https://api.prisminference.com/v1", "https://gigachat.devices.sberbank.ru/api/v1", ] @@ -1034,6 +1035,7 @@ openai_compatible_providers: Final[list] = [ "ragflow", "pinstripes", # Pinstripes - JSON-configured provider "darkbloom", + "vispark", # Vispark Lab - JSON-configured provider "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", "cortecs", @@ -1065,6 +1067,7 @@ openai_text_completion_compatible_providers: Final[list] = [ # providers that s "poe", "chutes", "v0", + "vispark", "lambda_ai", "hyperbolic", "wandb", diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 61ff4be3a46..6442244f110 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -218,5 +218,14 @@ "api_key_env": "SAIL_API_KEY", "api_base_env": "SAIL_API_BASE", "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + }, + "vispark": { + "base_url": "https://api.lab.vispark.in/v1", + "api_key_env": "VISPARK_LAB_API_KEY", + "api_base_env": "VISPARK_LAB_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "supported_endpoints": ["/v1/chat/completions", "/v1/completions"] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3b9956ad1a6..dadf188dd38 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47885,6 +47885,102 @@ "supports_tool_choice": true, "supports_vision": true }, + "vispark/vision-small": { + "input_cost_per_token": 1.05e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.16e-06, + "source": "https://lab.vispark.in/#vision", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-medium": { + "input_cost_per_token": 4.21e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.263e-05, + "source": "https://lab.vispark.in/#vision", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-large": { + "input_cost_per_token": 7.37e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.211e-05, + "source": "https://lab.vispark.in/#vision", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vercel_ai_gateway/alibaba/qwen-3-14b": { "input_cost_per_token": 8e-08, "litellm_provider": "vercel_ai_gateway", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 11d2ff61b95..80d630f3ab1 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3684,5 +3684,33 @@ } ], "default_model_placeholder": "cursor/claude-4-sonnet" + }, + { + "provider": "VISPARK", + "provider_display_name": "Vispark", + "litellm_provider": "vispark", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.lab.vispark.in/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "vispark/vision-medium" } ] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8c10b9e3497..150c25c53e6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4065,6 +4065,7 @@ class LlmProviders(str, Enum): MOONSHOT = "moonshot" PUBLICAI = "publicai" V0 = "v0" + VISPARK = "vispark" MORPH = "morph" LAMBDA_AI = "lambda_ai" INCEPTION = "inception" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3b9956ad1a6..dadf188dd38 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47885,6 +47885,102 @@ "supports_tool_choice": true, "supports_vision": true }, + "vispark/vision-small": { + "input_cost_per_token": 1.05e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.16e-06, + "source": "https://lab.vispark.in/#vision", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-medium": { + "input_cost_per_token": 4.21e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.263e-05, + "source": "https://lab.vispark.in/#vision", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "vispark/vision-large": { + "input_cost_per_token": 7.37e-06, + "litellm_provider": "vispark", + "max_input_tokens": 1000000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.211e-05, + "source": "https://lab.vispark.in/#vision", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vercel_ai_gateway/alibaba/qwen-3-14b": { "input_cost_per_token": 8e-08, "litellm_provider": "vercel_ai_gateway", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 9cbd326277e..d24ee40a5d4 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2626,6 +2626,23 @@ "interactions": true } }, + "vispark": { + "display_name": "Vispark (`vispark`)", + "url": "https://lab.vispark.in/#vision", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "vercel_ai_gateway": { "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py new file mode 100644 index 00000000000..92358fb2405 --- /dev/null +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -0,0 +1,180 @@ +""" +Tests for Vispark provider configuration and integration. +""" + +import litellm + + +class TestVisparkProviderConfig: + def test_vispark_in_provider_list(self): + from litellm import LlmProviders + + assert hasattr(LlmProviders, "VISPARK") + assert LlmProviders.VISPARK.value == "vispark" + assert "vispark" in litellm.provider_list + + def test_vispark_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("vispark") + + vispark = JSONProviderRegistry.get("vispark") + assert vispark is not None + assert vispark.base_url == "https://api.lab.vispark.in/v1" + assert vispark.api_key_env == "VISPARK_LAB_API_KEY" + assert vispark.api_base_env == "VISPARK_LAB_API_BASE" + assert vispark.param_mappings.get("max_completion_tokens") == "max_tokens" + + def test_vispark_in_openai_compatible_providers(self): + from litellm.constants import ( + openai_compatible_providers, + openai_text_completion_compatible_providers, + ) + + assert "vispark" in openai_compatible_providers + assert "vispark" in openai_text_completion_compatible_providers + + def test_vispark_provider_resolution(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert model == "vision-medium" + assert provider == "vispark" + assert api_base == "https://api.lab.vispark.in/v1" + + def test_vispark_api_base_override(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base="https://custom.vispark.in/v1", + api_key="sk-test", + ) + + assert provider == "vispark" + assert api_base == "https://custom.vispark.in/v1" + assert api_key == "sk-test" + + def test_vispark_url_autodetection(self): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="vision-medium", + custom_llm_provider=None, + api_base="https://api.lab.vispark.in/v1", + api_key=None, + ) + assert provider == "vispark" + assert api_base == "https://api.lab.vispark.in/v1" + + def test_vispark_max_completion_tokens_mapped(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("vispark") + assert provider is not None + config = create_config_class(provider)() + + optional_params = config.map_openai_params( + non_default_params={"max_completion_tokens": 256}, + optional_params={}, + model="vision-medium", + drop_params=False, + ) + assert optional_params["max_tokens"] == 256 + assert "max_completion_tokens" not in optional_params + + def test_vispark_router_config(self): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "vispark-chat", + "litellm_params": { + "model": "vispark/vision-medium", + "api_key": "test-key", + }, + } + ] + ) + + assert len(router.model_list) == 1 + assert router.model_list[0]["model_name"] == "vispark-chat" + + +class TestVisparkModelMetadata: + VISPARK_MODELS = ( + "vispark/vision-small", + "vispark/vision-medium", + "vispark/vision-large", + ) + + @staticmethod + def _load(path_parts): + import json + from pathlib import Path + + json_path = Path(__file__).parents[4].joinpath(*path_parts) + with open(json_path) as f: + return json.load(f) + + def test_vispark_models_registered_with_correct_metadata(self): + model_cost = self._load(("model_prices_and_context_window.json",)) + for model in self.VISPARK_MODELS: + info = model_cost.get(model) + assert info is not None, f"{model} missing from model_prices_and_context_window.json" + assert info["litellm_provider"] == "vispark" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] > 0 + assert info["output_cost_per_token"] > 0 + assert info["supports_function_calling"] is True + assert info["supports_tool_choice"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert info["supports_vision"] is True + + assert info["max_output_tokens"] == 65536 + assert info["max_tokens"] == info["max_output_tokens"] + assert info["max_input_tokens"] == 1000000 + + def test_vispark_models_synced_to_backup(self): + model_cost = self._load(("model_prices_and_context_window.json",)) + backup = self._load(("litellm", "model_prices_and_context_window_backup.json")) + for model in self.VISPARK_MODELS: + assert model in backup, f"{model} missing from backup json" + assert backup[model] == model_cost[model], f"{model} differs between root and backup json" + + +class TestVisparkDashboardRegistration: + @staticmethod + def _provider_create_fields(): + import json + from pathlib import Path + + import litellm + + path = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" + with open(path) as f: + return json.load(f) + + def test_vispark_is_selectable_in_the_add_model_form(self): + entries = [e for e in self._provider_create_fields() if e["litellm_provider"] == "vispark"] + assert len(entries) == 1, "vispark must appear exactly once in provider_create_fields.json" + + entry = entries[0] + assert entry["provider"] == "VISPARK" + assert entry["provider_display_name"] == "Vispark" + assert entry["default_model_placeholder"].startswith("vispark/") + + fields = {f["key"]: f for f in entry["credential_fields"]} + assert fields["api_key"]["required"] is True + assert fields["api_key"]["field_type"] == "password" + assert fields["api_base"]["required"] is False From 0a9cabdf8807654f62abe45dd26635bce4fb9866 Mon Sep 17 00:00:00 2001 From: Vansh Date: Fri, 2 Oct 2026 13:27:46 +0530 Subject: [PATCH 02/18] fix(litellm): keep basedpyright reportUnknownArgumentType budget flat for vispark --- litellm/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index b2dd33e192a..83c734e9000 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -33,6 +33,7 @@ from collections.abc import Mapping, Sequence from typing import ( Any, Callable, + cast, Dict, Final, get_args, @@ -956,7 +957,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "vispark": - vispark_models.add(key) + # cast: model keys are strings; avoids adding a reportUnknownArgumentType error + vispark_models.add(cast(str, key)) elif value.get("litellm_provider") == "morph": morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": From 130138b7716d8d3ca9a866cf834fc882179998bb Mon Sep 17 00:00:00 2001 From: Vansh Date: Fri, 2 Oct 2026 13:40:23 +0530 Subject: [PATCH 03/18] fix(litellm): satisfy type-discipline budget for vispark cast --- litellm/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 83c734e9000..97028866d6a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -957,8 +957,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "vispark": - # cast: model keys are strings; avoids adding a reportUnknownArgumentType error - vispark_models.add(cast(str, key)) + # cast keeps the basedpyright reportUnknownArgumentType budget flat (keys are model-name strings) + vispark_models.add(cast(str, key)) # cast-ok: model keys are strings elif value.get("litellm_provider") == "morph": morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": From b6a0f610edfe7e22dc2534363915ea9886aa65cb Mon Sep 17 00:00:00 2001 From: Vansh Date: Fri, 2 Oct 2026 14:35:07 +0530 Subject: [PATCH 04/18] chore: retrigger CI (prior MCP dep-resolution leg cancelled by infra) From 224f6f94c678cf5a733bbbf80c4bb30ce0fbfbe0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:56:19 +0000 Subject: [PATCH 05/18] test(vispark): cover routing, text completion and cost; sync endpoint matrix backup --- litellm/__init__.py | 6 +- ...odel_prices_and_context_window_backup.json | 6 +- .../provider_endpoints_support_backup.json | 18 + model_prices_and_context_window.json | 6 +- provider_endpoints_support.json | 3 +- .../llms/openai_like/test_vispark_provider.py | 327 +++++++++--------- 6 files changed, 190 insertions(+), 176 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 86ab0bb321f..b744f7245f4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -33,7 +33,6 @@ from collections.abc import Mapping, Sequence from typing import ( Any, Callable, - cast, Dict, Final, get_args, @@ -697,7 +696,7 @@ moonshot_models: Set = set() publicai_models: Set = set() darkbloom_models: Set = set() v0_models: Set = set() -vispark_models: Set = set() +vispark_models: Set[str] = set() morph_models: Set = set() lambda_ai_models: Set = set() inception_models: Set = set() @@ -969,8 +968,7 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje elif value.get("litellm_provider") == "v0": v0_models.add(key) elif value.get("litellm_provider") == "vispark": - # cast keeps the basedpyright reportUnknownArgumentType budget flat (keys are model-name strings) - vispark_models.add(cast(str, key)) # cast-ok: model keys are strings + vispark_models.add(key) elif value.get("litellm_provider") == "morph": morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 79c337259b6..08150f3d98a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48651,7 +48651,7 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.16e-06, - "source": "https://lab.vispark.in/#vision", + "source": "https://api.lab.vispark.in/v1/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -48683,7 +48683,7 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.263e-05, - "source": "https://lab.vispark.in/#vision", + "source": "https://api.lab.vispark.in/v1/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -48715,7 +48715,7 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.211e-05, - "source": "https://lab.vispark.in/#vision", + "source": "https://api.lab.vispark.in/v1/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index a10f2f93d48..a25eab5a5af 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2352,6 +2352,24 @@ "interactions": true } }, + "vispark": { + "display_name": "Vispark (`vispark`)", + "url": "https://lab.vispark.in/#vision", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "vercel_ai_gateway": { "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 79c337259b6..08150f3d98a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48651,7 +48651,7 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 3.16e-06, - "source": "https://lab.vispark.in/#vision", + "source": "https://api.lab.vispark.in/v1/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -48683,7 +48683,7 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 1.263e-05, - "source": "https://lab.vispark.in/#vision", + "source": "https://api.lab.vispark.in/v1/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" @@ -48715,7 +48715,7 @@ "max_tokens": 65536, "mode": "chat", "output_cost_per_token": 2.211e-05, - "source": "https://lab.vispark.in/#vision", + "source": "https://api.lab.vispark.in/v1/models", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions" diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 006e3ed84f4..53c244ddc81 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2728,7 +2728,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": false + "a2a": false, + "interactions": false } }, "vercel_ai_gateway": { diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 92358fb2405..5b93ae099ce 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -1,180 +1,177 @@ -""" -Tests for Vispark provider configuration and integration. -""" +import json +from pathlib import Path +from typing import Final + +import pytest +import respx import litellm +_CHAT_COMPLETION: Final = { + "id": "chatcmpl_vispark", + "object": "chat.completion", + "created": 1_790_000_000, + "model": "vision-medium", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello from Vispark"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} -class TestVisparkProviderConfig: - def test_vispark_in_provider_list(self): - from litellm import LlmProviders - - assert hasattr(LlmProviders, "VISPARK") - assert LlmProviders.VISPARK.value == "vispark" - assert "vispark" in litellm.provider_list - - def test_vispark_json_config_exists(self): - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - assert JSONProviderRegistry.exists("vispark") - - vispark = JSONProviderRegistry.get("vispark") - assert vispark is not None - assert vispark.base_url == "https://api.lab.vispark.in/v1" - assert vispark.api_key_env == "VISPARK_LAB_API_KEY" - assert vispark.api_base_env == "VISPARK_LAB_API_BASE" - assert vispark.param_mappings.get("max_completion_tokens") == "max_tokens" - - def test_vispark_in_openai_compatible_providers(self): - from litellm.constants import ( - openai_compatible_providers, - openai_text_completion_compatible_providers, - ) - - assert "vispark" in openai_compatible_providers - assert "vispark" in openai_text_completion_compatible_providers - - def test_vispark_provider_resolution(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="vispark/vision-medium", - custom_llm_provider=None, - api_base=None, - api_key=None, - ) - - assert model == "vision-medium" - assert provider == "vispark" - assert api_base == "https://api.lab.vispark.in/v1" - - def test_vispark_api_base_override(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="vispark/vision-medium", - custom_llm_provider=None, - api_base="https://custom.vispark.in/v1", - api_key="sk-test", - ) - - assert provider == "vispark" - assert api_base == "https://custom.vispark.in/v1" - assert api_key == "sk-test" - - def test_vispark_url_autodetection(self): - from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - - model, provider, api_key, api_base = get_llm_provider( - model="vision-medium", - custom_llm_provider=None, - api_base="https://api.lab.vispark.in/v1", - api_key=None, - ) - assert provider == "vispark" - assert api_base == "https://api.lab.vispark.in/v1" - - def test_vispark_max_completion_tokens_mapped(self): - from litellm.llms.openai_like.dynamic_config import create_config_class - from litellm.llms.openai_like.json_loader import JSONProviderRegistry - - provider = JSONProviderRegistry.get("vispark") - assert provider is not None - config = create_config_class(provider)() - - optional_params = config.map_openai_params( - non_default_params={"max_completion_tokens": 256}, - optional_params={}, - model="vision-medium", - drop_params=False, - ) - assert optional_params["max_tokens"] == 256 - assert "max_completion_tokens" not in optional_params - - def test_vispark_router_config(self): - from litellm import Router - - router = Router( - model_list=[ - { - "model_name": "vispark-chat", - "litellm_params": { - "model": "vispark/vision-medium", - "api_key": "test-key", - }, - } - ] - ) - - assert len(router.model_list) == 1 - assert router.model_list[0]["model_name"] == "vispark-chat" +_TEXT_COMPLETION: Final = { + "id": "cmpl_vispark", + "object": "text_completion", + "created": 1_790_000_000, + "model": "vision-small", + "choices": [{"index": 0, "text": "Hello from Vispark", "finish_reason": "stop", "logprobs": None}], + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, +} -class TestVisparkModelMetadata: - VISPARK_MODELS = ( - "vispark/vision-small", - "vispark/vision-medium", - "vispark/vision-large", +def test_vispark_provider_resolution(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + monkeypatch.delenv("VISPARK_LAB_API_BASE", raising=False) + + model, provider, api_key, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base=None, + api_key=None, ) - @staticmethod - def _load(path_parts): - import json - from pathlib import Path - - json_path = Path(__file__).parents[4].joinpath(*path_parts) - with open(json_path) as f: - return json.load(f) - - def test_vispark_models_registered_with_correct_metadata(self): - model_cost = self._load(("model_prices_and_context_window.json",)) - for model in self.VISPARK_MODELS: - info = model_cost.get(model) - assert info is not None, f"{model} missing from model_prices_and_context_window.json" - assert info["litellm_provider"] == "vispark" - assert info["mode"] == "chat" - assert info["input_cost_per_token"] > 0 - assert info["output_cost_per_token"] > 0 - assert info["supports_function_calling"] is True - assert info["supports_tool_choice"] is True - assert info["supports_reasoning"] is True - assert info["supports_response_schema"] is True - assert info["supports_vision"] is True - - assert info["max_output_tokens"] == 65536 - assert info["max_tokens"] == info["max_output_tokens"] - assert info["max_input_tokens"] == 1000000 - - def test_vispark_models_synced_to_backup(self): - model_cost = self._load(("model_prices_and_context_window.json",)) - backup = self._load(("litellm", "model_prices_and_context_window_backup.json")) - for model in self.VISPARK_MODELS: - assert model in backup, f"{model} missing from backup json" - assert backup[model] == model_cost[model], f"{model} differs between root and backup json" + assert model == "vision-medium" + assert provider == "vispark" + assert api_key == "vispark-env-key" + assert api_base == "https://api.lab.vispark.in/v1" -class TestVisparkDashboardRegistration: - @staticmethod - def _provider_create_fields(): - import json - from pathlib import Path +def test_vispark_provider_keeps_explicit_credentials(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - import litellm + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + monkeypatch.setenv("VISPARK_LAB_API_BASE", "https://vispark.env.example/v1") - path = Path(litellm.__file__).parent / "proxy" / "public_endpoints" / "provider_create_fields.json" - with open(path) as f: - return json.load(f) + _, provider, api_key, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base="https://vispark.internal.example/v1", + api_key="vispark-explicit-key", + ) - def test_vispark_is_selectable_in_the_add_model_form(self): - entries = [e for e in self._provider_create_fields() if e["litellm_provider"] == "vispark"] - assert len(entries) == 1, "vispark must appear exactly once in provider_create_fields.json" + assert provider == "vispark" + assert api_key == "vispark-explicit-key" + assert api_base == "https://vispark.internal.example/v1" - entry = entries[0] - assert entry["provider"] == "VISPARK" - assert entry["provider_display_name"] == "Vispark" - assert entry["default_model_placeholder"].startswith("vispark/") - fields = {f["key"]: f for f in entry["credential_fields"]} - assert fields["api_key"]["required"] is True - assert fields["api_key"]["field_type"] == "password" - assert fields["api_base"]["required"] is False +def test_vispark_api_base_env_overrides_default(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + monkeypatch.setenv("VISPARK_LAB_API_BASE", "https://vispark.env.example/v1") + + _, provider, _, api_base = get_llm_provider( + model="vispark/vision-medium", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + + assert provider == "vispark" + assert api_base == "https://vispark.env.example/v1" + + +def test_vispark_api_base_autodetects_provider(monkeypatch: pytest.MonkeyPatch): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setenv("VISPARK_LAB_API_KEY", "vispark-env-key") + + model, provider, api_key, api_base = get_llm_provider( + model="vision-medium", + custom_llm_provider=None, + api_base="https://api.lab.vispark.in/v1", + api_key=None, + ) + + assert model == "vision-medium" + assert provider == "vispark" + assert api_key == "vispark-env-key" + assert api_base == "https://api.lab.vispark.in/v1" + + +def test_vispark_chat_completion_request_maps_max_completion_tokens(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + response: Final = litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Say hello"}], + max_completion_tokens=64, + api_key="vispark-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert request.headers["authorization"] == "Bearer vispark-test-key" + assert body["model"] == "vision-medium" + assert body["max_tokens"] == 64 + assert "max_completion_tokens" not in body + assert response.choices[0].message.content == "Hello from Vispark" + + +def test_vispark_text_completion_request(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/completions").respond(200, json=_TEXT_COMPLETION) + response: Final = litellm.text_completion( + model="vispark/vision-small", + prompt="Say hello", + api_key="vispark-test-key", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert route.call_count == 1 + assert body["model"] == "vision-small" + assert body["prompt"] == "Say hello" + assert response.choices[0].text == "Hello from Vispark" + + +def test_vispark_cost_follows_the_cost_map(): + cost_entry: Final = litellm.model_cost["vispark/vision-large"] + + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-large", + custom_llm_provider="vispark", + prompt_tokens=1_000, + completion_tokens=500, + ) + + assert prompt_cost == pytest.approx(1_000 * cost_entry["input_cost_per_token"]) + assert completion_cost == pytest.approx(500 * cost_entry["output_cost_per_token"]) + + +def test_vispark_models_are_listed_for_the_provider(): + from litellm.utils import get_valid_models + + expected: Final = {name for name, entry in litellm.model_cost.items() if entry["litellm_provider"] == "vispark"} + + assert expected + assert set(litellm.models_by_provider["vispark"]) == expected + assert expected <= set(get_valid_models(custom_llm_provider="vispark")) + + +def test_vispark_endpoint_matrix_matches_between_root_and_backup(): + backup_path: Final = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" + root_path: Final = Path(litellm.__file__).parent.parent / "provider_endpoints_support.json" + + backup_entry: Final = json.loads(backup_path.read_text())["providers"]["vispark"] + root_entry: Final = json.loads(root_path.read_text())["providers"]["vispark"] + + assert backup_entry == root_entry + assert root_entry["endpoints"]["chat_completions"] is True From 0d8f8bba1db46b15e7df8786b920a2d964a47d0e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 18:13:35 +0000 Subject: [PATCH 06/18] test(vispark): tolerate cost-map entries without a provider when listing models --- tests/unit/llms/openai_like/test_vispark_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 5b93ae099ce..e1e2f79cbd4 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -159,7 +159,7 @@ def test_vispark_cost_follows_the_cost_map(): def test_vispark_models_are_listed_for_the_provider(): from litellm.utils import get_valid_models - expected: Final = {name for name, entry in litellm.model_cost.items() if entry["litellm_provider"] == "vispark"} + expected: Final = {name for name, entry in litellm.model_cost.items() if entry.get("litellm_provider") == "vispark"} assert expected assert set(litellm.models_by_provider["vispark"]) == expected From 0e8fb9b8726cd9e1c82c75804c958c1b668b3615 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:19:53 -0700 Subject: [PATCH 07/18] chore(vispark): regenerate the provider endpoint matrix with main's generator --- .../provider_endpoints_support_backup.json | 20 +++++++++++++------ provider_endpoints_support.json | 20 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 7a1c4d64279..d205f5f7322 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -4181,17 +4181,25 @@ "url": "https://lab.vispark.in/#vision", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "vercel_ai_gateway": { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 7a1c4d64279..d205f5f7322 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -4181,17 +4181,25 @@ "url": "https://lab.vispark.in/#vision", "endpoints": { "chat_completions": true, - "messages": false, - "responses": false, + "responses": true, + "text_completion": true, "embeddings": false, "image_generations": false, + "image_edits": false, "audio_transcriptions": false, "audio_speech": false, - "moderations": false, - "batches": false, "rerank": false, - "a2a": false, - "interactions": false + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true } }, "vercel_ai_gateway": { From 9ad9e425e9d91d6e4cbf669aa5579b7839eadc41 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:45:22 -0700 Subject: [PATCH 08/18] fix(vispark): bill the 600 input token minimum and bridge JSON schemas and PDF file parts Vispark charges every request for at least 600 input tokens, so the cost map gains min_billed_input_tokens and the generic cost path bills up to it. Vispark answers 500 to a json_schema that sets additionalProperties, which the OpenAI SDK always sends for a Pydantic model, so the provider strips it. Vispark reads PDFs only as image_url parts, so file parts are sent that way. --- .../litellm_core_utils/llm_cost_calc/utils.py | 22 ++++-- litellm/llms/openai_like/dynamic_config.py | 50 ++++++++++--- litellm/llms/openai_like/json_loader.py | 3 +- litellm/llms/openai_like/providers.json | 4 + ...odel_prices_and_context_window_backup.json | 3 + litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 3 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 32 ++++++++ .../llms/openai_like/test_vispark_provider.py | 73 ++++++++++++++++++- tests/unit/test_utils.py | 1 + 11 files changed, 171 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3b49f239afe..7df17577779 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1560,14 +1560,20 @@ def generic_cost_per_token( base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost ) - prompt_cost = _calculate_input_cost( - prompt_tokens_details=prompt_tokens_details, - model_info=resolved_model_info, - prompt_base_cost=base_costs.input_rate, - cache_read_cost=effective_cache_read_cost, - cache_creation_cost=base_costs.cache_creation_rate, - cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - service_tier=service_tier, + unbilled_minimum_input_tokens: Final = max( + (resolved_model_info.get("min_billed_input_tokens") or 0) - usage.prompt_tokens, 0 + ) + prompt_cost = ( + _calculate_input_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=resolved_model_info, + prompt_base_cost=base_costs.input_rate, + cache_read_cost=effective_cache_read_cost, + cache_creation_cost=base_costs.cache_creation_rate, + cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, + service_tier=service_tier, + ) + + unbilled_minimum_input_tokens * base_costs.input_rate ) ## CALCULATE OUTPUT COST diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 685561f55a5..1b2d54dde35 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -2,25 +2,42 @@ Dynamic configuration class generator for JSON-based providers. """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final, Literal, overload +from pydantic import JsonValue, TypeAdapter + from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, + migrate_file_to_image_url, ) from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage +from litellm.utils import remove_additional_properties from .json_loader import SimpleProviderConfig +_JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_REQUESTED_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + if TYPE_CHECKING: from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig +def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "user" or not isinstance(message["content"], list): + return message + content: Final = [ + migrate_file_to_image_url(part) if part["type"] == "file" else part for part in message["content"] + ] + updated: Final[ChatCompletionUserMessage] = {**message, "content": content} + return updated + + def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" @@ -46,14 +63,19 @@ def create_config_class(provider: SimpleProviderConfig): ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: """Transform messages based on special_handling config""" - # Handle content list to string conversion if configured - if provider.special_handling.get("convert_content_list_to_string"): - messages = handle_messages_with_content_list_to_str_conversion(messages) - + content_flattened: Final = ( + handle_messages_with_content_list_to_str_conversion(messages) + if provider.special_handling.get("convert_content_list_to_string") + else messages + ) + transformed: Final = ( + [_with_file_parts_as_image_url(message) for message in content_flattened] + if provider.special_handling.get("convert_file_parts_to_image_url") + else content_flattened + ) if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) - else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed, model=model, is_async=True) + return super()._transform_messages(messages=transformed, model=model, is_async=False) def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None @@ -140,12 +162,18 @@ def create_config_class(provider: SimpleProviderConfig): supported_params: Final = self.get_supported_openai_params(model) # Apply supported params - for param, value in non_default_params.items(): + requested_params: Final = _REQUESTED_PARAMS_ADAPTER.validate_python(non_default_params) + for param, value in requested_params.items(): # Check parameter mappings first if param in provider.param_mappings: optional_params[provider.param_mappings[param]] = value elif param in supported_params: - optional_params[param] = value + optional_params[param] = ( + remove_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) + if param == "response_format" + and provider.special_handling.get("remove_additional_properties_from_response_format") + else value + ) # Apply temperature constraints if present if "temperature" in optional_params: diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 23b63177137..d3474c931a9 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -12,6 +12,7 @@ from litellm._logging import verbose_logger from litellm.types.utils import ProviderSpecificModelInfo _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo) +_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(dict[str, bool]) class SimpleProviderConfig: @@ -25,7 +26,7 @@ class SimpleProviderConfig: self.base_class = data.get("base_class", "openai_gpt") self.param_mappings = data.get("param_mappings", {}) self.constraints = data.get("constraints", {}) - self.special_handling = data.get("special_handling", {}) + self.special_handling: Final = _SPECIAL_HANDLING_ADAPTER.validate_python(data.get("special_handling", {})) self.supported_endpoints = data.get("supported_endpoints", []) self.provider_info: ProviderSpecificModelInfo = ( _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {} diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index cab5a0cb5a4..ebef090b7f3 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -265,6 +265,10 @@ "param_mappings": { "max_completion_tokens": "max_tokens" }, + "special_handling": { + "convert_file_parts_to_image_url": true, + "remove_additional_properties_from_response_format": true + }, "supported_endpoints": ["/v1/chat/completions", "/v1/completions"] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43ee1088e25..11eca48f3be 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48599,6 +48599,7 @@ "litellm_provider": "vispark", "max_input_tokens": 1000000, "max_output_tokens": 65536, + "min_billed_input_tokens": 600, "mode": "chat", "output_cost_per_token": 3.16e-06, "source": "https://api.lab.vispark.in/v1/models", @@ -48630,6 +48631,7 @@ "litellm_provider": "vispark", "max_input_tokens": 1000000, "max_output_tokens": 65536, + "min_billed_input_tokens": 600, "mode": "chat", "output_cost_per_token": 1.263e-05, "source": "https://api.lab.vispark.in/v1/models", @@ -48661,6 +48663,7 @@ "litellm_provider": "vispark", "max_input_tokens": 1000000, "max_output_tokens": 65536, + "min_billed_input_tokens": 600, "mode": "chat", "output_cost_per_token": 2.211e-05, "source": "https://api.lab.vispark.in/v1/models", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e47baf2e7ff..cac11598628 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -365,6 +365,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None + min_billed_input_tokens: ReadOnly[int | None] off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates input_cost_per_character: float | None # only for vertex ai models input_cost_per_audio_token: float | None diff --git a/litellm/utils.py b/litellm/utils.py index 1e095966a43..ec4dbf19309 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6779,6 +6779,7 @@ def get_model_info_helper( ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), + min_billed_input_tokens=_model_info.get("min_billed_input_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None ), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43ee1088e25..11eca48f3be 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48599,6 +48599,7 @@ "litellm_provider": "vispark", "max_input_tokens": 1000000, "max_output_tokens": 65536, + "min_billed_input_tokens": 600, "mode": "chat", "output_cost_per_token": 3.16e-06, "source": "https://api.lab.vispark.in/v1/models", @@ -48630,6 +48631,7 @@ "litellm_provider": "vispark", "max_input_tokens": 1000000, "max_output_tokens": 65536, + "min_billed_input_tokens": 600, "mode": "chat", "output_cost_per_token": 1.263e-05, "source": "https://api.lab.vispark.in/v1/models", @@ -48661,6 +48663,7 @@ "litellm_provider": "vispark", "max_input_tokens": 1000000, "max_output_tokens": 65536, + "min_billed_input_tokens": 600, "mode": "chat", "output_cost_per_token": 2.211e-05, "source": "https://api.lab.vispark.in/v1/models", diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b527250c4..2425e8077f8 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -99,6 +99,38 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re assert completion_cost == pytest.approx(380 * 9.7e-7) +@pytest.mark.parametrize( + ("prompt_tokens", "billed_prompt_tokens"), + [(10, 600), (600, 600), (900, 900)], + ids=["below_the_minimum_bills_the_minimum", "at_the_minimum", "above_the_minimum_bills_as_reported"], +) +def test_generic_cost_per_token_bills_at_least_the_minimum_input_tokens( + prompt_tokens: int, billed_prompt_tokens: int +) -> None: + model_info: Final[ModelInfo] = { + "key": "minimum-input-fixture", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 2e-6, + "output_cost_per_token": 5e-6, + "min_billed_input_tokens": 600, + "litellm_provider": "openai", + "mode": "chat", + "supported_openai_params": None, + } + + prompt_cost, completion_cost = generic_cost_per_token( + model="minimum-input-fixture", + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7), + custom_llm_provider="openai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(billed_prompt_tokens * 2e-6) + assert completion_cost == pytest.approx(7 * 5e-6) + + @pytest.mark.parametrize( ("model_info", "expected_prompt_cost"), [ diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index e1e2f79cbd4..3b574e46950 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -126,6 +126,59 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens(): assert response.choices[0].message.content == "Hello from Vispark" +def test_vispark_strips_additional_properties_from_a_json_schema_response_format(): + schema: Final = { + "type": "object", + "properties": {"city": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}}, + "required": ["city"], + "additionalProperties": False, + } + response_format: Final = {"type": "json_schema", "json_schema": {"name": "City", "schema": schema, "strict": True}} + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Capital of France as JSON"}], + response_format=response_format, + api_key="vispark-test-key", + ) + + sent_format: Final = json.loads(route.calls.last.request.content)["response_format"] + assert "additionalProperties" not in json.dumps(sent_format) + assert sent_format["json_schema"]["strict"] is True + assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}} + assert schema["additionalProperties"] is False + + +def test_vispark_sends_a_pdf_file_part_as_an_image_url(): + pdf_data_url: Final = "data:application/pdf;base64,JVBERi0xLjQK" + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What code is in this PDF?"}, + {"type": "file", "file": {"file_data": pdf_data_url}}, + ], + } + ], + api_key="vispark-test-key", + ) + + sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"] + assert sent_content[0] == {"type": "text", "text": "What code is in this PDF?"} + assert sent_content[1]["type"] == "image_url" + assert sent_content[1]["image_url"]["url"] == pdf_data_url + assert all(part["type"] != "file" for part in sent_content) + + def test_vispark_text_completion_request(): with respx.mock() as upstream: route: Final = upstream.post("https://api.lab.vispark.in/v1/completions").respond(200, json=_TEXT_COMPLETION) @@ -144,18 +197,34 @@ def test_vispark_text_completion_request(): def test_vispark_cost_follows_the_cost_map(): cost_entry: Final = litellm.model_cost["vispark/vision-large"] + prompt_tokens: Final = cost_entry["min_billed_input_tokens"] + 1_000 prompt_cost, completion_cost = litellm.cost_per_token( model="vision-large", custom_llm_provider="vispark", - prompt_tokens=1_000, + prompt_tokens=prompt_tokens, completion_tokens=500, ) - assert prompt_cost == pytest.approx(1_000 * cost_entry["input_cost_per_token"]) + assert prompt_cost == pytest.approx(prompt_tokens * cost_entry["input_cost_per_token"]) assert completion_cost == pytest.approx(500 * cost_entry["output_cost_per_token"]) +@pytest.mark.parametrize("model", ["vision-small", "vision-medium", "vision-large"]) +def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str): + cost_entry: Final = litellm.model_cost[f"vispark/{model}"] + minimum: Final = cost_entry["min_billed_input_tokens"] + + prompt_cost, _ = litellm.cost_per_token( + model=model, + custom_llm_provider="vispark", + prompt_tokens=minimum // 2, + completion_tokens=1, + ) + + assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"]) + + def test_vispark_models_are_listed_for_the_provider(): from litellm.utils import get_valid_models diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38287b25169..b64efb4a635 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1160,6 +1160,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "prompt_cache_min_tokens": {"type": "number"}, + "min_billed_input_tokens": {"type": "number"}, "supports_prompt_cache_breakpoint": {"type": "boolean"}, "supports_thinking_cache_preservation": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, From c92f96e56683329960ccb861d55f32ec94bcc91c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:36:29 -0700 Subject: [PATCH 09/18] fix(vispark): bill the input token minimum under custom pricing and strip every additionalProperties value --- litellm/cost_calculator.py | 13 +++- litellm/llms/openai_like/dynamic_config.py | 13 +++- ...odel_prices_and_context_window_backup.json | 3 + litellm/router.py | 17 +++- model_prices_and_context_window.json | 3 + .../llms/openai_like/test_vispark_provider.py | 77 ++++++++++++++++++- 6 files changed, 119 insertions(+), 7 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 86b22069a39..055af080a9f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -522,8 +522,13 @@ def cost_per_token( if _is_anthropic_style: _normalized_prompt_tokens += _cache_read_tokens + _cache_creation_tokens + billed_prompt_tokens: Final = ( + max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider)) + if custom_cost_per_token is not None + else _normalized_prompt_tokens + ) response_cost: Final = _cost_per_token_custom_pricing_helper( - prompt_tokens=_normalized_prompt_tokens, + prompt_tokens=billed_prompt_tokens, completion_tokens=completion_tokens, response_time_ms=response_time_ms, cached_tokens=_cache_read_tokens, @@ -2436,6 +2441,12 @@ def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: return raw_entry if isinstance(raw_entry, Mapping) else None +def _min_billed_input_tokens(model: str, custom_llm_provider: str | None) -> int: + cost_entry: Final = _raw_cost_map_entry(model) or _raw_cost_map_entry(f"{custom_llm_provider}/{model}") or {} + minimum: Final = cost_entry.get("min_billed_input_tokens") + return minimum if isinstance(minimum, int) else 0 + + def pricing_entry_for_cost_calc( model: str | None, completion_response: object | None, diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 1b2d54dde35..7c283a9d7d4 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -17,7 +17,6 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage -from litellm.utils import remove_additional_properties from .json_loader import SimpleProviderConfig @@ -38,6 +37,16 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues return updated +def _without_additional_properties(value: JsonValue) -> JsonValue: + if isinstance(value, dict): + return { + key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties" + } + if isinstance(value, list): + return [_without_additional_properties(item) for item in value] + return value + + def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" @@ -169,7 +178,7 @@ def create_config_class(provider: SimpleProviderConfig): optional_params[provider.param_mappings[param]] = value elif param in supported_params: optional_params[param] = ( - remove_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) + _without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) if param == "response_format" and provider.special_handling.get("remove_additional_properties_from_response_format") else value diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 11eca48f3be..509ba98e5c6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48595,6 +48595,7 @@ "supports_vision": true }, "vispark/vision-small": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 1.05e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48627,6 +48628,7 @@ "supports_vision": true }, "vispark/vision-medium": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 4.21e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48659,6 +48661,7 @@ "supports_vision": true }, "vispark/vision-large": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 7.37e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..1a573816579 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11569,7 +11569,22 @@ class Router: backend_model=backend_model, custom_llm_provider=pricing_provider, ) - return model_info + billing_minimum: Final[Mapping[str, int]] = ( + Router._builtin_billing_minimum(backend_model=backend_model, custom_llm_provider=pricing_provider) + if model_info.get("input_cost_per_token") is not None + else MappingProxyType({}) + ) + return {**billing_minimum, **model_info} + + @staticmethod + def _builtin_billing_minimum(backend_model: str, custom_llm_provider: str | None) -> Mapping[str, int]: + backend_info: Final = lookup_model_info_for_registration( + model=backend_model, custom_llm_provider=custom_llm_provider + ) + minimum: Final = ( + None if isinstance(backend_info, ModelInfoLookupMiss) else backend_info.get("min_billed_input_tokens") + ) + return MappingProxyType({} if minimum is None else {"min_billed_input_tokens": minimum}) @staticmethod def _register_deployment_pricing(deployment: Deployment) -> None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 11eca48f3be..509ba98e5c6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48595,6 +48595,7 @@ "supports_vision": true }, "vispark/vision-small": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 1.05e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48627,6 +48628,7 @@ "supports_vision": true }, "vispark/vision-medium": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 4.21e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, @@ -48659,6 +48661,7 @@ "supports_vision": true }, "vispark/vision-large": { + "cache_read_input_token_cost": 0.0, "input_cost_per_token": 7.37e-06, "litellm_provider": "vispark", "max_input_tokens": 1000000, diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 3b574e46950..5d2a4764f25 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -126,10 +126,18 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens(): assert response.choices[0].message.content == "Hello from Vispark" -def test_vispark_strips_additional_properties_from_a_json_schema_response_format(): +@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}]) +def test_vispark_strips_additional_properties_from_a_json_schema_response_format( + additional_properties: bool | dict[str, str], +): + city_schema: Final = { + "type": "object", + "properties": {"name": {"type": "string"}}, + "additionalProperties": additional_properties, + } schema: Final = { "type": "object", - "properties": {"city": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}}, + "properties": {"city": city_schema}, "required": ["city"], "additionalProperties": False, } @@ -149,7 +157,30 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format assert "additionalProperties" not in json.dumps(sent_format) assert sent_format["json_schema"]["strict"] is True assert sent_format["json_schema"]["schema"]["properties"]["city"]["properties"] == {"name": {"type": "string"}} - assert schema["additionalProperties"] is False + assert city_schema["additionalProperties"] == additional_properties + + +def test_other_json_providers_keep_additional_properties_and_file_parts(): + file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}} + response_format: Final = { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}, "additionalProperties": False}, + }, + } + with respx.mock() as upstream: + route: Final = upstream.post("https://api.scaleway.ai/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) + litellm.completion( + model="scaleway/llama-3.3-70b-instruct", + messages=[{"role": "user", "content": [{"type": "text", "text": "Read this"}, file_part]}], + response_format=response_format, + api_key="scaleway-test-key", + ) + + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == response_format + assert body["messages"][0]["content"][1] == file_part def test_vispark_sends_a_pdf_file_part_as_an_image_url(): @@ -225,6 +256,46 @@ def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str): assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"]) +def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens(): + minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vispark-custom-priced", + "litellm_params": { + "model": "vispark/vision-small", + "api_key": "vispark-test-key", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 5e-6, + }, + } + ] + ) + with respx.mock() as upstream: + upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond(200, json=_CHAT_COMPLETION) + response: Final = router.completion( + model="vispark-custom-priced", messages=[{"role": "user", "content": "Say hello"}] + ) + + completion_tokens: Final = _CHAT_COMPLETION["usage"]["completion_tokens"] + assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6) + + +def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens(): + minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] + + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-small", + custom_llm_provider="vispark", + prompt_tokens=minimum // 2, + completion_tokens=3, + custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + ) + + assert prompt_cost == pytest.approx(minimum * 2e-6) + assert completion_cost == pytest.approx(3 * 5e-6) + + def test_vispark_models_are_listed_for_the_provider(): from litellm.utils import get_valid_models From b6be3afe70eda4f802f1639a44bd9e1adaa88d62 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:32:56 -0700 Subject: [PATCH 10/18] refactor(router): inline the billing minimum lookup into the deployment cost payload --- litellm/router.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 1a573816579..c637a739d4b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11569,22 +11569,17 @@ class Router: backend_model=backend_model, custom_llm_provider=pricing_provider, ) - billing_minimum: Final[Mapping[str, int]] = ( - Router._builtin_billing_minimum(backend_model=backend_model, custom_llm_provider=pricing_provider) + backend_info: Final = ( + lookup_model_info_for_registration(model=backend_model, custom_llm_provider=pricing_provider) if model_info.get("input_cost_per_token") is not None - else MappingProxyType({}) + else None ) - return {**billing_minimum, **model_info} - - @staticmethod - def _builtin_billing_minimum(backend_model: str, custom_llm_provider: str | None) -> Mapping[str, int]: - backend_info: Final = lookup_model_info_for_registration( - model=backend_model, custom_llm_provider=custom_llm_provider + billing_minimum: Final = ( + None + if backend_info is None or isinstance(backend_info, ModelInfoLookupMiss) + else backend_info.get("min_billed_input_tokens") ) - minimum: Final = ( - None if isinstance(backend_info, ModelInfoLookupMiss) else backend_info.get("min_billed_input_tokens") - ) - return MappingProxyType({} if minimum is None else {"min_billed_input_tokens": minimum}) + return model_info if billing_minimum is None else {"min_billed_input_tokens": billing_minimum, **model_info} @staticmethod def _register_deployment_pricing(deployment: Deployment) -> None: From 2105d7ac6d89f918bc9c08ece352d533c06daf13 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:42:43 -0700 Subject: [PATCH 11/18] fix(vispark): scope the input minimum per deployment and reject map-field schemas Keep a deployment's min_billed_input_tokens on its own cost-map entry so it no longer changes billing for other deployments of the same backend model. Strip boolean additionalProperties for Vispark, answer 400 for schema-valued ones unless drop_params is set, and do it without recursion. --- .../crates/model-catalog/src/model_info.rs | 3 + litellm/llms/openai_like/dynamic_config.py | 49 ++++++++--- litellm/router.py | 15 ++-- litellm/types/utils.py | 2 +- .../llms/openai_like/test_vispark_provider.py | 86 ++++++++++++++++++- .../unit/test_router_model_cost_isolation.py | 26 +++--- 6 files changed, 146 insertions(+), 35 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index ebee2c64140..3597795ba35 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -294,6 +294,9 @@ pub struct ModelInfo { /// Free-form notes about the entry (e.g. pricing derivation). #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option>, + /// Input tokens billed per request even when the prompt is shorter. + #[serde(skip_serializing_if = "Option::is_none")] + pub min_billed_input_tokens: Option, /// Primary API surface / task type of the model. #[serde(skip_serializing_if = "Option::is_none")] pub mode: Option, diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 7c283a9d7d4..04b93a94df6 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -2,12 +2,15 @@ Dynamic configuration class generator for JSON-based providers. """ -from collections.abc import Coroutine, Mapping +import json +from collections.abc import Callable, Coroutine, Mapping from typing import TYPE_CHECKING, Final, Literal, overload from pydantic import JsonValue, TypeAdapter +import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, migrate_file_to_image_url, @@ -37,14 +40,35 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues return updated -def _without_additional_properties(value: JsonValue) -> JsonValue: - if isinstance(value, dict): - return { - key: _without_additional_properties(item) for key, item in value.items() if key != "additionalProperties" - } - if isinstance(value, list): - return [_without_additional_properties(item) for item in value] - return value +def _without_additional_properties_flags(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + return {key: item for key, item in node.items() if key != "additionalProperties" or not isinstance(item, bool)} + + +def _without_additional_properties_key(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + return {key: item for key, item in node.items() if key != "additionalProperties"} + + +def _rebuilt_json( + value: JsonValue, object_hook: Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]] +) -> JsonValue: + return _JSON_VALUE_ADAPTER.validate_python(json.loads(json.dumps(value), object_hook=object_hook)) + + +def _response_format_without_additional_properties( + response_format: JsonValue, provider_slug: str, model: str, drop_params: bool +) -> JsonValue: + without_flags: Final = _rebuilt_json(response_format, _without_additional_properties_flags) + without_any: Final = _rebuilt_json(response_format, _without_additional_properties_key) + if without_flags == without_any or drop_params: + return without_any + raise UnsupportedParamsError( + message=( + f"{provider_slug} does not support schema-valued additionalProperties (map fields) in response_format. " + "To drop them, set `drop_params=True`" + ), + llm_provider=provider_slug, + model=model, + ) def create_config_class(provider: SimpleProviderConfig): @@ -178,7 +202,12 @@ def create_config_class(provider: SimpleProviderConfig): optional_params[provider.param_mappings[param]] = value elif param in supported_params: optional_params[param] = ( - _without_additional_properties(_JSON_VALUE_ADAPTER.validate_python(value)) + _response_format_without_additional_properties( + _JSON_VALUE_ADAPTER.validate_python(value), + provider_slug=provider.slug, + model=model, + drop_params=drop_params or litellm.drop_params is True, + ) if param == "response_format" and provider.special_handling.get("remove_additional_properties_from_response_format") else value diff --git a/litellm/router.py b/litellm/router.py index c637a739d4b..6f2b572c77c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10331,19 +10331,20 @@ class Router: model_info[field] = copy.deepcopy(backend_value) @staticmethod - def _inherit_builtin_base_rates_for_off_peak( + def _inherit_builtin_base_rates_for_scoped_pricing( model_info: dict, # mutable-ok: cost-map entry filled in place backend_model: str, custom_llm_provider: str | None, ) -> None: """Fill missing pricing fields on a deployment entry that only sets - ``off_peak_pricing``, from the backend model's built-in cost map entry. + ``off_peak_pricing`` or ``min_billed_input_tokens``, from the backend + model's built-in cost map entry. Cost lookup selects the deployment-scoped entry over the shared backend entry only when the deployment entry carries a base pricing field, and - ``off_peak_pricing`` is deliberately kept off the shared entry, so a - deployment spelling out only its off-peak schedule would otherwise - never receive the discount. The backend model's entire canonical cost + both fields are deliberately kept off the shared entry, so a deployment + spelling out only its off-peak schedule or input minimum would otherwise + never have it applied. The backend model's entire canonical cost map entry is copied, field by field, so threshold, tiered, service-tier, cache, character, and per-second rates as well as companion billing fields like ``web_search_billing_unit`` and the @@ -10357,7 +10358,7 @@ class Router: no-op when any base pricing field is already set or the backend model has no canonical entry. """ - if not model_info.get("off_peak_pricing"): + if not model_info.get("off_peak_pricing") and model_info.get("min_billed_input_tokens") is None: return if any( model_info.get(field) is not None @@ -11548,7 +11549,7 @@ class Router: model_info.update( {key: value for key, value in priced_info.items() if is_server_derived_pricing_key(key)} ) - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=pricing_provider, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cac11598628..025424d8810 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4130,7 +4130,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): return {k: v for k, v in model_info.items() if k not in cls.model_fields} -DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) +DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing", "min_billed_input_tokens"}) DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( {"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"} diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 5d2a4764f25..45636725b83 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -126,10 +126,8 @@ def test_vispark_chat_completion_request_maps_max_completion_tokens(): assert response.choices[0].message.content == "Hello from Vispark" -@pytest.mark.parametrize("additional_properties", [False, True, {"type": "string"}]) -def test_vispark_strips_additional_properties_from_a_json_schema_response_format( - additional_properties: bool | dict[str, str], -): +@pytest.mark.parametrize("additional_properties", [False, True]) +def test_vispark_strips_additional_properties_from_a_json_schema_response_format(additional_properties: bool): city_schema: Final = { "type": "object", "properties": {"name": {"type": "string"}}, @@ -160,6 +158,50 @@ def test_vispark_strips_additional_properties_from_a_json_schema_response_format assert city_schema["additionalProperties"] == additional_properties +def _map_field_response_format() -> dict[str, object]: + schema: Final = { + "type": "object", + "properties": {"facts": {"type": "object", "additionalProperties": {"type": "string"}}}, + "required": ["facts"], + "additionalProperties": False, + } + return {"type": "json_schema", "json_schema": {"name": "Facts", "schema": schema}} + + +def test_vispark_rejects_a_map_field_response_format_without_drop_params(): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + with pytest.raises(litellm.UnsupportedParamsError) as error: + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Facts about Paris as JSON"}], + response_format=_map_field_response_format(), + api_key="vispark-test-key", + ) + + assert error.value.status_code == 400 + assert route.call_count == 0 + + +def test_vispark_drops_a_map_field_schema_with_drop_params(): + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Facts about Paris as JSON"}], + response_format=_map_field_response_format(), + drop_params=True, + api_key="vispark-test-key", + ) + + sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"] + assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]} + + def test_other_json_providers_keep_additional_properties_and_file_parts(): file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}} response_format: Final = { @@ -281,6 +323,42 @@ def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_i assert response._hidden_params["response_cost"] == pytest.approx(minimum * 2e-6 + completion_tokens * 5e-6) +def test_vispark_deployment_minimum_override_stays_on_its_own_deployment(): + builtin: Final = litellm.model_cost["vispark/vision-small"] + minimum: Final = builtin["min_billed_input_tokens"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vispark-no-minimum", + "litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"}, + "model_info": {"min_billed_input_tokens": 0}, + }, + { + "model_name": "vispark-default", + "litellm_params": {"model": "vispark/vision-small", "api_key": "vispark-test-key"}, + }, + ] + ) + with respx.mock() as upstream: + upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json={**_CHAT_COMPLETION, "model": "vision-small"} + ) + no_minimum: Final = router.completion( + model="vispark-no-minimum", messages=[{"role": "user", "content": "Say hello"}] + ) + default: Final = router.completion(model="vispark-default", messages=[{"role": "user", "content": "Say hello"}]) + + usage: Final = _CHAT_COMPLETION["usage"] + completion_cost: Final = usage["completion_tokens"] * builtin["output_cost_per_token"] + assert no_minimum._hidden_params["response_cost"] == pytest.approx( + usage["prompt_tokens"] * builtin["input_cost_per_token"] + completion_cost + ) + assert default._hidden_params["response_cost"] == pytest.approx( + minimum * builtin["input_cost_per_token"] + completion_cost + ) + assert litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] == minimum + + def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_tokens(): minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..16ed772d3df 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -1637,7 +1637,7 @@ def test_cost_map_backend_model_uses_canonical_model_precedence( assert Router._cost_map_backend_model(deployment) == expected -def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): +def test_inherit_builtin_base_rates_for_scoped_pricing_fills_missing_rates(): """Direct unit test of the helper: an entry carrying only an off_peak_pricing block inherits the backend model's built-in base token rates, so cost lookup via the deployment id can bill standard rates @@ -1652,7 +1652,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): } model_info = {"off_peak_pricing": off_peak_block} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="openai", @@ -1663,7 +1663,7 @@ def test_inherit_builtin_base_rates_for_off_peak_fills_missing_rates(): assert model_info["off_peak_pricing"] == off_peak_block -def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): +def test_inherit_builtin_base_rates_for_scoped_pricing_carries_threshold_rates(): """A backend with above-threshold pricing hands the whole rate structure to the deployment entry, so peak-hour billing of large prompts through that entry matches the shared backend entry instead of flattening to the base @@ -1677,7 +1677,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="gemini", @@ -1692,7 +1692,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_threshold_rates(): ) -def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_fields(): +def test_inherit_builtin_base_rates_for_scoped_pricing_carries_companion_billing_fields(): """Billing rules that are not literal cost rates, like the web search billing unit, must ride along, or grounding and regional uplifts would bill differently through the deployment entry than through the shared @@ -1706,7 +1706,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider=None, @@ -1716,7 +1716,7 @@ def test_inherit_builtin_base_rates_for_off_peak_carries_companion_billing_field assert model_info["input_cost_per_token"] == raw_entry["input_cost_per_token"] -def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_zero(): +def test_inherit_builtin_base_rates_for_scoped_pricing_tiered_only_backend_stores_no_zero(): """A tiered-only backend has no flat token rates; get_model_info synthesizes zeros for them, and storing those would mark the deployment explicitly priced free. The tier table itself must carry over as an isolated copy so @@ -1729,7 +1729,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model=backend_model, custom_llm_provider="dashscope", @@ -1746,7 +1746,7 @@ def test_inherit_builtin_base_rates_for_off_peak_tiered_only_backend_stores_no_z assert raw_tiers[0] == original_first_tier -def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): +def test_inherit_builtin_base_rates_for_scoped_pricing_leaves_explicit_rates_alone(): """An entry that sets its own base rate beside the block already counts as a full custom pricing entry; the helper must not mix builtin rates into it. """ @@ -1755,7 +1755,7 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): "input_cost_per_token": 3e-06, } - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=model_info, backend_model="gpt-4o-mini", custom_llm_provider="openai", @@ -1765,12 +1765,12 @@ def test_inherit_builtin_base_rates_for_off_peak_leaves_explicit_rates_alone(): assert "output_cost_per_token" not in model_info -def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend(): +def test_inherit_builtin_base_rates_for_scoped_pricing_noop_without_block_or_backend(): """Nothing happens without an off_peak_pricing block, and an unmapped backend model leaves the entry unchanged rather than raising. """ plain_info = {"id": "dep-1"} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=plain_info, backend_model="gpt-4o-mini", custom_llm_provider="openai", @@ -1778,7 +1778,7 @@ def test_inherit_builtin_base_rates_for_off_peak_noop_without_block_or_backend() assert plain_info == {"id": "dep-1"} off_peak_info = {"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 5e-07}} - Router._inherit_builtin_base_rates_for_off_peak( + Router._inherit_builtin_base_rates_for_scoped_pricing( model_info=off_peak_info, backend_model="this-backend-model-does-not-exist-x9y8z7", custom_llm_provider=None, From 797126e72eb1ce5fb39f76e5b2e3e8dd6c6ebe5a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:58:43 -0700 Subject: [PATCH 12/18] fix(budget): reserve at least a deployment's billed input minimum --- .../spend_tracking/budget_reservation.py | 62 ++++++++++--------- .../llms/openai_like/test_vispark_provider.py | 12 ---- .../spend_tracking/test_budget_reservation.py | 32 ++++++++++ 3 files changed, 65 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0b91e57f01e..59097e88f4f 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -1530,49 +1530,57 @@ def _get_model_cost_infos( deployment can never leave the request under-reserved. """ try: - base: Final = _get_model_cost_info(model=model, llm_router=llm_router) - if base is None: + group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router) + if group_info is None: return [] - tiered_tables: Final = _get_deployment_tiered_pricing_tables(model=model, llm_router=llm_router) + deployment_infos: Final = _get_deployment_model_infos(model=model, llm_router=llm_router) except Exception: verbose_proxy_logger.debug( "Unable to load model cost info for budget reservation", exc_info=True, ) return [] - if not tiered_tables: - return [base] + billing_minimums: Final = [ + minimum + for deployment_info in deployment_infos + if (minimum := _to_int(deployment_info.get("min_billed_input_tokens"))) is not None + ] + base: Final = {**group_info, "min_billed_input_tokens": max(billing_minimums)} if billing_minimums else group_info + tiered_tables: Final = [ + table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None + ] return [base, *({**base, "tiered_pricing": table} for table in tiered_tables)] -def _deployment_tiered_pricing_table( - deployment: DeploymentTypedDict, - llm_router: Router, -) -> Sequence[Mapping[str, object]] | None: - model_id: Final = _get_value(_get_value(deployment, "model_info"), "id") - backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model") - if not isinstance(model_id, str) or not isinstance(backend_model, str): - return None - deployment_model_info: Final = llm_router.cached_deployment_model_info(model_id, backend_model) - if deployment_model_info is None: - return None - tiered_pricing: Final = deployment_model_info.get("tiered_pricing") +def _tiered_pricing_table(deployment_info: Mapping[str, object]) -> Sequence[Mapping[str, object]] | None: + tiered_pricing: Final = deployment_info.get("tiered_pricing") if isinstance(tiered_pricing, list) and tiered_pricing: return tiered_pricing return None -def _get_deployment_tiered_pricing_tables( +def _deployment_model_info( + deployment: DeploymentTypedDict, + llm_router: Router, +) -> Mapping[str, object] | None: + model_id: Final = _get_value(_get_value(deployment, "model_info"), "id") + backend_model: Final = _get_value(_get_value(deployment, "litellm_params"), "model") + if not isinstance(model_id, str) or not isinstance(backend_model, str): + return None + return llm_router.cached_deployment_model_info(model_id, backend_model) + + +def _get_deployment_model_infos( model: str, llm_router: Router | None, -) -> Sequence[Sequence[Mapping[str, object]]]: +) -> Sequence[Mapping[str, object]]: if llm_router is None: return [] deployments: Final = llm_router.get_model_list(model_name=model) or [] return [ - table + deployment_info for deployment in deployments - if (table := _deployment_tiered_pricing_table(deployment, llm_router)) is not None + if (deployment_info := _deployment_model_info(deployment, llm_router)) is not None ] @@ -1614,14 +1622,10 @@ def _estimate_input_tokens( if input_tokens is not None else count_input_tokens_for_model(request_body=request_body, model=model) ) - if counted is not None: - return counted - - max_input_tokens: Final = _to_int(model_info.get("max_input_tokens")) - if max_input_tokens is not None: - return max_input_tokens - - return None + estimated: Final = counted if counted is not None else _to_int(model_info.get("max_input_tokens")) + if estimated is None: + return None + return max(estimated, _to_int(model_info.get("min_billed_input_tokens")) or 0) DEFAULT_MAX_OUTPUT_TOKENS_FALLBACK: Final = 16384 diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 45636725b83..2d4d6ef2f0c 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -1,5 +1,4 @@ import json -from pathlib import Path from typing import Final import pytest @@ -382,14 +381,3 @@ def test_vispark_models_are_listed_for_the_provider(): assert expected assert set(litellm.models_by_provider["vispark"]) == expected assert expected <= set(get_valid_models(custom_llm_provider="vispark")) - - -def test_vispark_endpoint_matrix_matches_between_root_and_backup(): - backup_path: Final = Path(litellm.__file__).parent / "provider_endpoints_support_backup.json" - root_path: Final = Path(litellm.__file__).parent.parent / "provider_endpoints_support.json" - - backup_entry: Final = json.loads(backup_path.read_text())["providers"]["vispark"] - root_entry: Final = json.loads(root_path.read_text())["providers"]["vispark"] - - assert backup_entry == root_entry - assert root_entry["endpoints"]["chat_completions"] is True diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 024fe5229be..c133166023e 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -25,6 +25,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, + estimate_request_input_cost, estimate_request_max_cost, get_budget_window_start, release_unbound_budget_reservation, @@ -220,6 +221,37 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) +@pytest.mark.parametrize("deployment_minimum", [None, 0]) +def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_minimum: int | None) -> None: + backend_model: Final = "vispark/vision-small" + backend_info: Final = litellm.get_model_info(model=backend_model) + built_in_minimum: Final = backend_info.get("min_billed_input_tokens") + input_cost_per_token: Final = backend_info.get("input_cost_per_token") + assert built_in_minimum is not None and built_in_minimum > 0 + assert input_cost_per_token is not None + router: Final = Router( + model_list=[ + Deployment( + model_name="minimum-group", + litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"), + model_info=ModelInfo(id="minimum-deployment", min_billed_input_tokens=deployment_minimum), + ).model_dump() + ] + ) + prompt_tokens: Final = 12 + + reserved: Final = estimate_request_input_cost( + request_body={"model": "minimum-group", "messages": [{"role": "user", "content": "hi"}]}, + route="/chat/completions", + llm_router=router, + input_token_counts={"minimum-group": prompt_tokens}, + ) + + billed_tokens: Final = built_in_minimum if deployment_minimum is None else max(prompt_tokens, deployment_minimum) + assert reserved is not None + assert math.isclose(reserved, billed_tokens * input_cost_per_token) + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", From efa86a550b2a0819d4f0dcaef5e76e4c52c4f2fa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:16:44 -0700 Subject: [PATCH 13/18] test(budget): patch the renamed deployment info helper --- tests/unit/proxy/test_budget_reservation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/test_budget_reservation.py b/tests/unit/proxy/test_budget_reservation.py index 535c3da8c3a..4238992589b 100644 --- a/tests/unit/proxy/test_budget_reservation.py +++ b/tests/unit/proxy/test_budget_reservation.py @@ -1343,8 +1343,8 @@ def test_reservation_uses_most_expensive_deployment_in_group(): return_value={"max_output_tokens": 200000}, ), patch( - "litellm.proxy.spend_tracking.budget_reservation._get_deployment_tiered_pricing_tables", - return_value=[cheap, expensive], + "litellm.proxy.spend_tracking.budget_reservation._get_deployment_model_infos", + return_value=[{"tiered_pricing": cheap}, {"tiered_pricing": expensive}], ), patch( "litellm.proxy.spend_tracking.budget_reservation._estimate_input_tokens", From 85b41091ba9ed3ce74ff4c6365645e80baeb6179 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:18:50 -0700 Subject: [PATCH 14/18] fix(vispark): reject file_id parts, keep properties named additionalProperties, bill the minimum on dashscope and perplexity --- .../litellm_core_utils/llm_cost_calc/utils.py | 9 ++- litellm/llms/dashscope/cost_calculator.py | 5 +- litellm/llms/openai_like/dynamic_config.py | 74 ++++++++++++++++--- litellm/llms/openai_like/json_loader.py | 6 +- litellm/llms/perplexity/cost_calculator.py | 11 ++- .../code_coverage_tests/recursive_detector.py | 1 + .../test_dashscope_cost_calculator.py | 18 +++++ .../llms/openai_like/test_vispark_provider.py | 68 +++++++++++++++++ .../test_perplexity_cost_calculator.py | 20 +++++ 9 files changed, 190 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index c92462a2465..896682cfa7e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -659,6 +659,10 @@ def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) - return off_peak +def unbilled_minimum_input_tokens(model_info: ModelInfo, prompt_tokens: int) -> int: + return max((model_info.get("min_billed_input_tokens") or 0) - prompt_tokens, 0) + + def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates: """Swap in off-peak per-token rates when the current UTC time is inside one of the model's off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in @@ -1560,9 +1564,6 @@ def generic_cost_per_token( base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost ) - unbilled_minimum_input_tokens: Final = max( - (resolved_model_info.get("min_billed_input_tokens") or 0) - usage.prompt_tokens, 0 - ) prompt_cost = ( _calculate_input_cost( prompt_tokens_details=prompt_tokens_details, @@ -1573,7 +1574,7 @@ def generic_cost_per_token( cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, service_tier=service_tier, ) - + unbilled_minimum_input_tokens * base_costs.input_rate + + unbilled_minimum_input_tokens(resolved_model_info, usage.prompt_tokens) * base_costs.input_rate ) ## CALCULATE OUTPUT COST diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 55d6fb1a0e2..30995494eb9 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_completion_tokens_details, parse_prompt_tokens_details, resolve_explicit_cache_read_cost_rate, + unbilled_minimum_input_tokens, ) from litellm.types.utils import ModelInfo, Usage from litellm.utils import get_model_info @@ -151,4 +152,6 @@ def cost_per_token( rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate) ) - return _bill(breakdown, billed_rates) + prompt_cost, completion_cost = _bill(breakdown, billed_rates) + minimum_cost: Final = unbilled_minimum_input_tokens(model_info, usage.prompt_tokens) * billed_rates.input_rate + return prompt_cost + minimum_cost, completion_cost diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 85f25573987..e4754367baf 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -2,8 +2,7 @@ Dynamic configuration class generator for JSON-based providers. """ -import json -from collections.abc import Callable, Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, overload from pydantic import JsonValue, TypeAdapter @@ -11,6 +10,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, migrate_file_to_image_url, @@ -19,12 +19,17 @@ from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionUserMessage, + OpenAIMessageContentListBlock, +) from .json_loader import SimpleProviderConfig _JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _REQUESTED_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_SCHEMA_MAP_KEYWORDS: Final = frozenset({"properties", "patternProperties", "$defs", "definitions", "dependentSchemas"}) if TYPE_CHECKING: from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig @@ -40,25 +45,52 @@ def _with_file_parts_as_image_url(message: AllMessageValues) -> AllMessageValues return updated -def _without_additional_properties_flags(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: - return {key: item for key, item in node.items() if key != "additionalProperties" or not isinstance(item, bool)} +def _is_unreadable_file_part(part: OpenAIMessageContentListBlock) -> bool: + return part["type"] == "file" and not part["file"].get("file_data") -def _without_additional_properties_key(node: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: - return {key: item for key, item in node.items() if key != "additionalProperties"} +def _without_unreadable_file_parts(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "user" or not isinstance(message["content"], list): + return message + content: Final = [part for part in message["content"] if not _is_unreadable_file_part(part)] + updated: Final[ChatCompletionUserMessage] = {**message, "content": content} + return updated -def _rebuilt_json( - value: JsonValue, object_hook: Callable[[Mapping[str, JsonValue]], Mapping[str, JsonValue]] +def _messages_with_readable_file_parts( + messages: Sequence[AllMessageValues], provider_slug: str, drop_params: bool +) -> tuple[AllMessageValues, ...]: + readable: Final = tuple(_without_unreadable_file_parts(message) for message in messages) + if readable == tuple(messages) or drop_params: + return readable + raise UnsupportedParamsError( + message=( + f"{provider_slug} cannot read file parts sent by file_id. " + "Send the file as file_data, or set `drop_params=True` to drop them" + ), + llm_provider=provider_slug, + ) + + +def _without_additional_properties( + node: JsonValue, keep: Callable[[JsonValue], bool], is_schema_map: bool = False ) -> JsonValue: - return _JSON_VALUE_ADAPTER.validate_python(json.loads(json.dumps(value), object_hook=object_hook)) + if isinstance(node, list): + return [_without_additional_properties(item, keep) for item in node] + if not isinstance(node, dict): + return node + return { + key: _without_additional_properties(item, keep, not is_schema_map and key in _SCHEMA_MAP_KEYWORDS) + for key, item in node.items() + if is_schema_map or key != "additionalProperties" or keep(item) + } def _response_format_without_additional_properties( response_format: JsonValue, provider_slug: str, model: str, drop_params: bool ) -> JsonValue: - without_flags: Final = _rebuilt_json(response_format, _without_additional_properties_flags) - without_any: Final = _rebuilt_json(response_format, _without_additional_properties_key) + without_flags: Final = _without_additional_properties(response_format, lambda item: not isinstance(item, bool)) + without_any: Final = _without_additional_properties(response_format, lambda _: False) if without_flags == without_any or drop_params: return without_any raise UnsupportedParamsError( @@ -110,6 +142,24 @@ def create_config_class(provider: SimpleProviderConfig): return super()._transform_messages(messages=transformed, model=model, is_async=True) return super()._transform_messages(messages=transformed, model=model, is_async=False) + def _prompt_cache_ordered_messages( + self, + messages: list[AllMessageValues], # mutable-ok: overrides OpenAIGPTConfig's list signature + litellm_params: Mapping[str, object], + ) -> list[AllMessageValues]: # mutable-ok: overrides OpenAIGPTConfig's list signature + readable: Final = ( + list( + _messages_with_readable_file_parts( + messages, + provider_slug=provider.slug, + drop_params=drops_unsupported_params(litellm_params.get("drop_params")), + ) + ) + if provider.special_handling.get("convert_file_parts_to_image_url") + else messages + ) + return super()._prompt_cache_ordered_messages(readable, litellm_params) + def _should_preserve_cache_control_for_endpoint( self, custom_llm_provider: str | None, diff --git a/litellm/llms/openai_like/json_loader.py b/litellm/llms/openai_like/json_loader.py index 6e99b9e7fec..6b3fe763277 100644 --- a/litellm/llms/openai_like/json_loader.py +++ b/litellm/llms/openai_like/json_loader.py @@ -24,9 +24,9 @@ class SimpleProviderConfig: self.base_url = data["base_url"] self.api_key_env: str = data["api_key_env"] self.api_base_env: str | None = data.get("api_base_env") - self.base_class = data.get("base_class", "openai_gpt") - self.param_mappings = data.get("param_mappings", {}) - self.constraints = data.get("constraints", {}) + self.base_class: str = data.get("base_class", "openai_gpt") + self.param_mappings: Mapping[str, str] = data.get("param_mappings", {}) + self.constraints: Mapping[str, float] = data.get("constraints", {}) self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python( data.get("special_handling", {}) ) diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..e46d66c247a 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -6,7 +6,11 @@ Helper util for handling perplexity-specific cost calculation from datetime import datetime from typing import Final -from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + TokenRates, + apply_off_peak_pricing, + unbilled_minimum_input_tokens, +) from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -67,7 +71,10 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non output_cost_per_token: Final = rates.output_rate ## CALCULATE INPUT COST - prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token + prompt_tokens: Final = usage.prompt_tokens or 0 + prompt_cost: float = ( + prompt_tokens + unbilled_minimum_input_tokens(model_info, prompt_tokens) + ) * input_cost_per_token ## ADD CITATION TOKENS COST (if present) citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0 diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..7d65f4111f3 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -84,6 +84,7 @@ IGNORE_FUNCTIONS = [ "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. + "_without_additional_properties", # bounded by the nesting depth of a pydantic-validated JsonValue response_format (a finite JSON tree, no cycles possible). ] diff --git a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py index 551caab7153..7dadbe653fc 100644 --- a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py @@ -994,6 +994,24 @@ class TestDashscopeCostCalculator: assert prompt_cost == 0.0 assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10) + @pytest.mark.parametrize(("prompt_tokens", "billed_prompt_tokens"), [(10, 600), (900, 900)]) + def test_bills_at_least_the_minimum_input_tokens(self, prompt_tokens: int, billed_prompt_tokens: int): + litellm.model_cost["dashscope/qwen-minimum-test"] = { + "litellm_provider": "dashscope", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "min_billed_input_tokens": 600, + } + + prompt_cost, completion_cost = dashscope_cost_per_token( + model="qwen-minimum-test", + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7), + ) + + assert math.isclose(prompt_cost, billed_prompt_tokens * 1e-06, rel_tol=1e-10) + assert math.isclose(completion_cost, 7 * 2e-06, rel_tol=1e-10) + OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc) diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 2d4d6ef2f0c..05cc52435ab 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -201,6 +201,74 @@ def test_vispark_drops_a_map_field_schema_with_drop_params(): assert sent_schema == {"type": "object", "properties": {"facts": {"type": "object"}}, "required": ["facts"]} +def test_vispark_keeps_a_schema_property_named_additional_properties(): + schema: Final = { + "type": "object", + "properties": {"additionalProperties": {"type": "string", "additionalProperties": False}}, + "required": ["additionalProperties"], + "additionalProperties": False, + } + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + litellm.completion( + model="vispark/vision-medium", + messages=[{"role": "user", "content": "Describe Paris as JSON"}], + response_format={"type": "json_schema", "json_schema": {"name": "Notes", "schema": schema}}, + api_key="vispark-test-key", + ) + + sent_schema: Final = json.loads(route.calls.last.request.content)["response_format"]["json_schema"]["schema"] + assert sent_schema == { + "type": "object", + "properties": {"additionalProperties": {"type": "string"}}, + "required": ["additionalProperties"], + } + + +def _file_id_messages() -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this file"}, + {"type": "file", "file": {"file_id": "file-abc123"}}, + ], + } + ] + + +def test_vispark_rejects_a_file_part_sent_by_file_id_without_drop_params(): + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + with pytest.raises(litellm.BadRequestError, match="file_id") as error: + litellm.completion(model="vispark/vision-medium", messages=_file_id_messages(), api_key="vispark-test-key") + + assert error.value.status_code == 400 + assert route.call_count == 0 + + +@pytest.mark.asyncio +async def test_vispark_drops_a_file_part_sent_by_file_id_with_drop_params(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock() as upstream: + route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( + 200, json=_CHAT_COMPLETION + ) + await litellm.acompletion( + model="vispark/vision-medium", + messages=_file_id_messages(), + drop_params=True, + api_key="vispark-test-key", + ) + + sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"] + assert sent_content == [{"type": "text", "text": "Summarize this file"}] + + def test_other_json_providers_keep_additional_properties_and_file_parts(): file_part: Final = {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0xLjQK"}} response_format: Final = { diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..9f82514d6f4 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -140,6 +140,26 @@ class TestPerplexityCostCalculator: assert prompt_cost == 0.0 assert completion_cost == 0.008 + @pytest.mark.parametrize(("prompt_tokens", "billed_prompt_tokens"), [(10, 600), (900, 900)]) + def test_manual_pricing_bills_at_least_the_minimum_input_tokens( + self, prompt_tokens: int, billed_prompt_tokens: int + ): + litellm.model_cost["perplexity/sonar-minimum-test"] = { + "litellm_provider": "perplexity", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "min_billed_input_tokens": 600, + } + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-minimum-test", + usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=7, total_tokens=prompt_tokens + 7), + ) + + assert math.isclose(prompt_cost, billed_prompt_tokens * 1e-06, rel_tol=1e-10) + assert math.isclose(completion_cost, 7 * 2e-06, rel_tol=1e-10) + OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) From d71101d330ed49a220444efedbf1064b5155ec88 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:00:31 -0700 Subject: [PATCH 15/18] fix(vispark): send inline file_data when a file part also carries a file_id --- .../prompt_templates/common_utils.py | 2 +- .../llms/openai_like/test_vispark_provider.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 9bc53791b36..2630b4b76b5 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2238,7 +2238,7 @@ def migrate_file_to_image_url( image_url_object: Final = ChatCompletionImageObject( type="image_url", image_url=ChatCompletionImageUrlObject( - url=cast(str, file_id or file_data), + url=cast(str, file_data or file_id), ), ) if format and isinstance(image_url_object["image_url"], dict): diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index 05cc52435ab..a6dcb4ccf5a 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -292,8 +292,15 @@ def test_other_json_providers_keep_additional_properties_and_file_parts(): assert body["messages"][0]["content"][1] == file_part -def test_vispark_sends_a_pdf_file_part_as_an_image_url(): - pdf_data_url: Final = "data:application/pdf;base64,JVBERi0xLjQK" +_PDF_DATA_URL: Final = "data:application/pdf;base64,JVBERi0xLjQK" + + +@pytest.mark.parametrize( + "file", + [{"file_data": _PDF_DATA_URL}, {"file_id": "file-abc123", "file_data": _PDF_DATA_URL}], + ids=["file_data", "file_id_and_file_data"], +) +def test_vispark_sends_a_pdf_file_part_as_an_image_url(file: dict[str, str]): with respx.mock() as upstream: route: Final = upstream.post("https://api.lab.vispark.in/v1/chat/completions").respond( 200, json=_CHAT_COMPLETION @@ -305,7 +312,7 @@ def test_vispark_sends_a_pdf_file_part_as_an_image_url(): "role": "user", "content": [ {"type": "text", "text": "What code is in this PDF?"}, - {"type": "file", "file": {"file_data": pdf_data_url}}, + {"type": "file", "file": file}, ], } ], @@ -315,7 +322,7 @@ def test_vispark_sends_a_pdf_file_part_as_an_image_url(): sent_content: Final = json.loads(route.calls.last.request.content)["messages"][0]["content"] assert sent_content[0] == {"type": "text", "text": "What code is in this PDF?"} assert sent_content[1]["type"] == "image_url" - assert sent_content[1]["image_url"]["url"] == pdf_data_url + assert sent_content[1]["image_url"]["url"] == _PDF_DATA_URL assert all(part["type"] != "file" for part in sent_content) From aa6ecf0878a810afd5b34ad538ac21a3b3ed8f70 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:11:44 -0700 Subject: [PATCH 16/18] fix: keep provider-stated cost for minimum-only deployments and skip the minimum without a counted prompt A deployment that sets only min_billed_input_tokens no longer drops Perplexity or Kyma stated cost, streamed cost headers no longer bill the minimum before usage arrives, and tiered deployments now carry the minimum --- litellm/cost_calculator.py | 21 +++++++-- litellm/litellm_core_utils/litellm_logging.py | 12 +++-- .../litellm_core_utils/llm_cost_calc/utils.py | 2 + litellm/router.py | 2 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 9 +++- .../llms/openai_like/test_vispark_provider.py | 28 +++++++++-- tests/unit/test_cost_calculator.py | 46 +++++++++++++++++++ 7 files changed, 107 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index dd43988aea5..ce63d34d48d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -525,7 +525,7 @@ def cost_per_token( billed_prompt_tokens: Final = ( max(_normalized_prompt_tokens, _min_billed_input_tokens(model, custom_llm_provider)) - if custom_cost_per_token is not None + if custom_cost_per_token is not None and _normalized_prompt_tokens > 0 else _normalized_prompt_tokens ) response_cost: Final = _cost_per_token_custom_pricing_helper( @@ -1472,6 +1472,17 @@ def _store_cost_breakdown_in_logging_obj( # Don't fail the main cost calculation if breakdown storage fails +def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | None) -> bool: + from litellm.litellm_core_utils.litellm_logging import MODEL_INFO_RATE_KEYS, use_custom_pricing_for_model + + logged_params: Final = getattr(litellm_logging_obj, "litellm_params", None) + return ( + isinstance(logged_params, dict) + and use_custom_pricing_for_model(logged_params) + and not use_custom_pricing_for_model(logged_params, MODEL_INFO_RATE_KEYS) + ) + + def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: if usage is None or getattr(usage, "cost", None) is None: return usage @@ -1643,8 +1654,9 @@ def completion_cost( cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 provider_usage_object: Final = get_usage_object(completion_response=completion_response) + sets_prices: Final = custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj) cost_per_token_usage_object: Final[Usage | None] = ( - _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object + _without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None @@ -1659,7 +1671,10 @@ def completion_cost( explicit_pricing: Final = custom_pricing is True or base_model is not None caller_set_prices: Final = ( - explicit_pricing or custom_cost_per_token is not None or custom_cost_per_second is not None + sets_prices + or base_model is not None + or custom_cost_per_token is not None + or custom_cost_per_second is not None ) custom_priced_request_model: Final = model if custom_pricing is True else None configured_cost_per_query: Final = ( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c0e988412a1..b68cbd8a988 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -351,6 +351,7 @@ def _get_provider_request_id(original_exception: Exception) -> str | None: _CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) _METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS +MODEL_INFO_RATE_KEYS: Final[frozenset[str]] = _MODEL_INFO_CUSTOM_PRICING_KEYS - {"min_billed_input_tokens"} _UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset( ("user_api_key_auth", "user_api_key_budget_reservation") ) @@ -6384,13 +6385,16 @@ def _deployment_model_info(litellm_params: dict | None) -> Mapping[str, object]: return MappingProxyType({}) -def use_custom_pricing_for_model(litellm_params: dict | None) -> bool: +def use_custom_pricing_for_model( + litellm_params: dict | None, model_info_pricing_keys: frozenset[str] = _MODEL_INFO_CUSTOM_PRICING_KEYS +) -> bool: """ Check if the model uses custom pricing Returns True if any custom pricing field is present in `litellm_params`, or if - any custom pricing or deployment-scoped pricing field (such as - ``off_peak_pricing``) is present in the metadata ``model_info`` + any of `model_info_pricing_keys` (by default every custom pricing or + deployment-scoped pricing field, such as ``off_peak_pricing``) is present in + the metadata ``model_info`` """ if litellm_params is None: return False @@ -6408,7 +6412,7 @@ def use_custom_pricing_for_model(litellm_params: dict | None) -> bool: model_info: dict = metadata.get("model_info", {}) or {} if model_info: - matching_keys = _MODEL_INFO_CUSTOM_PRICING_KEYS & model_info.keys() + matching_keys = model_info_pricing_keys & model_info.keys() for key in matching_keys: if model_info.get(key) is not None: return True diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 896682cfa7e..de483efc072 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -660,6 +660,8 @@ def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) - def unbilled_minimum_input_tokens(model_info: ModelInfo, prompt_tokens: int) -> int: + if prompt_tokens <= 0: + return 0 return max((model_info.get("min_billed_input_tokens") or 0) - prompt_tokens, 0) diff --git a/litellm/router.py b/litellm/router.py index 6a193a5efb6..ca82f071992 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11918,7 +11918,7 @@ class Router(DeprecatedRouterMembers): ) backend_info: Final = ( lookup_model_info_for_registration(model=backend_model, custom_llm_provider=pricing_provider) - if model_info.get("input_cost_per_token") is not None + if model_info.get("input_cost_per_token") is not None or model_info.get("tiered_pricing") is not None else None ) billing_minimum: Final = ( diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 2425e8077f8..3b093815931 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -101,8 +101,13 @@ def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_re @pytest.mark.parametrize( ("prompt_tokens", "billed_prompt_tokens"), - [(10, 600), (600, 600), (900, 900)], - ids=["below_the_minimum_bills_the_minimum", "at_the_minimum", "above_the_minimum_bills_as_reported"], + [(0, 0), (10, 600), (600, 600), (900, 900)], + ids=[ + "no_counted_prompt_bills_no_minimum", + "below_the_minimum_bills_the_minimum", + "at_the_minimum", + "above_the_minimum_bills_as_reported", + ], ) def test_generic_cost_per_token_bills_at_least_the_minimum_input_tokens( prompt_tokens: int, billed_prompt_tokens: int diff --git a/tests/unit/llms/openai_like/test_vispark_provider.py b/tests/unit/llms/openai_like/test_vispark_provider.py index a6dcb4ccf5a..129a7083df0 100644 --- a/tests/unit/llms/openai_like/test_vispark_provider.py +++ b/tests/unit/llms/openai_like/test_vispark_provider.py @@ -372,7 +372,17 @@ def test_vispark_bills_a_short_prompt_at_its_minimum_input_tokens(model: str): assert prompt_cost == pytest.approx(minimum * cost_entry["input_cost_per_token"]) -def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens(): +@pytest.mark.parametrize( + "custom_pricing", + [ + {"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + {"tiered_pricing": [{"range": [0, 1_000_000], "input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}]}, + ], + ids=["flat", "tiered"], +) +def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_input_tokens( + custom_pricing: dict[str, object], +): minimum: Final = litellm.model_cost["vispark/vision-small"]["min_billed_input_tokens"] router: Final = litellm.Router( model_list=[ @@ -381,8 +391,7 @@ def test_vispark_router_deployment_with_custom_pricing_still_bills_the_minimum_i "litellm_params": { "model": "vispark/vision-small", "api_key": "vispark-test-key", - "input_cost_per_token": 2e-6, - "output_cost_per_token": 5e-6, + **custom_pricing, }, } ] @@ -448,6 +457,19 @@ def test_vispark_explicit_custom_cost_per_token_still_bills_the_minimum_input_to assert completion_cost == pytest.approx(3 * 5e-6) +def test_vispark_explicit_custom_cost_per_token_bills_no_minimum_without_a_counted_prompt(): + prompt_cost, completion_cost = litellm.cost_per_token( + model="vision-small", + custom_llm_provider="vispark", + prompt_tokens=0, + completion_tokens=3, + custom_cost_per_token={"input_cost_per_token": 2e-6, "output_cost_per_token": 5e-6}, + ) + + assert prompt_cost == 0.0 + assert completion_cost == pytest.approx(3 * 5e-6) + + def test_vispark_models_are_listed_for_the_provider(): from litellm.utils import get_valid_models diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 84f6dab9eff..c874e9964bc 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10589,3 +10589,49 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) + + +@pytest.mark.parametrize( + ("model", "chat_url", "usage_cost", "stated_cost"), + [ + ( + "perplexity/sonar", + "https://api.perplexity.ai/chat/completions", + {"request_cost": 0.0025, "total_cost": 0.00266}, + 0.00266, + ), + ("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721), + ], + ids=["perplexity", "kyma"], +) +def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost( + _local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float +): + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "deployment-with-minimum", + "litellm_params": {"model": model, "api_key": "provider-test-key"}, + "model_info": {"min_billed_input_tokens": 1000}, + } + ] + ) + with respx.mock() as upstream: + upstream.post(chat_url).respond( + 200, + json={ + "id": "chatcmpl_minimum", + "object": "chat.completion", + "created": 1_790_000_000, + "model": "sonar", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Paris"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8, "cost": usage_cost}, + }, + ) + response: Final = router.completion( + model="deployment-with-minimum", messages=[{"role": "user", "content": "Capital of France?"}] + ) + + assert response._hidden_params["response_cost"] == pytest.approx(stated_cost) From dda069ea01204b6d9a90f7e63232fc3cc932a305 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:13:14 -0700 Subject: [PATCH 17/18] fix: bill provider-stated cost for minimum-only deployments and charge a disconnect only the group's shared minimum --- litellm/cost_calculator.py | 12 +++++-- .../anthropic_passthrough_logging_handler.py | 3 +- .../spend_tracking/budget_reservation.py | 23 +++++++----- ...t_anthropic_passthrough_logging_handler.py | 18 ++++++++++ .../spend_tracking/test_budget_reservation.py | 36 +++++++++++++++++++ tests/unit/test_cost_calculator.py | 3 +- 6 files changed, 82 insertions(+), 13 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ce63d34d48d..ea873a61f0d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1483,6 +1483,12 @@ def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | N ) +def overrides_provider_stated_cost( + custom_pricing: bool | None, litellm_logging_obj: LitellmLoggingObject | None +) -> bool: + return custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj) + + def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: if usage is None or getattr(usage, "cost", None) is None: return usage @@ -1654,7 +1660,7 @@ def completion_cost( cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 provider_usage_object: Final = get_usage_object(completion_response=completion_response) - sets_prices: Final = custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj) + sets_prices: Final = overrides_provider_stated_cost(custom_pricing, litellm_logging_obj) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object ) @@ -2357,7 +2363,9 @@ def response_cost_calculator( getattr(response_object, HIDDEN_PARAMS_ATTR), ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) - if provider_response_cost is not None and not custom_pricing: + if provider_response_cost is not None and not overrides_provider_stated_cost( + custom_pricing, litellm_logging_obj + ): return provider_response_cost response_cost = completion_cost( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c1fc6479421..66f39fcd3c9 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE +from litellm.cost_calculator import overrides_provider_stated_cost from litellm.litellm_core_utils.core_helpers import ( native_messages_reported_cost, override_stamped_response_cost, @@ -346,7 +347,7 @@ class AnthropicPassthroughLoggingHandler: if isinstance(litellm_model_response, ModelResponse) else None ) - if reported_cost is not None and not custom_pricing: + if reported_cost is not None and not overrides_provider_stated_cost(custom_pricing, logging_obj): return reported_cost return litellm.completion_cost( completion_response=litellm_model_response, diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 04882a5da12..d09784bf564 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -4,7 +4,7 @@ import asyncio import json import math import time -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone from itertools import chain @@ -1352,7 +1352,7 @@ def _estimate_request_input_cost_for_model( model_info=model_info, input_tokens=input_tokens, ) - for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) + for model_info in _get_model_cost_infos(model=model, llm_router=llm_router, group_billing_minimum=min) ] valid_estimates: Final = [estimate for estimate in estimates if estimate is not None] return max(valid_estimates) if valid_estimates else None @@ -1521,6 +1521,7 @@ def _get_model_cost_info( def _get_model_cost_infos( model: str, llm_router: Router | None, + group_billing_minimum: Callable[[Sequence[int]], int] = max, ) -> Sequence[Mapping[str, object]]: """Cost-info candidates to estimate a request against for one model group. @@ -1528,7 +1529,9 @@ def _get_model_cost_infos( is unknown. Rather than guess, we estimate the cost against every eligible pricing shape in the group (the group's flat rates plus each deployment's tiered table) and let the caller reserve the maximum, so a cheaper sibling - deployment can never leave the request under-reserved. + deployment can never leave the request under-reserved. The deployments' + billed input minimums fold into one through ``group_billing_minimum``, the + largest for admission and the smallest for the input cost a disconnect is charged. """ try: group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router) @@ -1541,12 +1544,14 @@ def _get_model_cost_infos( exc_info=True, ) return [] - billing_minimums: Final = [ - minimum - for deployment_info in deployment_infos - if (minimum := _to_int(deployment_info.get("min_billed_input_tokens"))) is not None - ] - base: Final = {**group_info, "min_billed_input_tokens": max(billing_minimums)} if billing_minimums else group_info + billing_minimums: Final = tuple( + _to_int(deployment_info.get("min_billed_input_tokens")) or 0 for deployment_info in deployment_infos + ) + base: Final = ( + {**group_info, "min_billed_input_tokens": group_billing_minimum(billing_minimums)} + if any(billing_minimums) + else group_info + ) tiered_tables: Final = [ table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None ] diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 47b6af9aeb3..59743bfc522 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2951,6 +2951,24 @@ def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delt assert result["result"]._hidden_params["response_cost"] == 0.00412 +def test_streamed_messages_from_a_deployment_that_only_sets_a_billing_minimum_bill_the_reported_cost(): + logging_obj = _go_logging_obj() + logging_obj.litellm_params = {"metadata": {"model_info": {"min_billed_input_tokens": 100000}}} + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=_go_stream_chunks(), + end_time=datetime.now(), + ) + + assert result["kwargs"]["response_cost"] == 0.00412 + + def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field(): logging_obj = _go_logging_obj() logging_obj.model_call_details["custom_llm_provider"] = "anthropic" diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 78c229dd1f5..5918e17be79 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -253,6 +253,42 @@ def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_m assert math.isclose(reserved, billed_tokens * input_cost_per_token) +@pytest.mark.parametrize("sibling_minimum", [0, None]) +def test_disconnect_charge_is_only_the_minimum_every_deployment_in_the_group_bills(sibling_minimum: int | None) -> None: + backend_model: Final = "vispark/vision-small" + backend_info: Final = litellm.get_model_info(model=backend_model) + built_in_minimum: Final = backend_info.get("min_billed_input_tokens") + input_cost_per_token: Final = backend_info.get("input_cost_per_token") + assert built_in_minimum is not None and built_in_minimum > 0 + assert input_cost_per_token is not None + largest_minimum: Final = built_in_minimum * 100 + router: Final = Router( + model_list=[ + Deployment( + model_name="mixed-minimum-group", + litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"), + model_info=ModelInfo(id=deployment_id, min_billed_input_tokens=minimum), + ).model_dump() + for deployment_id, minimum in (("largest-minimum", largest_minimum), ("sibling", sibling_minimum)) + ] + ) + prompt_tokens: Final = 12 + request_body: Final = {"model": "mixed-minimum-group", "messages": [{"role": "user", "content": "hi"}]} + token_counts: Final = {"mixed-minimum-group": prompt_tokens} + + disconnect_charge: Final = estimate_request_input_cost( + request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts + ) + admission_estimate: Final = estimate_request_max_cost( + request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts + ) + + shared_minimum: Final = built_in_minimum if sibling_minimum is None else sibling_minimum + assert disconnect_charge is not None and admission_estimate is not None + assert math.isclose(disconnect_charge, max(prompt_tokens, shared_minimum) * input_cost_per_token) + assert admission_estimate >= largest_minimum * input_cost_per_token + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index c874e9964bc..c6bedb758bd 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10601,8 +10601,9 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( 0.00266, ), ("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721), + ("openrouter/openai/gpt-5.4-nano", "https://openrouter.ai/api/v1/chat/completions", 0.00721, 0.00721), ], - ids=["perplexity", "kyma"], + ids=["perplexity", "kyma", "openrouter"], ) def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost( _local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float From 0c0203f4e190ed0684ee84bc40b5019328f2ddc4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:36:56 -0700 Subject: [PATCH 18/18] style: drop the restating comment on the vispark provider entry --- litellm/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index d2dc4533af7..549693c2546 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1140,7 +1140,7 @@ openai_compatible_providers: Final[list[str]] = [ "pinstripes", # Pinstripes - JSON-configured provider "openinfer", "darkbloom", - "vispark", # Vispark Lab - JSON-configured provider + "vispark", "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", "coralbricks",