From 539a61be080b9ed8100ef00fca77f5b6175d8853 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sun, 16 Aug 2026 14:44:09 -0400 Subject: [PATCH 1/5] feat(perplexity): add Agent API third-party models (DeepSeek V4 Flash, GLM 5.2, Kimi K3, Kimi K2.7 Code) --- ...odel_prices_and_context_window_backup.json | 44 +++++++++++++++++++ model_prices_and_context_window.json | 44 +++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e6c6cab0631..b786a84ada7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34286,6 +34286,50 @@ "supports_reasoning": false, "supports_function_calling": true }, + "perplexity/perplexity/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 2.6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 1.5e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, "perplexity/pplx-embed-v1-0.6b": { "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e6c6cab0631..b786a84ada7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34286,6 +34286,50 @@ "supports_reasoning": false, "supports_function_calling": true }, + "perplexity/perplexity/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 2.8e-08, + "input_cost_per_token": 1.3e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 2.6e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/glm-5.2": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4.4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 1.5e-05, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": true, + "supports_function_calling": true + }, + "perplexity/perplexity/kimi-k2.7-code": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 9.5e-07, + "litellm_provider": "perplexity", + "mode": "responses", + "output_cost_per_token": 4e-06, + "source": "https://docs.perplexity.ai/docs/agent-api/models", + "supports_web_search": true, + "supports_reasoning": false, + "supports_function_calling": true + }, "perplexity/pplx-embed-v1-0.6b": { "input_cost_per_token": 4e-09, "litellm_provider": "perplexity", From 782746553c109bdec6aa5ecddf8e674f5167f575 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Sun, 16 Aug 2026 14:57:11 -0400 Subject: [PATCH 2/5] fix(perplexity): accept float usage.cost in cost_per_token, not just dict ResponseAPIUsage.parse_cost already flattens Perplexity's usage.cost.total_cost dict down to a float before it reaches the perplexity cost calculator, so the isinstance(cost_info, dict) check was always False on that path. Every Responses-mode Perplexity model was silently falling back to manual token-rate calculation and recording $0 spend whenever static per-token rates were missing. --- litellm/llms/perplexity/cost_calculator.py | 19 ++++++++------ .../test_perplexity_cost_calculator.py | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 337fa8e630d..27835ecbfe8 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -21,14 +21,19 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ ## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE - ## Perplexity returns accurate cost in usage.cost.total_cost including request fees + ## Perplexity returns accurate cost in usage.cost.total_cost including request fees. + ## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened + ## that dict down to a float, so both shapes must be accepted. cost_info: Final = getattr(usage, "cost", None) - if cost_info is not None and isinstance(cost_info, dict): - total_cost: Final = cost_info.get("total_cost") - if total_cost is not None: - # Return total cost as completion_cost (prompt_cost=0) since Perplexity - # doesn't break down by input/output in their cost object - return (0.0, float(total_cost)) + total_cost: float | None = None + if isinstance(cost_info, dict): + total_cost = cost_info.get("total_cost") + elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): + total_cost = float(cost_info) + if total_cost is not None: + # Return total cost as completion_cost (prompt_cost=0) since Perplexity + # doesn't break down by input/output in their cost object + return (0.0, float(total_cost)) ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index 46c1e457d7c..71ccb494cc7 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -400,6 +400,31 @@ class TestPerplexityCostCalculator: assert completion_cost == 0.008 assert prompt_cost + completion_cost == 0.008 + def test_uses_perplexity_provided_cost_when_normalized_to_float(self): + """ + Regression: for Responses API / Agent API models, `ResponseAPIUsage.parse_cost` + (litellm/types/llms/openai.py) already flattens Perplexity's + `usage.cost.total_cost` dict down to a plain float before + `_transform_response_api_usage_to_chat_usage` (litellm/responses/utils.py) copies + it onto the chat `Usage` object. So `usage.cost` arrives here as a float, not a + dict, on that path. + + Pre-fix, the `isinstance(cost_info, dict)` check was always False for a float, + so the pre-calculated cost branch was dead code for every Responses-mode + Perplexity model and it silently fell back to manual token-rate calculation, + recording $0 for any model missing static per-token rates (e.g. + perplexity/openai/gpt-5.2 before rates existed). + """ + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + usage.cost = 0.008 + + prompt_cost, completion_cost = perplexity_cost_per_token( + model="sonar-pro", usage=usage + ) + + assert prompt_cost == 0.0 + assert completion_cost == 0.008 + def test_falls_back_to_manual_calculation_when_no_cost_provided(self): """ Test that manual cost calculation is used when Perplexity doesn't From 0bdaa98b28e0866e62a6c3514c0128fed1b6ffb0 Mon Sep 17 00:00:00 2001 From: mateo-berri Date: Thu, 20 Aug 2026 12:48:41 -0700 Subject: [PATCH 3/5] fix(perplexity): correct glm-5.2 cache read rate to the published catalog rate The new perplexity/perplexity/glm-5.2 row carried 2.6e-07, which is glm-5.3's cache read rate. api.perplexity.ai/v1/models publishes 0.14 usd per 1M cached input tokens for glm-5.2, so the rate is 1.4e-07. --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9813d3039fe..2223e2ea7c4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34736,7 +34736,7 @@ "supports_function_calling": true }, "perplexity/perplexity/glm-5.2": { - "cache_read_input_token_cost": 2.6e-07, + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, "litellm_provider": "perplexity", "mode": "responses", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9813d3039fe..2223e2ea7c4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34736,7 +34736,7 @@ "supports_function_calling": true }, "perplexity/perplexity/glm-5.2": { - "cache_read_input_token_cost": 2.6e-07, + "cache_read_input_token_cost": 1.4e-07, "input_cost_per_token": 1.4e-06, "litellm_provider": "perplexity", "mode": "responses", From 88ef47377fb3b90f04c0a21ee8f0bf737f2b711e Mon Sep 17 00:00:00 2001 From: mateo-berri Date: Thu, 20 Aug 2026 12:48:41 -0700 Subject: [PATCH 4/5] fix(utils): resolve model ids that repeat the litellm provider prefix Perplexity's Agent API model ids already start with perplexity/, so a litellm model like perplexity/perplexity/glm-5.2 normalizes to model perplexity/glm-5.2 with custom_llm_provider perplexity. Every existing candidate in the cost map lookup ladder reads that leading perplexity/ as the litellm prefix and strips it, so the four new rows never matched and supports_reasoning, get_model_info, and completion_cost all failed on them. Add the doubled form as a last-resort candidate, tried after every candidate tried today and still ahead of the capability generalization rules, so only lookups that already raised can newly succeed. Sweeping all 3062 cost map keys across the three _get_potential_model_names branches (8433 probes) shows 7 lookups change, all of them errors that now resolve: the 4 new perplexity rows plus openrouter/openrouter/auto, /free, and /bodybuilder. Nothing regresses. --- litellm/utils.py | 23 ++++++- .../test_perplexity_cost_calculator.py | 49 +++++++++++++ tests/test_litellm/test_utils.py | 69 +++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 867f7a93452..b8a9ea37e05 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5245,7 +5245,7 @@ def _check_provider_match(model_info: dict, custom_llm_provider: str | None) -> return True -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class PotentialModelNamesAndCustomLLMProvider(TypedDict): @@ -5253,6 +5253,7 @@ class PotentialModelNamesAndCustomLLMProvider(TypedDict): combined_model_name: str stripped_model_name: str combined_stripped_model_name: str + provider_prefixed_model_name: ReadOnly[str] custom_llm_provider: str @@ -5280,6 +5281,7 @@ def _get_model_info_from_generalization( potential_model_names["split_model"], potential_model_names["combined_stripped_model_name"], potential_model_names["stripped_model_name"], + potential_model_names["provider_prefixed_model_name"], ) if any(_get_model_cost_key(candidate) is not None for candidate in candidates): return None @@ -5304,6 +5306,7 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P combined_model_name = model stripped_model_name = _strip_model_name(model=model, custom_llm_provider=custom_llm_provider) combined_stripped_model_name = stripped_model_name + provider_prefixed_model_name = model elif custom_llm_provider and model.startswith( custom_llm_provider + "/" ): # handle case where custom_llm_provider is provided and model starts with custom_llm_provider @@ -5311,11 +5314,13 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P combined_model_name = model stripped_model_name = _strip_model_name(model=split_model, custom_llm_provider=custom_llm_provider) combined_stripped_model_name = f"{custom_llm_provider}/{stripped_model_name}" + provider_prefixed_model_name = f"{custom_llm_provider}/{model}" else: split_model = model combined_model_name = f"{custom_llm_provider}/{model}" stripped_model_name = _strip_model_name(model=model, custom_llm_provider=custom_llm_provider) combined_stripped_model_name = f"{custom_llm_provider}/{stripped_model_name}" + provider_prefixed_model_name = combined_model_name if custom_llm_provider in ("bedrock", "bedrock_converse"): from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix @@ -5327,6 +5332,7 @@ def _get_potential_model_names(model: str, custom_llm_provider: str | None) -> P combined_model_name=combined_model_name, stripped_model_name=stripped_model_name, combined_stripped_model_name=combined_stripped_model_name, + provider_prefixed_model_name=provider_prefixed_model_name, custom_llm_provider=cast(str, custom_llm_provider), ) @@ -5435,6 +5441,7 @@ def _get_model_info_helper( combined_model_name: Final = potential_model_names["combined_model_name"] stripped_model_name: Final = potential_model_names["stripped_model_name"] combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"] + provider_prefixed_model_name: Final = potential_model_names["provider_prefixed_model_name"] split_model: Final = potential_model_names["split_model"] custom_llm_provider = potential_model_names["custom_llm_provider"] model_cost_custom_llm_provider: Final = custom_llm_provider @@ -5493,6 +5500,10 @@ def _get_model_info_helper( 3. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8" 4. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given. 5. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given. + 6. 'provider_prefixed_model_name' in litellm.model_cost, for providers whose own model ids repeat the + litellm provider name. Checks "perplexity/perplexity/glm-5.2" if model="perplexity/glm-5.2" and + custom_llm_provider="perplexity", where 1-5 all read the leading "perplexity/" as the litellm prefix + and strip it. Tried last so no model that already resolves through 1-5 can change. """ _model_info: dict[str, Any] | None = None @@ -5548,6 +5559,16 @@ def _get_model_info_helper( custom_llm_provider=model_cost_custom_llm_provider, ): _model_info = None + if _model_info is None: + _matched_key = _get_model_cost_key(provider_prefixed_model_name) + if _matched_key is not None: + key = _matched_key + _model_info = _get_model_info_from_model_cost(key=cast(str, key)) + if not _check_provider_match( + model_info=_model_info, + custom_llm_provider=model_cost_custom_llm_provider, + ): + _model_info = None if _model_info is None: generalization: Final = _get_model_info_from_generalization( diff --git a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py index 71ccb494cc7..16708e062e4 100644 --- a/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/test_litellm/llms/perplexity/test_perplexity_cost_calculator.py @@ -476,3 +476,52 @@ class TestPerplexityCostCalculator: assert math.isclose(prompt_cost, expected_prompt, rel_tol=1e-9) assert math.isclose(completion_cost, expected_completion, rel_tol=1e-9) + + @pytest.mark.parametrize( + "model_id, usd_per_1m_input, usd_per_1m_output, usd_per_1m_cache_read", + [ + ("deepseek-v4-flash-0731", 0.13, 0.26, 0.028), + ("glm-5.2", 1.4, 4.4, 0.14), + ("kimi-k3", 3.0, 15.0, 0.3), + ("kimi-k2.7-code", 0.95, 4.0, 0.19), + ], + ) + def test_agent_api_entries_carry_perplexity_published_rates( + self, model_id, usd_per_1m_input, usd_per_1m_output, usd_per_1m_cache_read + ): + """The Agent API third-party models are priced from Perplexity's own catalog + (GET https://api.perplexity.ai/v1/models, `pricing` in usd_per_1m_tokens). + Perplexity's model id already starts with `perplexity/`, so the cost-map key + doubles the prefix. Regression: glm-5.2 shipped glm-5.3's 0.26 cache-read rate, + copied from the neighbouring catalog row, an 86% overcharge on cached input. + """ + info = get_model_info( + model=f"perplexity/{model_id}", custom_llm_provider="perplexity" + ) + + assert info["key"] == f"perplexity/perplexity/{model_id}" + assert info["litellm_provider"] == "perplexity" + assert info["mode"] == "responses" + assert math.isclose(info["input_cost_per_token"], usd_per_1m_input / 1e6, rel_tol=1e-9) + assert math.isclose(info["output_cost_per_token"], usd_per_1m_output / 1e6, rel_tol=1e-9) + assert math.isclose( + info["cache_read_input_token_cost"], usd_per_1m_cache_read / 1e6, rel_tol=1e-9 + ) + + def test_agent_api_fallback_rates_price_a_response_without_metered_cost(self): + """Perplexity meters cost on the response, but when `usage.cost` is absent the + calculator falls back to the mapped per-token rates. Regression: that fallback + raised "This model isn't mapped yet" for every Agent API third-party model, + because the doubled cost-map key was unreachable from the resolution ladder. + """ + from litellm import ModelResponse + + response = ModelResponse() + response.usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + response.model = "perplexity/perplexity/glm-5.2" + + total_cost = completion_cost( + completion_response=response, custom_llm_provider="perplexity" + ) + + assert math.isclose(total_cost, 1000 * 1.4e-06 + 500 * 4.4e-06, rel_tol=1e-9) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index cb58038e081..df9dab1278e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -36,6 +36,7 @@ from litellm.utils import ( ProviderConfigManager, TextCompletionStreamWrapper, _check_provider_match, + _get_potential_model_names, _is_streaming_request, get_api_key, get_llm_provider, @@ -129,6 +130,74 @@ def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map assert generalized["supports_adaptive_thinking"] is True +def test_potential_model_names_keeps_provider_prefixed_candidate(): + """A provider whose own model ids repeat the litellm provider name (Perplexity's + Agent API serves `perplexity/glm-5.2`, mapped as `perplexity/perplexity/glm-5.2`) + needs the un-stripped `/` candidate. Every other candidate reads + the leading `perplexity/` as the litellm prefix and strips it away.""" + already_prefixed = _get_potential_model_names( + model="perplexity/glm-5.2", custom_llm_provider="perplexity" + ) + assert already_prefixed["provider_prefixed_model_name"] == "perplexity/perplexity/glm-5.2" + assert already_prefixed["split_model"] == "glm-5.2" + assert already_prefixed["combined_model_name"] == "perplexity/glm-5.2" + assert already_prefixed["combined_stripped_model_name"] == "perplexity/glm-5.2" + + bare = _get_potential_model_names(model="glm-5.2", custom_llm_provider="perplexity") + assert bare["provider_prefixed_model_name"] == bare["combined_model_name"] == "perplexity/glm-5.2" + + +def test_get_model_info_resolves_provider_prefixed_model_ids(local_model_cost_map): + """Perplexity's Agent API third-party models are keyed `perplexity/perplexity/` + because Perplexity's own id already starts with `perplexity/`. Callers run + `get_llm_provider` first, which hands `_get_potential_model_names` model + `perplexity/glm-5.2` with provider `perplexity`, and every candidate but the + provider-prefixed one strips that second `perplexity/` off. Regression: the + entries were unreachable from `supports_reasoning` and from the cost calculator's + per-token fallback, so a mapped model reported no reasoning support and raised + "This model isn't mapped yet" on the only path where its rates are ever used.""" + for model, reasoning in ( + ("perplexity/perplexity/glm-5.2", True), + ("perplexity/perplexity/kimi-k3", True), + ("perplexity/perplexity/deepseek-v4-flash-0731", True), + ("perplexity/perplexity/kimi-k2.7-code", False), + ): + assert litellm.supports_reasoning(model=model) is reasoning, model + + via_provider = litellm.get_model_info( + model="perplexity/glm-5.2", custom_llm_provider="perplexity" + ) + assert via_provider["key"] == "perplexity/perplexity/glm-5.2" + assert via_provider["input_cost_per_token"] == 1.4e-06 + assert via_provider["output_cost_per_token"] == 4.4e-06 + assert via_provider["mode"] == "responses" + + +def test_provider_prefixed_lookup_never_outranks_an_existing_row(local_model_cost_map): + """The provider-prefixed candidate is tried last, after every candidate that + already existed, so no model that resolves today can change answer. `perplexity/sonar` + is the case that proves it: both `perplexity/sonar` and `perplexity/perplexity/sonar` + are cost-map keys, and the shorter one must keep winning.""" + sonar = litellm.get_model_info(model="sonar", custom_llm_provider="perplexity") + assert sonar["key"] == "perplexity/sonar" + assert sonar["mode"] == "chat" + assert sonar["input_cost_per_token"] == 1e-06 + + still_sonar = litellm.get_model_info( + model="perplexity/sonar", custom_llm_provider="perplexity" + ) + assert still_sonar["key"] == "perplexity/sonar" + assert still_sonar["mode"] == "chat" + + for model, provider, expected_key in ( + ("claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"), + ("anthropic/claude-sonnet-4-5", "anthropic", "claude-sonnet-4-5"), + ("gemini/gemini-2.0-flash", "gemini", "gemini/gemini-2.0-flash"), + ("openrouter/openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"), + ): + assert litellm.get_model_info(model=model, custom_llm_provider=provider)["key"] == expected_key + + def test_check_provider_match_azure_ai_allows_openai_and_azure(): """ Test that azure_ai provider can match openai and azure models. From a369cb0da7b6581066bd15fd91e169633ede17e8 Mon Sep 17 00:00:00 2001 From: mateo-berri Date: Thu, 20 Aug 2026 13:11:45 -0700 Subject: [PATCH 5/5] fix(bridge): keep the provider's own model prefix on chat-to-responses calls completion() strips the litellm routing prefix before it dispatches to the responses bridge, but responses() runs get_llm_provider() again, so a model id that itself starts with the provider name lost a second prefix and reached the provider as a name it does not know. Handing responses() the prefixed model back makes its own resolve a no-op: across the 3061 cost map entries, 76 reach the responses bridge and only the four perplexity Agent API models change. --- .../handler.py | 26 +++---- ...itellm_responses_transformation_handler.py | 73 +++++++++++++++++++ 2 files changed, 86 insertions(+), 13 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 33206629b41..727c39c16ec 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -25,6 +25,11 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): encoding: object +def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: + """`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from.""" + return f"{custom_llm_provider}/{model}" + + class ResponsesToCompletionBridgeHandler: def __init__(self): from .transformation import LiteLLMResponsesTransformationHandler @@ -184,14 +189,11 @@ class ResponsesToCompletionBridgeHandler: client=kwargs.get("client"), ) - # Pin the resolved provider so `responses()` doesn't re-run - # `get_llm_provider()` on the model string and strip a second - # provider prefix (see GitHub issue #28505). request_data already - # carries `custom_llm_provider` via the spread of - # `sanitized_litellm_params`; overwriting it on the dict (rather - # than adding an explicit kwarg) avoids the duplicate-keyword - # TypeError that would otherwise fire on the real bridge path. + # Set on request_data rather than passed as explicit kwargs: the spread of + # `sanitized_litellm_params` already carries both, so passing them again + # would raise a duplicate-keyword TypeError. request_data["custom_llm_provider"] = custom_llm_provider + request_data["model"] = _restore_routing_prefix(model, custom_llm_provider) result: Final = responses( **request_data, ) @@ -282,13 +284,11 @@ class ResponsesToCompletionBridgeHandler: except Exception as e: raise e - # Pin the resolved provider so `aresponses()` doesn't re-run - # `get_llm_provider()` on the model string and strip a second - # provider prefix (see GitHub issue #28505). Set on request_data - # rather than passed as a separate kwarg to avoid the duplicate- - # keyword TypeError when `sanitized_litellm_params` already - # carries `custom_llm_provider`. + # Set on request_data rather than passed as explicit kwargs: the spread of + # `sanitized_litellm_params` already carries both, so passing them again + # would raise a duplicate-keyword TypeError. request_data["custom_llm_provider"] = custom_llm_provider + request_data["model"] = _restore_routing_prefix(model, custom_llm_provider) result: Final = await aresponses( **request_data, aresponses=True, diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index 8ecb7f4c6f0..42b5ba235bc 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -7,11 +7,13 @@ import pytest sys.path.insert(0, os.path.abspath("../../..")) +import litellm from litellm.completion_extras.litellm_responses_transformation.handler import ( ResponsesToCompletionBridgeHandler, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse @@ -265,3 +267,74 @@ def test_completion_streams_completed_model_response(): assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "pong", ( f"completed response did not stream its content: {chunks}" ) + + +_PROVIDER_NATIVE_MODEL_CASES = [ + ("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"), + ("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"), + ("openai", "gpt-5.4", "gpt-5.4"), +] + + +def _upstream_model_for(handed_model: str, custom_llm_provider: str) -> str: + upstream_model, _, _, _ = litellm.get_llm_provider( + model=handed_model, + litellm_params=GenericLiteLLMParams(custom_llm_provider=custom_llm_provider), + ) + return upstream_model + + +@pytest.mark.parametrize( + "custom_llm_provider, bridge_model, expected_upstream_model", + _PROVIDER_NATIVE_MODEL_CASES, +) +def test_completion_keeps_provider_native_model_id_through_responses( + custom_llm_provider, bridge_model, expected_upstream_model +): + """responses() resolves the provider itself, so the bridge must not hand it an already-stripped model.""" + cached = ModelResponse(id="chatcmpl-cached", model=bridge_model) + bridge = ResponsesToCompletionBridgeHandler() + kwargs = _bridge_kwargs(stream=False) + kwargs["model"] = bridge_model + kwargs["custom_llm_provider"] = custom_llm_provider + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": bridge_model, "input": "hi"}, + ), + patch("litellm.responses", return_value=cached) as responses_call, + ): + bridge.completion(**kwargs) + + handed_model = responses_call.call_args.kwargs["model"] + assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "custom_llm_provider, bridge_model, expected_upstream_model", + _PROVIDER_NATIVE_MODEL_CASES, +) +async def test_acompletion_keeps_provider_native_model_id_through_responses( + custom_llm_provider, bridge_model, expected_upstream_model +): + cached = ModelResponse(id="chatcmpl-cached-async", model=bridge_model) + bridge = ResponsesToCompletionBridgeHandler() + kwargs = _bridge_kwargs(stream=False) + kwargs["model"] = bridge_model + kwargs["custom_llm_provider"] = custom_llm_provider + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": bridge_model, "input": "hi"}, + ), + patch("litellm.aresponses", new=AsyncMock(return_value=cached)) as responses_call, + ): + await bridge.acompletion(**kwargs) + + handed_model = responses_call.call_args.kwargs["model"] + assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model