mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(bridge): keep the provider's own model prefix on chat-to-responses calls
completion() strips the litellm routing prefix before it dispatches to the responses bridge, but responses() runs get_llm_provider() again, so a model id that itself starts with the provider name lost a second prefix and reached the provider as a name it does not know. Handing responses() the prefixed model back makes its own resolve a no-op: across the 3061 cost map entries, 76 reach the responses bridge and only the four perplexity Agent API models change.
This commit is contained in:
parent
88ef47377f
commit
a369cb0da7
2 changed files with 86 additions and 13 deletions
|
|
@ -25,6 +25,11 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
|
|||
encoding: object
|
||||
|
||||
|
||||
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
|
||||
"""`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from."""
|
||||
return f"{custom_llm_provider}/{model}"
|
||||
|
||||
|
||||
class ResponsesToCompletionBridgeHandler:
|
||||
def __init__(self):
|
||||
from .transformation import LiteLLMResponsesTransformationHandler
|
||||
|
|
@ -184,14 +189,11 @@ class ResponsesToCompletionBridgeHandler:
|
|||
client=kwargs.get("client"),
|
||||
)
|
||||
|
||||
# Pin the resolved provider so `responses()` doesn't re-run
|
||||
# `get_llm_provider()` on the model string and strip a second
|
||||
# provider prefix (see GitHub issue #28505). request_data already
|
||||
# carries `custom_llm_provider` via the spread of
|
||||
# `sanitized_litellm_params`; overwriting it on the dict (rather
|
||||
# than adding an explicit kwarg) avoids the duplicate-keyword
|
||||
# TypeError that would otherwise fire on the real bridge path.
|
||||
# Set on request_data rather than passed as explicit kwargs: the spread of
|
||||
# `sanitized_litellm_params` already carries both, so passing them again
|
||||
# would raise a duplicate-keyword TypeError.
|
||||
request_data["custom_llm_provider"] = custom_llm_provider
|
||||
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
|
||||
result: Final = responses(
|
||||
**request_data,
|
||||
)
|
||||
|
|
@ -282,13 +284,11 @@ class ResponsesToCompletionBridgeHandler:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
# Pin the resolved provider so `aresponses()` doesn't re-run
|
||||
# `get_llm_provider()` on the model string and strip a second
|
||||
# provider prefix (see GitHub issue #28505). Set on request_data
|
||||
# rather than passed as a separate kwarg to avoid the duplicate-
|
||||
# keyword TypeError when `sanitized_litellm_params` already
|
||||
# carries `custom_llm_provider`.
|
||||
# Set on request_data rather than passed as explicit kwargs: the spread of
|
||||
# `sanitized_litellm_params` already carries both, so passing them again
|
||||
# would raise a duplicate-keyword TypeError.
|
||||
request_data["custom_llm_provider"] = custom_llm_provider
|
||||
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
|
||||
result: Final = await aresponses(
|
||||
**request_data,
|
||||
aresponses=True,
|
||||
|
|
|
|||
|
|
@ -7,11 +7,13 @@ import pytest
|
|||
|
||||
sys.path.insert(0, os.path.abspath("../../.."))
|
||||
|
||||
import litellm
|
||||
from litellm.completion_extras.litellm_responses_transformation.handler import (
|
||||
ResponsesToCompletionBridgeHandler,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
||||
|
|
@ -265,3 +267,74 @@ def test_completion_streams_completed_model_response():
|
|||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "pong", (
|
||||
f"completed response did not stream its content: {chunks}"
|
||||
)
|
||||
|
||||
|
||||
_PROVIDER_NATIVE_MODEL_CASES = [
|
||||
("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"),
|
||||
("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"),
|
||||
("openai", "gpt-5.4", "gpt-5.4"),
|
||||
]
|
||||
|
||||
|
||||
def _upstream_model_for(handed_model: str, custom_llm_provider: str) -> str:
|
||||
upstream_model, _, _, _ = litellm.get_llm_provider(
|
||||
model=handed_model,
|
||||
litellm_params=GenericLiteLLMParams(custom_llm_provider=custom_llm_provider),
|
||||
)
|
||||
return upstream_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider, bridge_model, expected_upstream_model",
|
||||
_PROVIDER_NATIVE_MODEL_CASES,
|
||||
)
|
||||
def test_completion_keeps_provider_native_model_id_through_responses(
|
||||
custom_llm_provider, bridge_model, expected_upstream_model
|
||||
):
|
||||
"""responses() resolves the provider itself, so the bridge must not hand it an already-stripped model."""
|
||||
cached = ModelResponse(id="chatcmpl-cached", model=bridge_model)
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
kwargs = _bridge_kwargs(stream=False)
|
||||
kwargs["model"] = bridge_model
|
||||
kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
bridge.transformation_handler,
|
||||
"transform_request",
|
||||
return_value={"model": bridge_model, "input": "hi"},
|
||||
),
|
||||
patch("litellm.responses", return_value=cached) as responses_call,
|
||||
):
|
||||
bridge.completion(**kwargs)
|
||||
|
||||
handed_model = responses_call.call_args.kwargs["model"]
|
||||
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider, bridge_model, expected_upstream_model",
|
||||
_PROVIDER_NATIVE_MODEL_CASES,
|
||||
)
|
||||
async def test_acompletion_keeps_provider_native_model_id_through_responses(
|
||||
custom_llm_provider, bridge_model, expected_upstream_model
|
||||
):
|
||||
cached = ModelResponse(id="chatcmpl-cached-async", model=bridge_model)
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
kwargs = _bridge_kwargs(stream=False)
|
||||
kwargs["model"] = bridge_model
|
||||
kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
bridge.transformation_handler,
|
||||
"transform_request",
|
||||
return_value={"model": bridge_model, "input": "hi"},
|
||||
),
|
||||
patch("litellm.aresponses", new=AsyncMock(return_value=cached)) as responses_call,
|
||||
):
|
||||
await bridge.acompletion(**kwargs)
|
||||
|
||||
handed_model = responses_call.call_args.kwargs["model"]
|
||||
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue