mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
fix(bridge): keep the provider's own model prefix on chat-to-responses calls
completion() strips the litellm routing prefix before it dispatches to the responses bridge, but responses() runs get_llm_provider() again, so a model id that itself starts with the provider name lost a second prefix and reached the provider as a name it does not know. Handing responses() the prefixed model back makes its own resolve a no-op: across the 3061 cost map entries, 76 reach the responses bridge and only the four perplexity Agent API models change.
This commit is contained in:
parent
88ef47377f
commit
a369cb0da7
2 changed files with 86 additions and 13 deletions
|
|
@ -25,6 +25,11 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
|
||||||
encoding: object
|
encoding: object
|
||||||
|
|
||||||
|
|
||||||
|
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
|
||||||
|
"""`responses()` runs `get_llm_provider()` itself, so hand back the prefixed model `completion()` started from."""
|
||||||
|
return f"{custom_llm_provider}/{model}"
|
||||||
|
|
||||||
|
|
||||||
class ResponsesToCompletionBridgeHandler:
|
class ResponsesToCompletionBridgeHandler:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
from .transformation import LiteLLMResponsesTransformationHandler
|
from .transformation import LiteLLMResponsesTransformationHandler
|
||||||
|
|
@ -184,14 +189,11 @@ class ResponsesToCompletionBridgeHandler:
|
||||||
client=kwargs.get("client"),
|
client=kwargs.get("client"),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Pin the resolved provider so `responses()` doesn't re-run
|
# Set on request_data rather than passed as explicit kwargs: the spread of
|
||||||
# `get_llm_provider()` on the model string and strip a second
|
# `sanitized_litellm_params` already carries both, so passing them again
|
||||||
# provider prefix (see GitHub issue #28505). request_data already
|
# would raise a duplicate-keyword TypeError.
|
||||||
# carries `custom_llm_provider` via the spread of
|
|
||||||
# `sanitized_litellm_params`; overwriting it on the dict (rather
|
|
||||||
# than adding an explicit kwarg) avoids the duplicate-keyword
|
|
||||||
# TypeError that would otherwise fire on the real bridge path.
|
|
||||||
request_data["custom_llm_provider"] = custom_llm_provider
|
request_data["custom_llm_provider"] = custom_llm_provider
|
||||||
|
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
|
||||||
result: Final = responses(
|
result: Final = responses(
|
||||||
**request_data,
|
**request_data,
|
||||||
)
|
)
|
||||||
|
|
@ -282,13 +284,11 @@ class ResponsesToCompletionBridgeHandler:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
# Pin the resolved provider so `aresponses()` doesn't re-run
|
# Set on request_data rather than passed as explicit kwargs: the spread of
|
||||||
# `get_llm_provider()` on the model string and strip a second
|
# `sanitized_litellm_params` already carries both, so passing them again
|
||||||
# provider prefix (see GitHub issue #28505). Set on request_data
|
# would raise a duplicate-keyword TypeError.
|
||||||
# rather than passed as a separate kwarg to avoid the duplicate-
|
|
||||||
# keyword TypeError when `sanitized_litellm_params` already
|
|
||||||
# carries `custom_llm_provider`.
|
|
||||||
request_data["custom_llm_provider"] = custom_llm_provider
|
request_data["custom_llm_provider"] = custom_llm_provider
|
||||||
|
request_data["model"] = _restore_routing_prefix(model, custom_llm_provider)
|
||||||
result: Final = await aresponses(
|
result: Final = await aresponses(
|
||||||
**request_data,
|
**request_data,
|
||||||
aresponses=True,
|
aresponses=True,
|
||||||
|
|
|
||||||
|
|
@ -7,11 +7,13 @@ import pytest
|
||||||
|
|
||||||
sys.path.insert(0, os.path.abspath("../../.."))
|
sys.path.insert(0, os.path.abspath("../../.."))
|
||||||
|
|
||||||
|
import litellm
|
||||||
from litellm.completion_extras.litellm_responses_transformation.handler import (
|
from litellm.completion_extras.litellm_responses_transformation.handler import (
|
||||||
ResponsesToCompletionBridgeHandler,
|
ResponsesToCompletionBridgeHandler,
|
||||||
)
|
)
|
||||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||||
|
from litellm.types.router import GenericLiteLLMParams
|
||||||
from litellm.types.utils import ModelResponse
|
from litellm.types.utils import ModelResponse
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -265,3 +267,74 @@ def test_completion_streams_completed_model_response():
|
||||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "pong", (
|
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == "pong", (
|
||||||
f"completed response did not stream its content: {chunks}"
|
f"completed response did not stream its content: {chunks}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_PROVIDER_NATIVE_MODEL_CASES = [
|
||||||
|
("perplexity", "perplexity/kimi-k3", "perplexity/kimi-k3"),
|
||||||
|
("perplexity", "openai/gpt-5.2", "openai/gpt-5.2"),
|
||||||
|
("openai", "gpt-5.4", "gpt-5.4"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _upstream_model_for(handed_model: str, custom_llm_provider: str) -> str:
|
||||||
|
upstream_model, _, _, _ = litellm.get_llm_provider(
|
||||||
|
model=handed_model,
|
||||||
|
litellm_params=GenericLiteLLMParams(custom_llm_provider=custom_llm_provider),
|
||||||
|
)
|
||||||
|
return upstream_model
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"custom_llm_provider, bridge_model, expected_upstream_model",
|
||||||
|
_PROVIDER_NATIVE_MODEL_CASES,
|
||||||
|
)
|
||||||
|
def test_completion_keeps_provider_native_model_id_through_responses(
|
||||||
|
custom_llm_provider, bridge_model, expected_upstream_model
|
||||||
|
):
|
||||||
|
"""responses() resolves the provider itself, so the bridge must not hand it an already-stripped model."""
|
||||||
|
cached = ModelResponse(id="chatcmpl-cached", model=bridge_model)
|
||||||
|
bridge = ResponsesToCompletionBridgeHandler()
|
||||||
|
kwargs = _bridge_kwargs(stream=False)
|
||||||
|
kwargs["model"] = bridge_model
|
||||||
|
kwargs["custom_llm_provider"] = custom_llm_provider
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
bridge.transformation_handler,
|
||||||
|
"transform_request",
|
||||||
|
return_value={"model": bridge_model, "input": "hi"},
|
||||||
|
),
|
||||||
|
patch("litellm.responses", return_value=cached) as responses_call,
|
||||||
|
):
|
||||||
|
bridge.completion(**kwargs)
|
||||||
|
|
||||||
|
handed_model = responses_call.call_args.kwargs["model"]
|
||||||
|
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"custom_llm_provider, bridge_model, expected_upstream_model",
|
||||||
|
_PROVIDER_NATIVE_MODEL_CASES,
|
||||||
|
)
|
||||||
|
async def test_acompletion_keeps_provider_native_model_id_through_responses(
|
||||||
|
custom_llm_provider, bridge_model, expected_upstream_model
|
||||||
|
):
|
||||||
|
cached = ModelResponse(id="chatcmpl-cached-async", model=bridge_model)
|
||||||
|
bridge = ResponsesToCompletionBridgeHandler()
|
||||||
|
kwargs = _bridge_kwargs(stream=False)
|
||||||
|
kwargs["model"] = bridge_model
|
||||||
|
kwargs["custom_llm_provider"] = custom_llm_provider
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(
|
||||||
|
bridge.transformation_handler,
|
||||||
|
"transform_request",
|
||||||
|
return_value={"model": bridge_model, "input": "hi"},
|
||||||
|
),
|
||||||
|
patch("litellm.aresponses", new=AsyncMock(return_value=cached)) as responses_call,
|
||||||
|
):
|
||||||
|
await bridge.acompletion(**kwargs)
|
||||||
|
|
||||||
|
handed_model = responses_call.call_args.kwargs["model"]
|
||||||
|
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue