mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(router): route Responses API input through the auto-router
Auto-router strategy hook returned None whenever the request carried input instead of messages, so tagged /v1/responses requests (Codex CLI) never picked a tier and tag filtering left nothing to route to. Resolve input through the shared prompt-template helper before matching routes.
This commit is contained in:
parent
9ec0145986
commit
dd18389365
2 changed files with 71 additions and 3 deletions
|
|
@ -128,13 +128,14 @@ class AutoRouter(CustomLogger):
|
|||
"""
|
||||
from semantic_router.routers import SemanticRouter
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
|
||||
from litellm.router_strategy.auto_router.litellm_encoder import (
|
||||
LiteLLMRouterEncoder,
|
||||
)
|
||||
from litellm.types.router import PreRoutingHookResponse
|
||||
|
||||
if messages is None:
|
||||
# do nothing, return same inputs
|
||||
resolved_messages: Final = resolve_structured_messages(messages=messages, request_kwargs=request_kwargs)
|
||||
if resolved_messages is None:
|
||||
return None
|
||||
|
||||
routelayer = self.routelayer
|
||||
|
|
@ -153,7 +154,7 @@ class AutoRouter(CustomLogger):
|
|||
)
|
||||
self.routelayer = routelayer
|
||||
|
||||
message_content: Final = self._extract_text_from_messages(messages)
|
||||
message_content: Final = self._extract_text_from_messages(resolved_messages)
|
||||
route_name: Final = self._matched_route_name(routelayer, message_content)
|
||||
|
||||
return PreRoutingHookResponse(
|
||||
|
|
|
|||
|
|
@ -479,3 +479,70 @@ class TestAutoRouterEmbeddingInputCap:
|
|||
|
||||
assert auto_router.routelayer is not None
|
||||
assert auto_router.routelayer.encoder.max_input_chars == 777
|
||||
|
||||
|
||||
class TestAutoRouterRoutesResponsesApiInput:
|
||||
"""Responses API requests carry the prompt in `input`, not `messages`, and still have to reach the route layer."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_route_a_string_input_when_messages_is_none(self):
|
||||
from semantic_router.schema import RouteChoice
|
||||
|
||||
layer: Final = FixedRouteLayer(RouteChoice(name="code-model"))
|
||||
auto_router: Final = _auto_router(layer)
|
||||
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router",
|
||||
request_kwargs={
|
||||
"input": "fix this stack trace",
|
||||
"litellm_metadata": {"user_api_key_request_route": "/v1/responses"},
|
||||
},
|
||||
messages=None,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "code-model"
|
||||
assert result.messages is None
|
||||
assert layer.seen_text == "fix this stack trace"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_route_a_list_input_with_instructions_when_messages_is_none(self):
|
||||
from semantic_router.schema import RouteChoice
|
||||
|
||||
layer: Final = FixedRouteLayer(RouteChoice(name="code-model"))
|
||||
auto_router: Final = _auto_router(layer)
|
||||
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router",
|
||||
request_kwargs={
|
||||
"instructions": "You are a coding agent.",
|
||||
"input": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "fix this stack trace"}],
|
||||
}
|
||||
],
|
||||
"litellm_metadata": {"user_api_key_request_route": "/v1/responses"},
|
||||
},
|
||||
messages=None,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.model == "code-model"
|
||||
assert layer.seen_text is not None
|
||||
assert "fix this stack trace" in layer.seen_text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_skip_routing_when_neither_messages_nor_input_is_present(self):
|
||||
layer: Final = FixedRouteLayer(None)
|
||||
auto_router: Final = _auto_router(layer)
|
||||
|
||||
result: Final = await auto_router.async_pre_routing_hook(
|
||||
model="my-auto-router",
|
||||
request_kwargs={"litellm_metadata": {"user_api_key_request_route": "/v1/responses"}},
|
||||
messages=None,
|
||||
)
|
||||
|
||||
assert result is None
|
||||
assert layer.seen_text is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue