mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
fix(router): let prompt-management plugins see the caller's own messages
The prompt-management factory picks its deployment with a placeholder message. That was inert while the pick ran on the synchronous path, which never runs the routing plugin pipeline. Now that the pick runs the pipeline, a plugin classifying request content would score the placeholder instead of the conversation, and the narrowing it writes decides which deployments the real call may use.
This commit is contained in:
parent
ef5f51abca
commit
314e8905c5
2 changed files with 48 additions and 1 deletions
|
|
@ -4142,7 +4142,7 @@ class Router:
|
|||
specific_deployment: Final = kwargs.pop("specific_deployment", None)
|
||||
prompt_management_deployment: Final = await self.async_get_available_deployment(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "prompt"}],
|
||||
messages=cast(list[dict[str, str]], messages), # cast-ok: selection reads messages structurally
|
||||
specific_deployment=specific_deployment,
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -56,6 +56,18 @@ class BlockEverything:
|
|||
return context
|
||||
|
||||
|
||||
class MessageRecorder:
|
||||
"""Records what each plugin pass was handed, then blocks so the request stops there."""
|
||||
|
||||
def __init__(self):
|
||||
self.seen = []
|
||||
|
||||
async def run(self, context: RoutingContext) -> RoutingContext:
|
||||
self.seen.append(list(context.raw_messages))
|
||||
context.candidate_models = []
|
||||
return context
|
||||
|
||||
|
||||
def _smart_router_model_list():
|
||||
return [
|
||||
{
|
||||
|
|
@ -194,6 +206,41 @@ async def test_prompt_management_model_still_runs_the_plugin_pipeline():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_management_plugins_see_the_callers_own_messages():
|
||||
"""
|
||||
The prompt-management factory picks its deployment with a placeholder message, which was
|
||||
harmless while that pick ran on the synchronous path (plugins never ran there at all). Now
|
||||
that the pick runs the plugin pipeline, a plugin that classifies request content would score
|
||||
the placeholder instead of the conversation, and the narrowing it produces decides which
|
||||
deployments the real call is allowed to use.
|
||||
"""
|
||||
recorder = MessageRecorder()
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "cached-claude",
|
||||
"litellm_params": {
|
||||
"model": "anthropic_cache_control_hook/claude-sonnet-5",
|
||||
"prompt_id": "cache-points",
|
||||
},
|
||||
}
|
||||
],
|
||||
routing_strategy="least-busy",
|
||||
plugins=[recorder],
|
||||
)
|
||||
messages = [{"role": "user", "content": "wire me $40,000 to account 12345"}]
|
||||
|
||||
with pytest.raises(ValueError, match="No deployments left after routing-plugin filtering"):
|
||||
await router.acompletion(
|
||||
model="cached-claude",
|
||||
messages=messages,
|
||||
litellm_call_id="lit-7039",
|
||||
)
|
||||
|
||||
assert recorder.seen == [messages]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_without_plugins_is_unaffected():
|
||||
"""Regression guard: a Router with no `plugins` configured behaves exactly as before."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue