fix(router): let prompt-management plugins see the caller's own messages

The prompt-management factory picks its deployment with a placeholder message. That
was inert while the pick ran on the synchronous path, which never runs the routing
plugin pipeline. Now that the pick runs the pipeline, a plugin classifying request
content would score the placeholder instead of the conversation, and the narrowing
it writes decides which deployments the real call may use.
This commit is contained in:
mateo-berri 2026-09-06 02:44:45 -07:00
parent ef5f51abca
commit 314e8905c5
2 changed files with 48 additions and 1 deletions

View file

@ -4142,7 +4142,7 @@ class Router:
specific_deployment: Final = kwargs.pop("specific_deployment", None)
prompt_management_deployment: Final = await self.async_get_available_deployment(
model=model,
messages=[{"role": "user", "content": "prompt"}],
messages=cast(list[dict[str, str]], messages), # cast-ok: selection reads messages structurally
specific_deployment=specific_deployment,
request_kwargs=kwargs,
)

View file

@ -56,6 +56,18 @@ class BlockEverything:
return context
class MessageRecorder:
"""Records what each plugin pass was handed, then blocks so the request stops there."""
def __init__(self):
self.seen = []
async def run(self, context: RoutingContext) -> RoutingContext:
self.seen.append(list(context.raw_messages))
context.candidate_models = []
return context
def _smart_router_model_list():
return [
{
@ -194,6 +206,41 @@ async def test_prompt_management_model_still_runs_the_plugin_pipeline():
)
@pytest.mark.asyncio
async def test_prompt_management_plugins_see_the_callers_own_messages():
"""
The prompt-management factory picks its deployment with a placeholder message, which was
harmless while that pick ran on the synchronous path (plugins never ran there at all). Now
that the pick runs the plugin pipeline, a plugin that classifies request content would score
the placeholder instead of the conversation, and the narrowing it produces decides which
deployments the real call is allowed to use.
"""
recorder = MessageRecorder()
router = Router(
model_list=[
{
"model_name": "cached-claude",
"litellm_params": {
"model": "anthropic_cache_control_hook/claude-sonnet-5",
"prompt_id": "cache-points",
},
}
],
routing_strategy="least-busy",
plugins=[recorder],
)
messages = [{"role": "user", "content": "wire me $40,000 to account 12345"}]
with pytest.raises(ValueError, match="No deployments left after routing-plugin filtering"):
await router.acompletion(
model="cached-claude",
messages=messages,
litellm_call_id="lit-7039",
)
assert recorder.seen == [messages]
@pytest.mark.asyncio
async def test_router_without_plugins_is_unaffected():
"""Regression guard: a Router with no `plugins` configured behaves exactly as before."""