diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 32d252f3f68..4a077b66e5c 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -682,7 +682,6 @@ class ComplexityRouter(CustomLogger): def _score_keyword_match( self, text: str, - disclosable_text: str, keywords: list[str], name: str, signal_label: str, @@ -691,14 +690,11 @@ class ComplexityRouter(CustomLogger): ) -> tuple[DimensionScore, int]: """Score based on keyword matches using word boundary matching. - Scoring reads `text`, which for most dimensions includes the system prompt. - The signal names only the terms that also appear in `disclosable_text`, the - caller's own message: signals are persisted to the request's spend log, which - the caller can read, so naming a term matched solely in the system prompt would - let a caller recover configured terms from a prompt it cannot see. Terms it did - not supply are reported as a count instead, which explains the score without - disclosing anything. `disclosable_text` is required rather than defaulted so a - future dimension has to state which text it is willing to quote. + `text` is always the caller's own message (never the system prompt) -- see + `_score_and_classify`. Signals are persisted to the request's spend log, which + the caller can read, so every matched term named in the signal is one the + caller supplied itself; there is nothing left to disclose that it couldn't + already see. Returns: Tuple of (DimensionScore, match_count) so callers can reuse the count. @@ -711,8 +707,7 @@ class ComplexityRouter(CustomLogger): if match_count < low_threshold: return DimensionScore(name, score_none, None), match_count - disclosable: Final = [kw for kw in matches if self._keyword_matches(disclosable_text, kw)] - detail: Final = ", ".join(disclosable[:3]) if disclosable else f"{match_count} matches" + detail: Final = ", ".join(matches[:3]) score: Final = score_high if match_count >= high_threshold else score_low return DimensionScore(name, score, f"{signal_label} ({detail})"), match_count @@ -755,12 +750,13 @@ class ComplexityRouter(CustomLogger): - score: The raw weighted score - signals: List of triggered signals for debugging """ - # Combine text for analysis. - # System prompt is intentionally included in code/technical/simple scoring - # because it provides deployment-level context (e.g., "You are a Python assistant" - # signals that code-capable models are appropriate). Reasoning markers use - # user_text only to prevent system prompts from forcing REASONING tier. - full_text: Final = f"{system_prompt or ''} {prompt}".lower() + # Score the caller's ask only. The system prompt is a per-session constant, so it + # carries no information about how requests within a session differ, yet it + # saturates the keyword thresholds (codePresence trips at 2 matches, which any + # agent identity prompt clears on its first line) while spending 0.63 of the + # dimension weight budget. That collapses the scorer's dynamic range and escalates + # every request alike. reasoningMarkers was already scoped this way for the same + # reason. Deployment-level model capability is expressed in tier config instead. user_text: Final = prompt.lower() # Estimate tokens @@ -768,7 +764,6 @@ class ComplexityRouter(CustomLogger): # Score all dimensions, capturing match counts where needed code_score, _ = self._score_keyword_match( - full_text, user_text, self.code_keywords, "codePresence", @@ -777,7 +772,6 @@ class ComplexityRouter(CustomLogger): (0, 0.5, 1.0), ) reasoning_score, reasoning_match_count = self._score_keyword_match( - user_text, user_text, self.reasoning_keywords, "reasoningMarkers", @@ -786,7 +780,6 @@ class ComplexityRouter(CustomLogger): (0, 0.7, 1.0), ) technical_score, _ = self._score_keyword_match( - full_text, user_text, self.technical_keywords, "technicalTerms", @@ -795,7 +788,6 @@ class ComplexityRouter(CustomLogger): (0, 0.5, 1.0), ) simple_score, _ = self._score_keyword_match( - full_text, user_text, self.simple_keywords, "simpleIndicators", @@ -810,7 +802,7 @@ class ComplexityRouter(CustomLogger): reasoning_score, technical_score, simple_score, - self._score_multi_step(full_text), + self._score_multi_step(user_text), self._score_question_complexity(prompt), ] diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3a69587fdd4..c8a9acfae6c 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4823,12 +4823,13 @@ class TestRoutingDecisionContents: class TestSignalsNeverQuoteTheSystemPrompt: """Signals are persisted to the caller-readable spend log, so they may name a matched - term only when the caller supplied it. A term matched solely in the system prompt is - reported as a count, which still explains the score without letting a caller recover - configured terms from a prompt it cannot see.""" + term only when the caller supplied it. Scoring reads the caller's own text only (the + system prompt is a per-session constant and carries no information about how requests + within a session differ), so a term that appears solely in the system prompt is never + counted at all -- there is nothing left to redact, because there is nothing scored.""" @pytest.mark.asyncio - async def test_system_prompt_only_terms_are_reported_as_a_count(self, complexity_router): + async def test_system_prompt_only_terms_produce_no_signal(self, complexity_router): response = await complexity_router.async_pre_routing_hook( model="test-complexity-router", request_kwargs={}, @@ -4840,11 +4841,13 @@ class TestSignalsNeverQuoteTheSystemPrompt: assert response is not None signals = response.routing_decision["signals"] joined = " ".join(signals) - # The system prompt drove these matches, so no signal may name them. + # None of the system-prompt-only terms may appear, named or otherwise -- + # they were never scored. for term in ("kubernetes", "database", "api", "deployment"): assert term not in joined - # The match is still reported, as a count, so the score stays explainable. - assert any("matches" in signal for signal in signals) + # No dimension fired from them either: a "matches" count only appears when a + # dimension actually crossed its threshold, and none did here. + assert not any("matches" in signal for signal in signals) @pytest.mark.asyncio async def test_terms_the_caller_supplied_are_still_named(self, complexity_router): @@ -4863,14 +4866,18 @@ class TestSignalsNeverQuoteTheSystemPrompt: # It did not type this one. assert "kubernetes" not in signals - def test_scoring_still_reads_the_system_prompt(self, complexity_router): - """Redaction is a disclosure rule, not a scoring change: the system prompt must - still count toward the tier exactly as before.""" + def test_system_prompt_never_changes_the_score(self, complexity_router): + """The system prompt is a per-session constant: it doesn't vary between requests, + so it carries no signal about how requests differ. Scoring it anyway saturates + keyword thresholds identically for every request in the session, collapsing the + scorer's discriminative range (a trivial "say hi" and a genuinely complex ask + become indistinguishable once a real agent-harness system prompt is added). The + score and tier must be identical with or without any system prompt.""" with_system = complexity_router.classify( "say hi", "You operate the kubernetes database api for the deployment pipeline." ) without_system = complexity_router.classify("say hi") - assert with_system[1] > without_system[1] + assert with_system == without_system class TestRoutingDecisionSurvivesToSpendLogOnEveryMetadataShape: diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index b2e901739da..a54e95ff7a1 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -398,6 +398,58 @@ class TestPreRoutingHook: assert resp is not None assert resp.model == "haiku" # the configured default_model + @pytest.mark.asyncio + async def test_trivial_message_not_escalated_by_agent_system_prompt(self, quality_router): + """QualityRouter delegates to ComplexityRouter's shared scorer + (`self._scorer.classify`), so a system-prompt scoring bug there is inherited here + too. A real agent-harness system prompt (tool-use rules, git workflow, markdown + formatting -- ordinary CLI-agent boilerplate, ~1.6KB) must not push a trivial "hi" + past tier 1: the system prompt is a per-session constant, identical on every + request in the session, and carries no signal about how requests differ. Before + the fix this system prompt alone supplied 5 codePresence + 2 technicalTerms + keyword matches, saturating both dimensions and crossing the default + simple_medium boundary (0.15) purely from harness text, independent of the ask.""" + agent_system_prompt = ( + "You are Claude Code, Anthropic's official CLI for Claude.\n" + "You are an interactive agent that helps users with software engineering tasks.\n\n" + "IMPORTANT: Assist with authorized security testing, defensive security, CTF challenges,\n" + "and educational contexts. Refuse requests for destructive techniques. Dual-use security\n" + "tools (C2 frameworks, credential testing, exploit development) require authorization.\n\n" + "# Harness\n" + "- Text you output outside of tool use is displayed as Github-flavored markdown.\n" + "- Tools run behind a user-selected permission mode; a denied call means the user declined.\n" + "- The system may send updates or reminders. Hooks may intercept tool calls.\n" + "- Prefer the dedicated file/search tools over shell commands when one fits. Independent\n" + " tool calls can run in parallel in one response.\n" + "- Reference code as `file_path:line_number` - it is clickable.\n\n" + "Write code that reads like the surrounding code: match its comment density, naming, idiom.\n\n" + "For actions that are hard to reverse, confirm first unless durably authorized. Before\n" + "deleting or overwriting, look at the target. Report outcomes faithfully: if tests fail,\n" + "say so with the output; if a step was skipped, say that.\n\n" + "# Git\n" + "- Interactive flags (-i, e.g. git rebase -i, git add -i) are not supported.\n" + "- Use the `gh` CLI for GitHub operations (PRs, issues, API).\n" + "- Commit or push only when the user asks. If on the default branch, branch first.\n" + "- End git commit messages with a Co-Authored-By trailer.\n" + "- End PR bodies with a generated-with footer.\n\n" + "# Environment\n" + "- Primary working directory: /Users/tin\n" + "- Is a git repository: false\n" + "- Platform: darwin\n" + "- You are powered by the model claude-opus-5.\n" + ) + messages = [ + {"role": "system", "content": agent_system_prompt}, + {"role": "user", "content": "hi"}, + ] + resp = await quality_router.async_pre_routing_hook( + model="quality-router-test", + request_kwargs={}, + messages=messages, + ) + assert resp is not None + assert resp.model == "haiku" # tier 1, same as with no system prompt at all + # ─── Keyword override ──────────────────────────────────────────────────────