mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
chore(ui): sync API descriptions with the current default branch
This commit is contained in:
commit
5735587133
5 changed files with 156 additions and 11 deletions
|
|
@ -455,6 +455,13 @@ If 2+ reasoning markers are detected in the user message, the request is promote
|
|||
|
||||
Reasoning markers in the system prompt do **not** trigger the reasoning override. This prevents system prompts like "Think step by step before answering" from forcing all requests to the reasoning tier.
|
||||
|
||||
For requests identified by a `claude-cli/` or `claude-code/` user agent, the LLM classifier omits caller system
|
||||
text to avoid classifying environment, agent, and skill catalogs. The current ask, configured prior-turn context,
|
||||
and trajectory signal remain unchanged. The routed completion still receives the original system text. This
|
||||
also excludes genuine task constraints supplied only in Claude Code system messages. Other clients keep the
|
||||
existing system-context behavior. The browser routing preview has no client-identity field and retains that
|
||||
generic behavior; use the real client when checking Claude Code routing.
|
||||
|
||||
### Harness Reminder Blocks
|
||||
|
||||
Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
request_contains_image_content,
|
||||
)
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
|
||||
from litellm.llms.anthropic.common_utils import is_claude_code_user_agent
|
||||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
|
||||
from litellm.router_strategy.complexity_router.tier_predictor import (
|
||||
|
|
@ -1993,8 +1994,7 @@ class ComplexityRouter(CustomLogger):
|
|||
|
||||
Args:
|
||||
prompt: The current user ask text (already extracted as the real human ask, not tool results)
|
||||
system_prompt: The caller's system prompt (task constraints), always included so later
|
||||
turns never lose it
|
||||
system_prompt: Caller task constraints, omitted from classification for Claude Code requests
|
||||
request_kwargs: Request metadata for spend attribution
|
||||
messages: Full message history for extracting prior turns and the trajectory signal
|
||||
"""
|
||||
|
|
@ -2027,9 +2027,18 @@ class ComplexityRouter(CustomLogger):
|
|||
)
|
||||
|
||||
encrypted_task: Final = _encrypted_classifier_task(request_kwargs, marker_pairs)
|
||||
caller_system_prompt: Final = (
|
||||
None
|
||||
if any(
|
||||
is_claude_code_user_agent(user_agent)
|
||||
for metadata in (self._iter_metadata_dicts(request_kwargs) if request_kwargs is not None else ())
|
||||
if isinstance(user_agent := metadata.get("user_agent"), str)
|
||||
)
|
||||
else system_prompt
|
||||
)
|
||||
user_payload: Final = self._build_classifier_user_payload(
|
||||
prompt="The delegated task in the following agent_message." if encrypted_task is not None else prompt,
|
||||
system_prompt=system_prompt,
|
||||
system_prompt=caller_system_prompt,
|
||||
prior_turns=prior_turns,
|
||||
messages=messages,
|
||||
has_prior_conversation=has_prior_conversation,
|
||||
|
|
|
|||
|
|
@ -971,9 +971,11 @@ class ComplexityRouterConfig(BaseModel):
|
|||
"classified against what it refers to. Counts turns of both roles when "
|
||||
"classifier_context_include_assistant_turns is enabled. These turns are sent to the classifier "
|
||||
"model, which may "
|
||||
"be a different deployment or provider than the routed completion model; that call already "
|
||||
"carries the current user ask and the caller's system prompt in full. Set to 0 to send neither "
|
||||
"prior turns nor any conversation context beyond the current ask. Only applies when "
|
||||
"be a different deployment or provider than the routed completion model; that call carries "
|
||||
"the current user ask and, except for Claude Code requests, the extracted system-role text in full. "
|
||||
"Claude Code system text is omitted to avoid classifying harness instructions; the routed "
|
||||
"completion still receives it. Set to 0 to send neither prior turns nor "
|
||||
"any conversation context beyond the current ask. Only applies when "
|
||||
"classifier_type is 'llm'."
|
||||
),
|
||||
)
|
||||
|
|
@ -985,9 +987,9 @@ class ComplexityRouterConfig(BaseModel):
|
|||
"context window, per classification call. Turns are taken newest first and quoted whole "
|
||||
"while they fit, so a conversation small enough to quote entirely is never cut; once the "
|
||||
"budget runs out the older turns are dropped whole and only the turn straddling the "
|
||||
"boundary is truncated, into whatever space is left. The current ask and the caller's "
|
||||
"system prompt sit outside this budget and are always sent in full, as does the numbering "
|
||||
"each quoted turn carries. A budget under 120 leaves no room to quote a turn and "
|
||||
"boundary is truncated, into whatever space is left. The current ask and, except for Claude "
|
||||
"Code requests, the extracted system-role text sit outside this budget and are sent in full, as does "
|
||||
"the numbering each quoted turn carries. A budget under 120 leaves no room to quote a turn and "
|
||||
"suppresses the block; set classifier_context_window_size to 0 to turn context off "
|
||||
"deliberately. Only applies when classifier_type is 'llm'."
|
||||
),
|
||||
|
|
|
|||
|
|
@ -2676,6 +2676,26 @@ class TestEncryptedTaskClassifier:
|
|||
assert "source-secret" not in json.dumps(call)
|
||||
assert "originating_request_masked" not in call["proxy_server_request"]["body"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_code_encrypted_task_omits_caller_instructions(self):
|
||||
router, dependency = _native_classifier_router()
|
||||
task: Final = _encrypted_agent_task()
|
||||
request: Final = {
|
||||
"input": [task],
|
||||
"instructions": "CLAUDE_CODE_SYSTEM",
|
||||
"litellm_metadata": {"user_agent": "claude-cli/2.1.233"},
|
||||
}
|
||||
original: Final = deepcopy(request)
|
||||
|
||||
result: Final = await router.async_pre_routing_hook(model="encrypted-router", request_kwargs=request)
|
||||
|
||||
assert result.routing_decision["cause"] == "llm_classifier"
|
||||
assert request == original
|
||||
call: Final = dependency.aresponses.call_args.kwargs
|
||||
assert call["instructions"] == classification_system_prompt(router.config.classifier_context_window_size)
|
||||
assert "CLAUDE_CODE_SYSTEM" not in json.dumps(call["input"][:-1])
|
||||
assert call["input"][-1] == task
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"items",
|
||||
|
|
@ -7994,6 +8014,113 @@ _CODEX_ENVELOPES: Final = (
|
|||
class TestContextAwareClassifier:
|
||||
"""Test the new classifier context window and trajectory signals."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"request_metadata,forwards_system",
|
||||
[
|
||||
({"metadata": {"user_agent": "claude-cli/2.1.233"}}, False),
|
||||
({"litellm_metadata": {"user_agent": "claude-code/2.1.233"}}, False),
|
||||
({"metadata": {"user_agent": "curl/8.7.1"}}, True),
|
||||
({"litellm_metadata": {}}, True),
|
||||
(
|
||||
{"metadata": {"user_agent": "claude-cli/2.1.233"}, "litellm_metadata": {"user_agent": "curl/8.7.1"}},
|
||||
False,
|
||||
),
|
||||
({"metadata": {"user_agent": "Claude-Code/2.1.233"}}, True),
|
||||
],
|
||||
)
|
||||
async def test_claude_code_classifier_omits_harness_system_prompt(
|
||||
self,
|
||||
llm_classifier_config: dict[str, object],
|
||||
request_metadata: dict[str, object],
|
||||
forwards_system: bool,
|
||||
) -> None:
|
||||
dependency: Final = MagicMock(acompletion=AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}')))
|
||||
router: Final = ComplexityRouter(
|
||||
"test-complexity-router",
|
||||
dependency,
|
||||
{
|
||||
**llm_classifier_config,
|
||||
"classifier_context_include_assistant_turns": True,
|
||||
},
|
||||
)
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Design the retry state machine"},
|
||||
{"role": "assistant", "content": "The design needs a lease and fencing token"},
|
||||
{"role": "user", "content": "Now prove it cannot livelock"},
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "ENVIRONMENT_CATALOG\nAGENT_CATALOG\nSKILL_CATALOG"}],
|
||||
},
|
||||
]
|
||||
top_level_system: Final = [{"type": "text", "text": "TOP_LEVEL_HARNESS_SYSTEM"}]
|
||||
claude_kwargs: Final = {
|
||||
"metadata": {"user_agent": "claude-cli/2.1.233"},
|
||||
"system": top_level_system,
|
||||
"proxy_server_request": {"body": {"system": top_level_system}},
|
||||
}
|
||||
compared_kwargs: Final = {
|
||||
**request_metadata,
|
||||
"system": top_level_system,
|
||||
"proxy_server_request": {"body": {"system": top_level_system}},
|
||||
}
|
||||
original_messages: Final = deepcopy(messages)
|
||||
original_kwargs: Final = deepcopy((claude_kwargs, compared_kwargs))
|
||||
results: Final = (
|
||||
await router.async_pre_routing_hook("test-complexity-router", claude_kwargs, messages),
|
||||
await router.async_pre_routing_hook("test-complexity-router", compared_kwargs, messages),
|
||||
)
|
||||
|
||||
assert all(result is not None and result.routing_decision["cause"] == "llm_classifier" for result in results)
|
||||
assert all(result is not None and result.messages == original_messages for result in results)
|
||||
assert messages == original_messages
|
||||
assert (claude_kwargs, compared_kwargs) == original_kwargs
|
||||
calls: Final = tuple(call.kwargs["messages"] for call in dependency.acompletion.await_args_list)
|
||||
assert calls[0][0]["content"] == calls[1][0]["content"] == classification_system_prompt(
|
||||
router.config.classifier_context_window_size
|
||||
)
|
||||
payloads: Final = (calls[0][1]["content"], calls[1][1]["content"])
|
||||
for payload, expected_system in zip(payloads, (False, forwards_system)):
|
||||
assert payload.endswith("Classify this message:\nNow prove it cannot livelock")
|
||||
assert ("ENVIRONMENT_CATALOG" in payload) is expected_system
|
||||
assert ("AGENT_CATALOG" in payload) is expected_system
|
||||
assert ("SKILL_CATALOG" in payload) is expected_system
|
||||
assert "Design the retry state machine" in payload
|
||||
assert "lease and fencing token" in payload
|
||||
assert "TOP_LEVEL_HARNESS_SYSTEM" not in payload
|
||||
assert "Conversation so far: ~35 tokens across the request" in payload
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_claude_code_first_turn_without_context_omits_harness_system_prompt(
|
||||
self, llm_classifier_config: dict[str, object]
|
||||
) -> None:
|
||||
dependency: Final = MagicMock(acompletion=AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')))
|
||||
router: Final = ComplexityRouter(
|
||||
"test-complexity-router",
|
||||
dependency,
|
||||
{**llm_classifier_config, "classifier_context_window_size": 0},
|
||||
)
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "What is two plus two?"},
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "ENVIRONMENT_CATALOG\nAGENT_CATALOG\nSKILL_CATALOG"}],
|
||||
},
|
||||
]
|
||||
request_kwargs: Final = {"litellm_metadata": {"user_agent": "claude-code/2.1.233"}}
|
||||
original: Final = deepcopy((messages, request_kwargs))
|
||||
|
||||
result: Final = await router.async_pre_routing_hook("test-complexity-router", request_kwargs, messages)
|
||||
|
||||
assert result is not None and result.routing_decision["cause"] == "llm_classifier"
|
||||
assert result.messages == messages == original[0]
|
||||
assert request_kwargs == original[1]
|
||||
classifier_messages: Final = dependency.acompletion.call_args.kwargs["messages"]
|
||||
assert classifier_messages[0]["content"] == classification_system_prompt(
|
||||
router.config.classifier_context_window_size
|
||||
)
|
||||
assert classifier_messages[1]["content"].strip() == "Classify this message:\nWhat is two plus two?"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tail,expected",
|
||||
(
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -35064,7 +35064,7 @@ export interface components {
|
|||
classification_prompt?: string | null;
|
||||
/**
|
||||
* Classifier Context Budget Chars
|
||||
* @description Maximum characters of prior-turn text quoted to the LLM classifier, across the whole context window, per classification call. Turns are taken newest first and quoted whole while they fit, so a conversation small enough to quote entirely is never cut; once the budget runs out the older turns are dropped whole and only the turn straddling the boundary is truncated, into whatever space is left. The current ask and the caller's system prompt sit outside this budget and are always sent in full, as does the numbering each quoted turn carries. A budget under 120 leaves no room to quote a turn and suppresses the block; set classifier_context_window_size to 0 to turn context off deliberately. Only applies when classifier_type is 'llm'.
|
||||
* @description Maximum characters of prior-turn text quoted to the LLM classifier, across the whole context window, per classification call. Turns are taken newest first and quoted whole while they fit, so a conversation small enough to quote entirely is never cut; once the budget runs out the older turns are dropped whole and only the turn straddling the boundary is truncated, into whatever space is left. The current ask and, except for Claude Code requests, the extracted system-role text sit outside this budget and are sent in full, as does the numbering each quoted turn carries. A budget under 120 leaves no room to quote a turn and suppresses the block; set classifier_context_window_size to 0 to turn context off deliberately. Only applies when classifier_type is 'llm'.
|
||||
* @default 8000
|
||||
*/
|
||||
classifier_context_budget_chars: number;
|
||||
|
|
@ -35081,7 +35081,7 @@ export interface components {
|
|||
classifier_context_per_turn_chars?: number | null;
|
||||
/**
|
||||
* Classifier Context Window Size
|
||||
* @description Number of prior user turns (tool output and harness reminders excluded) to include as context in the LLM classifier prompt, so a follow-up like 'now do the same for the streaming path' is classified against what it refers to. Counts turns of both roles when classifier_context_include_assistant_turns is enabled. These turns are sent to the classifier model, which may be a different deployment or provider than the routed completion model; that call already carries the current user ask and the caller's system prompt in full. Set to 0 to send neither prior turns nor any conversation context beyond the current ask. Only applies when classifier_type is 'llm'.
|
||||
* @description Number of prior user turns (tool output and harness reminders excluded) to include as context in the LLM classifier prompt, so a follow-up like 'now do the same for the streaming path' is classified against what it refers to. Counts turns of both roles when classifier_context_include_assistant_turns is enabled. These turns are sent to the classifier model, which may be a different deployment or provider than the routed completion model; that call carries the current user ask and, except for Claude Code requests, the extracted system-role text in full. Claude Code system text is omitted to avoid classifying harness instructions; the routed completion still receives it. Set to 0 to send neither prior turns nor any conversation context beyond the current ask. Only applies when classifier_type is 'llm'.
|
||||
* @default 3
|
||||
*/
|
||||
classifier_context_window_size: number;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue