feat(router): select conversation headers for session affinity (#52193)

* feat(router): select conversation headers for session affinity

* test(router): assert legacy session routing results

* fix(router): validate conversation request fields at typed boundaries

* fix(router): synchronize conversation header API types

---------

Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
joshua-berri 2026-11-16 21:37:48 -08:00 • committed by GitHub
parent adcaa4d374
commit 87c94e58b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 444 additions and 33 deletions

View file

@ -1686,6 +1686,8 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD: Final = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY: Final = "_session_deployment_affinity_conversation"
REDACTED_HEADER_VALUE: Final = "***REDACTED***"
OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"})
CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"

View file

@ -19,6 +19,7 @@ from litellm.constants import (
MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
@ -564,6 +565,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset(
"_pipeline_managed_guardrails",
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
BAGGAGE_MAPPED_TAGS_METADATA_KEY,
CLIENT_OUTPUT_CEILING_METADATA_KEY,

View file

@ -40,8 +40,10 @@ from litellm.constants import (
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
OTEL_SERVICE_NAME_METADATA_KEYS,
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
REDACTED_HEADER_VALUE,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
SESSION_ID_OMITTED_METADATA_KEY,
@ -121,7 +123,6 @@ from litellm.types.proxy.auth.customer_hierarchy import customer_budget_scopes_m
# Cache special headers as a frozenset for O(1) lookup performance
_SPECIAL_HEADERS_CACHE: Final = frozenset(str(v.value).lower() for v in SpecialHeaders)
_REDACTED_HEADER_VALUE: Final = "***REDACTED***"
_DECODED_JSON: Final = TypeAdapter(object)
_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names() | frozenset(
@ -582,6 +583,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
"routing_decision",
GATEWAY_INJECTED_CACHE_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
CONSUMED_REQUEST_TAGS_METADATA_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
@ -1804,8 +1806,7 @@ def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
the stored copy and the logging callbacks JSON-serialize it.
"""
return {
header: (_REDACTED_HEADER_VALUE if _is_credential_header(header) else value)
for header, value in headers.items()
header: (REDACTED_HEADER_VALUE if _is_credential_header(header) else value) for header, value in headers.items()
}

View file

@ -31,6 +31,7 @@ from litellm.constants import (
CONSUMED_REQUEST_TAGS_METADATA_KEY,
DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
OUTPUT_TOKEN_CEILING_PARAMS,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.integrations.otel.routing import routing_decision_attributes
@ -863,6 +864,7 @@ class PreRouter:
if selected is None:
await arm_compaction(request_kwargs, None)
_stamp_or_clear_metadata_key(request_kwargs, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, None)
_stamp_or_clear_metadata_key(request_kwargs, SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, None)
_stamp_or_clear_metadata_key(request_kwargs, CONSUMED_REQUEST_TAGS_METADATA_KEY, None)
return None
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
@ -946,6 +948,11 @@ class PreRouter:
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
response.session_affinity_ttl_seconds if response else None,
)
_stamp_or_clear_metadata_key(
request_kwargs,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
response.session_affinity_conversation_id if response else None,
)
request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs)
consumed: Final = (
ConsumedRequestTagsStamp(model_group=response.model, tags=selected.tags)

View file

@ -485,6 +485,16 @@ without storing a tier pin. `session_affinity: true` retains precedence and the
`session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds.
When the session ID changes per message, set `conversation_header` to the header your application uses to name the conversation
```yaml
complexity_router_config:
session_affinity: true
conversation_header: x-conversation-id
```
The tier pin, the per-tier model pin, the deployment pin and `user_turn` continuations then follow that header's value instead of the session ID. The header name matches case-insensitively and the value is used exactly as sent. Pins stay scoped to the calling key or JWT user, and a `user_turn` child keeps its own pin inside the conversation. A request without the header, with an empty value, or naming a credential header that the proxy redacts gets no conversation-based reuse from this router and never falls back to the session ID. The same applies to SDK calls that carry no proxy request headers. Spend logs and traces keep the request's own session ID. A separately enabled Router-level `session_affinity` pre-call check keeps its own session ID behavior for those requests
### Mid-task stall escalation
A weak model working an agentic task can get stuck: it keeps calling the same tool with the

View file

@ -43,6 +43,7 @@ from litellm.constants import (
EMPTY_MAPPING,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
OUTPUT_TOKEN_CEILING_PARAMS,
REDACTED_HEADER_VALUE,
RETURN_RAW_MODEL_NAME_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
)
@ -837,7 +838,11 @@ _CLAUDE_CODE_REQUEST_CLASS_HEADER: Final = "x-claude-code-request-class"
_CLAUDE_CODE_AUXILIARY_SENTINEL: Final = f"{_CLAUDE_CODE_REQUEST_CLASS_HEADER}: auxiliary"
def _claude_code_request_class(request_kwargs: Mapping[str, object]) -> str | None:
_REQUEST_KWARGS: Final = TypeAdapter(Mapping[str, object])
_NO_REQUEST_KWARGS: Final[Mapping[str, object]] = MappingProxyType({})
def _proxy_request_header(request_kwargs: Mapping[str, object], header_name: str) -> str | None:
proxy_request: Final = request_kwargs.get("proxy_server_request")
if not isinstance(proxy_request, Mapping):
return None
@ -846,14 +851,23 @@ def _claude_code_request_class(request_kwargs: Mapping[str, object]) -> str | No
return None
return next(
(
value.strip().lower()
value
for key, value in cast(Mapping[object, object], headers).items() # cast-ok: header map from the proxy
if isinstance(key, str) and key.lower() == _CLAUDE_CODE_REQUEST_CLASS_HEADER and isinstance(value, str)
if isinstance(key, str) and key.lower() == header_name and isinstance(value, str)
),
None,
)
def _claude_code_request_class(request_kwargs: Mapping[str, object]) -> str | None:
value: Final = _proxy_request_header(request_kwargs, _CLAUDE_CODE_REQUEST_CLASS_HEADER)
return value.strip().lower() if value is not None else None
def _conversation_pin_id(parts: tuple[str, ...]) -> str:
return "conversation-header:" + hashlib.sha256(json.dumps(parts).encode()).hexdigest()
_CODEX_TURN_METADATA_KEY: Final = "x-codex-turn-metadata"
_CODEX_COMPACTION_SUMMARY_PREFIX: Final = (
"Another language model started to solve this problem and produced a summary of its thinking process."
@ -3204,7 +3218,7 @@ class ComplexityRouter(CustomLogger):
if retained_model is not None and retained_model in candidates:
self._restamp_adaptive_choice(request_kwargs, model, retained_model)
return retained_model
session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs)
session_id: Final = self._affinity_session_id(request_kwargs)
if session_id is None:
return model
cache_key: Final = TierModelPin(
@ -4504,19 +4518,42 @@ class ComplexityRouter(CustomLogger):
return str(session_id)
return None
def _child_tier_session_metadata(self, request_kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
if self.config.classification_mode != "user_turn" or self.config.session_affinity:
return None
return next(
(
metadata
for metadata in self._iter_metadata_dicts(request_kwargs)
if COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY in metadata
),
None,
)
def _configured_conversation_id(self, request_kwargs: Mapping[str, object], header_name: str) -> str | None:
conversation: Final = _proxy_request_header(request_kwargs, header_name)
if not conversation or conversation == REDACTED_HEADER_VALUE:
return None
child_metadata: Final = self._child_tier_session_metadata(request_kwargs)
if child_metadata is None:
return _conversation_pin_id((header_name, conversation))
child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY]
if not isinstance(child_session, str) or not child_session:
return None
return _conversation_pin_id((header_name, conversation, child_session))
def _affinity_session_id(self, request_kwargs: Mapping[str, object]) -> str | None:
if self.config.conversation_header is not None:
return self._configured_conversation_id(request_kwargs, self.config.conversation_header)
return self._get_session_id_from_request_kwargs(request_kwargs)
def _get_tier_pin_session_id(self, request_kwargs: Mapping[str, object]) -> str | None:
if self.config.classification_mode == "user_turn" and not self.config.session_affinity:
child_metadata: Final = next(
(
metadata
for metadata in self._iter_metadata_dicts(request_kwargs)
if COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY in metadata
),
None,
)
if child_metadata is not None:
child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY]
return child_session if isinstance(child_session, str) and child_session else None
if self.config.conversation_header is not None:
return self._configured_conversation_id(request_kwargs, self.config.conversation_header)
child_metadata: Final = self._child_tier_session_metadata(request_kwargs)
if child_metadata is not None:
child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY]
return child_session if isinstance(child_session, str) and child_session else None
return self._get_session_id_from_request_kwargs(request_kwargs)
@staticmethod
@ -4533,7 +4570,13 @@ class ComplexityRouter(CustomLogger):
def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> TierSessionPin:
return TierSessionPin(
router=self.model_name,
caller=self._get_user_api_key_hash_from_request_kwargs(request_kwargs),
caller=(
DeploymentAffinityCheck.get_user_key_from_request_kwargs(
_REQUEST_KWARGS.validate_python(request_kwargs)
)
if self.config.conversation_header is not None
else self._get_user_api_key_hash_from_request_kwargs(request_kwargs)
),
session_id=session_id,
)
@ -4553,11 +4596,23 @@ class ComplexityRouter(CustomLogger):
return bool(self.config.deployment_affinity and not self.config.plugins) or self._uses_tier_pin
def _with_session_deployment_affinity(
self, response: PreRoutingHookResponse | None
self, response: PreRoutingHookResponse | None, request_kwargs: Mapping[str, object]
) -> PreRoutingHookResponse | None:
if response is None or not self._uses_deployment_pin:
return response
return response.model_copy(update={"session_affinity_ttl_seconds": self.config.session_affinity_ttl_seconds})
if self.config.conversation_header is None:
return response.model_copy(
update={"session_affinity_ttl_seconds": self.config.session_affinity_ttl_seconds}
)
conversation_id: Final = self._configured_conversation_id(request_kwargs, self.config.conversation_header)
if conversation_id is None:
return response
return response.model_copy(
update={
"session_affinity_ttl_seconds": self.config.session_affinity_ttl_seconds,
"session_affinity_conversation_id": conversation_id,
}
)
@with_service_target(ROUTER_SESSION_PINS_TARGET)
async def async_pre_routing_hook(
@ -4778,7 +4833,10 @@ class ComplexityRouter(CustomLogger):
request_kwargs,
context_fit,
retained_pin,
)
),
_REQUEST_KWARGS.validate_python(request_kwargs)
if self.config.conversation_header is not None
else _NO_REQUEST_KWARGS,
)
routed_response: Final = await self._classify_and_route(
@ -4833,7 +4891,12 @@ class ComplexityRouter(CustomLogger):
),
self.config.session_affinity_ttl_seconds,
)
return self._with_session_deployment_affinity(response)
return self._with_session_deployment_affinity(
response,
_REQUEST_KWARGS.validate_python(request_kwargs)
if self.config.conversation_header is not None
else _NO_REQUEST_KWARGS,
)
async def _apply_prompt_cache_routing(
self,

View file

@ -1667,6 +1667,29 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
"idle time for the session's routing decisions rather than total session length"
),
)
conversation_header: str | None = Field(
default=None,
description=(
"Request header whose value names the application conversation for session_affinity, "
"deployment_affinity and user_turn pin reuse, instead of the request's session_id. The name "
"matches case-insensitively and the value is used as-is, so short and case-sensitive ids work. "
"A request without a usable value (header missing, empty, or a redacted credential header, and "
"any SDK call without proxy request headers) gets no conversation-based reuse from this router and "
"never falls back to its session_id. A separately enabled Router-level session_affinity pre-call "
"check keeps its own session_id behavior. Spend logs and traces keep the request's own session_id. "
"Unset keeps the session_id behavior"
),
)
@field_validator("conversation_header")
@classmethod
def _normalize_conversation_header(cls, value: str | None) -> str | None:
if value is None:
return None
name: Final = value.strip().lower()
if not name:
raise ValueError("conversation_header must be a non-empty header name")
return name
plugins: list[RoutingPlugin] | None = Field(
default=None,

View file

@ -26,7 +26,11 @@ from litellm.caching.affinity_cache import (
claim_affinity_pin,
set_local_affinity_pin,
)
from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
from litellm.constants import (
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request
from litellm.redis.stores.router_session_pins import (
@ -324,7 +328,19 @@ class DeploymentAffinityCheck(CustomLogger):
@staticmethod
def _get_session_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None:
for metadata in DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs):
metadata_dicts: Final = DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs)
conversation_id: Final = next(
(
value
for metadata in metadata_dicts
if isinstance(value := metadata.get(SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY), str)
and value
),
None,
)
if conversation_id is not None:
return conversation_id
for metadata in metadata_dicts:
session_id = DeploymentAffinityCheck._get_session_id_from_metadata_dict(metadata=metadata)
if session_id is not None:
return session_id

View file

@ -1428,6 +1428,7 @@ class PreRoutingHookResponse(LiteLLMBaseModel):
messages: list[dict[str, Any]] | None
routing_decision: StandardLoggingRoutingDecision | None = None
session_affinity_ttl_seconds: int | None = None
session_affinity_conversation_id: str | None = None
litellm_params: Mapping[str, object] | None = None

View file

@ -45806,6 +45806,11 @@ export interface components {
* @default 0.95
*/
context_window_escalation_buffer: number;
/**
* Conversation Header
* @description Request header whose value names the application conversation for session_affinity, deployment_affinity and user_turn pin reuse, instead of the request's session_id. The name matches case-insensitively and the value is used as-is, so short and case-sensitive ids work. A request without a usable value (header missing, empty, or a redacted credential header, and any SDK call without proxy request headers) gets no conversation-based reuse from this router and never falls back to its session_id. A separately enabled Router-level session_affinity pre-call check keeps its own session_id behavior. Spend logs and traces keep the request's own session_id. Unset keeps the session_id behavior
*/
conversation_header?: string | null;
/**
* Custom Dimensions
* @description Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, backreferences and lookarounds are rejected. Conservative work limits include alternation paths, repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. Only heuristic, heuristic_first and hybrid accept this field. Uses the existing heuristic tuning quota.

View file

@ -1592,6 +1592,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"routing_decision": {"cause": "forged", "routed_model": "spoofed"},
"litellm_gateway_injected_cache": "forged-deployment-id",
"_session_deployment_affinity_ttl": 999999,
"_session_deployment_affinity_conversation": "forged-conversation",
"internal_call_origin": "autorouter_classifier",
"_managed_background_response": True,
"_guardrail_pipelines": [{"name": "spoofed"}],
@ -1653,6 +1654,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"routing_decision",
"litellm_gateway_injected_cache",
"_session_deployment_affinity_ttl",
"_session_deployment_affinity_conversation",
"internal_call_origin",
"_managed_background_response",
"_guardrail_pipelines",

View file

@ -7616,7 +7616,7 @@ class _DummyPlugin:
class TestChildSessionTierPins:
@staticmethod
def _router(session_affinity: bool = False) -> Router:
def _router(session_affinity: bool = False, conversation_header: str | None = None) -> Router:
return Router(
model_list=[
{
@ -7628,6 +7628,7 @@ class TestChildSessionTierPins:
"classification_mode": "user_turn",
"session_affinity": session_affinity,
"keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}],
**({"conversation_header": conversation_header} if conversation_header else {}),
},
},
},
@ -7674,10 +7675,20 @@ class TestChildSessionTierPins:
self, router: Router, surface: str, session_id: str | None, prompt: str | None,
expected_tier: str, expected_cause: str, caller: str = "caller-a", child: bool = True,
expected_deployment: str | None = None,
) -> str:
data, bucket = self._normalized_request(surface, session_id, caller, child)
return await self._route_request(
router, surface, data, bucket, prompt, (expected_tier, expected_cause), "parent-session",
expected_deployment,
)
@staticmethod
async def _route_request(
router: Router, surface: str, data: dict[str, object], bucket: str, prompt: str | None,
expected_decision: tuple[str, str], expected_log_id: str, expected_deployment: str | None = None,
) -> str:
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck
data, bucket = self._normalized_request(surface, session_id, caller, child)
turn: Final[dict[str, object]] = (
{"role": "user", "content": prompt} if prompt is not None else
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "done"}]}
@ -7703,11 +7714,9 @@ class TestChildSessionTierPins:
metadata: Final = data[bucket]
assert isinstance(metadata, dict)
assert (metadata["session_id"], metadata["trace_id"], data["litellm_session_id"], data["litellm_trace_id"]) == (
"parent-session", "parent-session", "parent-session", "parent-session"
)
assert (metadata["routing_decision"]["tier"], metadata["routing_decision"]["cause"]) == (
expected_tier, expected_cause
expected_log_id, expected_log_id, expected_log_id, expected_log_id
)
assert (metadata["routing_decision"]["tier"], metadata["routing_decision"]["cause"]) == expected_decision
affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck))
if expected_deployment is not None:
filtered: Final = await affinity.async_filter_deployments(
@ -7784,6 +7793,196 @@ class TestChildSessionTierPins:
router.discard()
class TestConversationHeaderPins:
REASONING: Final = ("REASONING", "literal_keyword_match")
SIMPLE: Final = ("SIMPLE", "heuristic_scorer")
CONTINUED: Final = ("REASONING", "user_turn_continuation")
@staticmethod
def _request(
surface: str, headers: Mapping[str, str], caller: tuple[str, str] = ("user_api_key_hash", "caller-a")
) -> tuple[dict[str, object], str]:
from litellm.proxy.litellm_pre_call_utils import (
LiteLLMProxyRequestSetup,
metadata_variable_name_for_route,
redact_credential_headers,
)
route: Final = {
"chat": "/v1/chat/completions", "responses": "/v1/responses", "messages": "/v1/messages"
}[surface]
bucket: Final = metadata_variable_name_for_route(route)
data: Final[dict[str, object]] = {
bucket: {caller[0]: caller[1], "user_api_key_request_route": route},
"proxy_server_request": {"headers": redact_credential_headers(headers)},
}
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
headers=dict(headers), data=data, _metadata_variable_name=bucket
)
return data, bucket
async def _route(
self, router: Router, surface: str, headers: Mapping[str, str], prompt: str | None,
expected_decision: tuple[str, str], caller: tuple[str, str] = ("user_api_key_hash", "caller-a"),
expected_deployment: str | None = None,
) -> str:
data, bucket = self._request(surface, headers, caller)
return await TestChildSessionTierPins._route_request(
router, surface, data, bucket, prompt, expected_decision,
headers.get("x-parent-session-id") or headers["X-Session-Id"], expected_deployment,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("session_affinity", [False, True])
@pytest.mark.parametrize("surface", ["chat", "responses", "messages"])
async def test_one_conversation_keeps_its_pins_across_message_ids(
self, surface: str, session_affinity: bool
) -> None:
router: Final = TestChildSessionTierPins._router(session_affinity, conversation_header=" X-App-Conversation ")
replay: Final = "session_affinity_pin" if session_affinity else "user_turn_continuation"
try:
first: Final = await self._route(
router, surface, {"x-app-conversation": "Ab", "X-Session-Id": "message-0001"}, "reason carefully",
self.REASONING,
)
await self._route(
router, surface, {"X-App-Conversation": "ab", "X-Session-Id": "message-0001"}, None, self.SIMPLE
)
assert await self._route(
router, surface, {"X-APP-CONVERSATION": "Ab", "X-Session-Id": "message-0002"}, None,
("REASONING", replay), expected_deployment=first,
) == first
finally:
router.discard()
@pytest.mark.asyncio
@pytest.mark.parametrize("child", [False, True])
@pytest.mark.parametrize(
"conversation_header,conversation",
[
("x-app-conversation", {}),
("x-app-conversation", {"x-app-conversation": ""}),
("authorization", {"Authorization": "Bearer test-only"}),
],
ids=["absent", "empty", "redacted"],
)
async def test_configured_header_without_a_usable_value_disables_reuse(
self, conversation_header: str, conversation: Mapping[str, str], child: bool
) -> None:
from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck
router: Final = TestChildSessionTierPins._router(conversation_header=conversation_header)
headers: Final = {
**conversation,
"X-Session-Id": "message-0001",
**({"x-parent-session-id": "parent-session"} if child else {}),
}
try:
first: Final = await self._route(router, "chat", headers, "reason carefully", self.REASONING)
data, bucket = self._request("chat", headers)
await TestChildSessionTierPins._route_request(
router, "chat", data, bucket, None, self.SIMPLE, "parent-session" if child else "message-0001"
)
affinity: Final = next(
cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)
)
first_group: Final = first.rsplit("-", 1)[0]
filtered: Final = await affinity.async_filter_deployments(
model=first_group, healthy_deployments=router.get_model_list(first_group) or [],
messages=None, request_kwargs=data,
)
assert sorted(deployment["model_info"]["id"] for deployment in filtered) == [
f"{first_group}-0", f"{first_group}-1"
]
finally:
router.discard()
@pytest.mark.asyncio
async def test_configured_children_stay_inside_their_conversation(self) -> None:
router: Final = TestChildSessionTierPins._router(conversation_header="x-app-conversation")
child: Final = {"x-app-conversation": "c1", "x-parent-session-id": "parent-0001", "X-Session-Id": "child-000a"}
try:
parent: Final = await self._route(
router, "chat", {"x-app-conversation": "c1", "X-Session-Id": "parent-0001"}, "reason carefully",
self.REASONING,
)
await self._route(router, "chat", child, "Hello!", self.SIMPLE)
await self._route(router, "chat", child, None, ("SIMPLE", "user_turn_continuation"))
assert await self._route(
router, "chat", {"x-app-conversation": "c1", "X-Session-Id": "parent-0002"}, None, self.CONTINUED,
expected_deployment=parent,
) == parent
await self._route(
router, "chat", {**child, "X-Session-Id": "parent-0001"}, None, self.SIMPLE
)
finally:
router.discard()
@pytest.mark.asyncio
@pytest.mark.parametrize("identity", ["user_api_key_hash", "user_api_key_user_id"])
async def test_callers_sharing_a_conversation_id_never_share_pins(self, identity: str) -> None:
router: Final = TestChildSessionTierPins._router(conversation_header="x-app-conversation")
try:
first: Final = await self._route(
router, "chat", {"x-app-conversation": "shared", "X-Session-Id": "message-0001"}, "reason carefully",
self.REASONING, caller=(identity, "tenant-a"),
)
await self._route(
router, "chat", {"x-app-conversation": "shared", "X-Session-Id": "message-0002"}, None, self.SIMPLE,
caller=(identity, "tenant-b"),
)
assert await self._route(
router, "chat", {"x-app-conversation": "shared", "X-Session-Id": "message-0003"}, None,
self.CONTINUED, caller=(identity, "tenant-a"), expected_deployment=first,
) == first
finally:
router.discard()
@pytest.mark.asyncio
async def test_unset_header_keeps_session_id_pins_under_the_key_hash(self) -> None:
from litellm.redis.stores.router_session_pins import TierSessionPin
router: Final = TestChildSessionTierPins._router()
strategy: Final = router.complexity_routers["child-tier-router"][0].strategy
try:
await strategy.ctx.health().session_pins.set(
TierSessionPin(router="child-tier-router", caller="caller-a", session_id="message-0001"),
{"model": "reasoning", "tier": "REASONING"},
60,
)
replayed: Final = await self._route(
router, "chat", {"x-app-conversation": "Ab", "X-Session-Id": "message-0001"}, None, self.CONTINUED
)
fresh: Final = await self._route(
router, "chat", {"x-app-conversation": "Ab", "X-Session-Id": "message-0002"}, None, self.SIMPLE
)
assert (replayed.rsplit("-", 1)[0], fresh.rsplit("-", 1)[0]) == ("reasoning", "simple")
finally:
router.discard()
@pytest.mark.asyncio
async def test_tier_model_pin_follows_the_conversation(self) -> None:
router: Final = TestChildSessionTierPins._router(conversation_header="x-app-conversation")
strategy: Final = router.complexity_routers["child-tier-router"][0].strategy
candidates: Final = ("reasoning", "reasoning-peer")
try:
for headers, offered, pinned in (
({"X-Session-Id": "message-0001"}, "reasoning-peer", "reasoning-peer"),
({"X-Session-Id": "message-0001"}, "reasoning", "reasoning"),
({"x-app-conversation": "Ab", "X-Session-Id": "message-0001"}, "reasoning", "reasoning"),
({"x-app-conversation": "Ab", "X-Session-Id": "message-0002"}, "reasoning-peer", "reasoning"),
({"x-app-conversation": "ab", "X-Session-Id": "message-0001"}, "reasoning-peer", "reasoning-peer"),
):
request, _ = self._request("chat", headers)
assert await strategy._pin_model_for_tier("REASONING", offered, candidates, request) == pinned
finally:
router.discard()
def test_conversation_header_must_name_a_header(self) -> None:
with pytest.raises(ValidationError, match="conversation_header"):
ComplexityRouterConfig.model_validate({"conversation_header": " "})
class TestClaudeCodeTierPins:
SESSION: Final = "11111111-2222-3333-4444-555555555555"
AGENT_A: Final = "a7256c04dfd99c9b9"

View file

@ -16,6 +16,7 @@ from litellm.redis.stores.router_session_pins import (
TierModelPin,
)
from litellm.constants import (
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
)
@ -392,7 +393,9 @@ MOCK_RESPONSES_API_RESPONSE = {
}
def _smart_router(session_affinity=True, ttl_seconds=777, deployment_affinity=True):
def _smart_router(
session_affinity=True, ttl_seconds=777, deployment_affinity=True, conversation_header: str | None = None
):
return litellm.Router(
model_list=[
{
@ -404,6 +407,7 @@ def _smart_router(session_affinity=True, ttl_seconds=777, deployment_affinity=Tr
"session_affinity": session_affinity,
"deployment_affinity": deployment_affinity,
"session_affinity_ttl_seconds": ttl_seconds,
**({"conversation_header": conversation_header} if conversation_header else {}),
"tiers": {
"SIMPLE": "target-group",
"MEDIUM": "target-group",
@ -814,6 +818,82 @@ async def test_pre_routing_hook_stamps_or_clears_the_marker_per_attempt(
_cleanup_router_callbacks(router)
async def _conversation_turn(
router: litellm.Router, model: str, headers: Mapping[str, str], metadata: Mapping[str, object]
) -> tuple[list[str], Mapping[str, object]]:
request_kwargs: Final[dict[str, object]] = {
"proxy_server_request": {"headers": headers},
"metadata": {**metadata, "user_api_key_hash": "key-1"},
}
await router.async_pre_routing_hook(
model=model, request_kwargs=request_kwargs, messages=[{"role": "user", "content": "Hello"}]
)
affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck))
filtered: Final = await affinity.async_filter_deployments(
model="target-group",
healthy_deployments=router.get_model_list("target-group") or [],
messages=None,
request_kwargs=request_kwargs,
)
routed: Final = request_kwargs["metadata"]
assert isinstance(routed, dict)
await affinity.async_pre_call_deployment_hook(
kwargs={
**request_kwargs,
"model_info": {"id": "deployment-2"},
"metadata": {**routed, "deployment_model_name": "target-group"},
},
call_type=None,
)
return sorted(str(deployment["model_info"]["id"]) for deployment in filtered), routed
@pytest.mark.asyncio
async def test_configured_conversation_header_owns_the_deployment_pin() -> None:
router: Final = _smart_router(conversation_header="x-app-conversation")
both: Final = ["deployment-1", "deployment-2"]
try:
affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck))
await affinity.async_pre_call_deployment_hook(
kwargs={
"model_info": {"id": "deployment-1"},
"metadata": {
"deployment_model_name": "target-group",
"session_id": "message-1",
"user_api_key_hash": "key-1",
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 777,
},
},
call_type=None,
)
first, routed = await _conversation_turn(
router, "smart-router", {"x-app-conversation": "Ab"}, {"session_id": "message-1"}
)
conversation: Final = routed[SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY]
forged: Final = {
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 777,
SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY: conversation,
}
same_conversation, _ = await _conversation_turn(
router, "smart-router", {"x-app-conversation": "Ab"}, {"session_id": "message-2"}
)
other_conversation, _ = await _conversation_turn(
router, "smart-router", {"x-app-conversation": "ab"}, {"session_id": "message-2"}
)
unusable_headers: Final = [
(await _conversation_turn(router, "smart-router", headers, {**forged, "session_id": "message-1"}))[0]
for headers in ({}, {"x-app-conversation": ""})
]
rerouted, _ = await _conversation_turn(
router, "target-group", {"x-app-conversation": "Ab"}, {**forged, "session_id": "message-2"}
)
assert (first, same_conversation, other_conversation, unusable_headers, rerouted) == (
both, ["deployment-2"], both, [both, both], both
)
finally:
_cleanup_router_callbacks(router)
def test_complexity_router_with_deployment_affinity_registers_affinity_callback():
enabled = _smart_router()
session_only = _smart_router(session_affinity=True, deployment_affinity=False)