diff --git a/litellm/constants.py b/litellm/constants.py index dd71a4ce5f7..d1ca30bf226 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1686,6 +1686,8 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata" OLD_LITELLM_METADATA_FIELD: Final = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name" SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" +SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY: Final = "_session_deployment_affinity_conversation" +REDACTED_HEADER_VALUE: Final = "***REDACTED***" OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"}) CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling" CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index c39f56b6a08..b89ed6feced 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -19,6 +19,7 @@ from litellm.constants import ( MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH, PRE_CALL_EXECUTED_GUARDRAILS_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.custom_logger import CustomLogger @@ -564,6 +565,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( "_pipeline_managed_guardrails", PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, BAGGAGE_MAPPED_TAGS_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index b5cf27c41f0..c3966aad81a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -40,8 +40,10 @@ from litellm.constants import ( MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, OTEL_SERVICE_NAME_METADATA_KEYS, PRE_CALL_EXECUTED_GUARDRAILS_KEY, + REDACTED_HEADER_VALUE, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, @@ -121,7 +123,6 @@ from litellm.types.proxy.auth.customer_hierarchy import customer_budget_scopes_m # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE: Final = frozenset(str(v.value).lower() for v in SpecialHeaders) -_REDACTED_HEADER_VALUE: Final = "***REDACTED***" _DECODED_JSON: Final = TypeAdapter(object) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CREDENTIAL_HEADER_NAMES: Final = SpecialHeaders.litellm_credential_header_names() | frozenset( @@ -582,6 +583,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "routing_decision", GATEWAY_INJECTED_CACHE_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -1804,8 +1806,7 @@ def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]: the stored copy and the logging callbacks JSON-serialize it. """ return { - header: (_REDACTED_HEADER_VALUE if _is_credential_header(header) else value) - for header, value in headers.items() + header: (REDACTED_HEADER_VALUE if _is_credential_header(header) else value) for header, value in headers.items() } diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py index 093ba86f7a1..5cd183e353a 100644 --- a/litellm/router_rust/steps/prerouting.py +++ b/litellm/router_rust/steps/prerouting.py @@ -31,6 +31,7 @@ from litellm.constants import ( CONSUMED_REQUEST_TAGS_METADATA_KEY, DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, OUTPUT_TOKEN_CEILING_PARAMS, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) from litellm.integrations.otel.routing import routing_decision_attributes @@ -863,6 +864,7 @@ class PreRouter: if selected is None: await arm_compaction(request_kwargs, None) _stamp_or_clear_metadata_key(request_kwargs, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, None) + _stamp_or_clear_metadata_key(request_kwargs, SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, None) _stamp_or_clear_metadata_key(request_kwargs, CONSUMED_REQUEST_TAGS_METADATA_KEY, None) return None from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter @@ -946,6 +948,11 @@ class PreRouter: SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, response.session_affinity_ttl_seconds if response else None, ) + _stamp_or_clear_metadata_key( + request_kwargs, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, + response.session_affinity_conversation_id if response else None, + ) request_tags: Final = get_routing_tags_from_request_kwargs(request_kwargs) consumed: Final = ( ConsumedRequestTagsStamp(model_group=response.model, tags=selected.tags) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 000915ea727..7fe3a9a9b7e 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -485,6 +485,16 @@ without storing a tier pin. `session_affinity: true` retains precedence and the `session_affinity_ttl_seconds` is the idle window for both the model pin selected by session affinity and the deployment pin. Every request that reuses a pin refreshes its TTL, so a session actively sending requests stays pinned. After the window passes with no pin reuse, the next request classifies again and creates a fresh pin. Omit the setting to track the default of 3600 seconds. +When the session ID changes per message, set `conversation_header` to the header your application uses to name the conversation + +```yaml +complexity_router_config: + session_affinity: true + conversation_header: x-conversation-id +``` + +The tier pin, the per-tier model pin, the deployment pin and `user_turn` continuations then follow that header's value instead of the session ID. The header name matches case-insensitively and the value is used exactly as sent. Pins stay scoped to the calling key or JWT user, and a `user_turn` child keeps its own pin inside the conversation. A request without the header, with an empty value, or naming a credential header that the proxy redacts gets no conversation-based reuse from this router and never falls back to the session ID. The same applies to SDK calls that carry no proxy request headers. Spend logs and traces keep the request's own session ID. A separately enabled Router-level `session_affinity` pre-call check keeps its own session ID behavior for those requests + ### Mid-task stall escalation A weak model working an agentic task can get stuck: it keeps calling the same tool with the diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 51ab2256093..848a982a375 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -43,6 +43,7 @@ from litellm.constants import ( EMPTY_MAPPING, INTERNAL_CALL_ORIGIN_METADATA_KEY, OUTPUT_TOKEN_CEILING_PARAMS, + REDACTED_HEADER_VALUE, RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, ) @@ -837,7 +838,11 @@ _CLAUDE_CODE_REQUEST_CLASS_HEADER: Final = "x-claude-code-request-class" _CLAUDE_CODE_AUXILIARY_SENTINEL: Final = f"{_CLAUDE_CODE_REQUEST_CLASS_HEADER}: auxiliary" -def _claude_code_request_class(request_kwargs: Mapping[str, object]) -> str | None: +_REQUEST_KWARGS: Final = TypeAdapter(Mapping[str, object]) +_NO_REQUEST_KWARGS: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _proxy_request_header(request_kwargs: Mapping[str, object], header_name: str) -> str | None: proxy_request: Final = request_kwargs.get("proxy_server_request") if not isinstance(proxy_request, Mapping): return None @@ -846,14 +851,23 @@ def _claude_code_request_class(request_kwargs: Mapping[str, object]) -> str | No return None return next( ( - value.strip().lower() + value for key, value in cast(Mapping[object, object], headers).items() # cast-ok: header map from the proxy - if isinstance(key, str) and key.lower() == _CLAUDE_CODE_REQUEST_CLASS_HEADER and isinstance(value, str) + if isinstance(key, str) and key.lower() == header_name and isinstance(value, str) ), None, ) +def _claude_code_request_class(request_kwargs: Mapping[str, object]) -> str | None: + value: Final = _proxy_request_header(request_kwargs, _CLAUDE_CODE_REQUEST_CLASS_HEADER) + return value.strip().lower() if value is not None else None + + +def _conversation_pin_id(parts: tuple[str, ...]) -> str: + return "conversation-header:" + hashlib.sha256(json.dumps(parts).encode()).hexdigest() + + _CODEX_TURN_METADATA_KEY: Final = "x-codex-turn-metadata" _CODEX_COMPACTION_SUMMARY_PREFIX: Final = ( "Another language model started to solve this problem and produced a summary of its thinking process." @@ -3204,7 +3218,7 @@ class ComplexityRouter(CustomLogger): if retained_model is not None and retained_model in candidates: self._restamp_adaptive_choice(request_kwargs, model, retained_model) return retained_model - session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs) + session_id: Final = self._affinity_session_id(request_kwargs) if session_id is None: return model cache_key: Final = TierModelPin( @@ -4504,19 +4518,42 @@ class ComplexityRouter(CustomLogger): return str(session_id) return None + def _child_tier_session_metadata(self, request_kwargs: Mapping[str, object]) -> Mapping[str, object] | None: + if self.config.classification_mode != "user_turn" or self.config.session_affinity: + return None + return next( + ( + metadata + for metadata in self._iter_metadata_dicts(request_kwargs) + if COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY in metadata + ), + None, + ) + + def _configured_conversation_id(self, request_kwargs: Mapping[str, object], header_name: str) -> str | None: + conversation: Final = _proxy_request_header(request_kwargs, header_name) + if not conversation or conversation == REDACTED_HEADER_VALUE: + return None + child_metadata: Final = self._child_tier_session_metadata(request_kwargs) + if child_metadata is None: + return _conversation_pin_id((header_name, conversation)) + child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY] + if not isinstance(child_session, str) or not child_session: + return None + return _conversation_pin_id((header_name, conversation, child_session)) + + def _affinity_session_id(self, request_kwargs: Mapping[str, object]) -> str | None: + if self.config.conversation_header is not None: + return self._configured_conversation_id(request_kwargs, self.config.conversation_header) + return self._get_session_id_from_request_kwargs(request_kwargs) + def _get_tier_pin_session_id(self, request_kwargs: Mapping[str, object]) -> str | None: - if self.config.classification_mode == "user_turn" and not self.config.session_affinity: - child_metadata: Final = next( - ( - metadata - for metadata in self._iter_metadata_dicts(request_kwargs) - if COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY in metadata - ), - None, - ) - if child_metadata is not None: - child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY] - return child_session if isinstance(child_session, str) and child_session else None + if self.config.conversation_header is not None: + return self._configured_conversation_id(request_kwargs, self.config.conversation_header) + child_metadata: Final = self._child_tier_session_metadata(request_kwargs) + if child_metadata is not None: + child_session: Final = child_metadata[COMPLEXITY_ROUTER_CHILD_SESSION_ID_METADATA_KEY] + return child_session if isinstance(child_session, str) and child_session else None return self._get_session_id_from_request_kwargs(request_kwargs) @staticmethod @@ -4533,7 +4570,13 @@ class ComplexityRouter(CustomLogger): def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> TierSessionPin: return TierSessionPin( router=self.model_name, - caller=self._get_user_api_key_hash_from_request_kwargs(request_kwargs), + caller=( + DeploymentAffinityCheck.get_user_key_from_request_kwargs( + _REQUEST_KWARGS.validate_python(request_kwargs) + ) + if self.config.conversation_header is not None + else self._get_user_api_key_hash_from_request_kwargs(request_kwargs) + ), session_id=session_id, ) @@ -4553,11 +4596,23 @@ class ComplexityRouter(CustomLogger): return bool(self.config.deployment_affinity and not self.config.plugins) or self._uses_tier_pin def _with_session_deployment_affinity( - self, response: PreRoutingHookResponse | None + self, response: PreRoutingHookResponse | None, request_kwargs: Mapping[str, object] ) -> PreRoutingHookResponse | None: if response is None or not self._uses_deployment_pin: return response - return response.model_copy(update={"session_affinity_ttl_seconds": self.config.session_affinity_ttl_seconds}) + if self.config.conversation_header is None: + return response.model_copy( + update={"session_affinity_ttl_seconds": self.config.session_affinity_ttl_seconds} + ) + conversation_id: Final = self._configured_conversation_id(request_kwargs, self.config.conversation_header) + if conversation_id is None: + return response + return response.model_copy( + update={ + "session_affinity_ttl_seconds": self.config.session_affinity_ttl_seconds, + "session_affinity_conversation_id": conversation_id, + } + ) @with_service_target(ROUTER_SESSION_PINS_TARGET) async def async_pre_routing_hook( @@ -4778,7 +4833,10 @@ class ComplexityRouter(CustomLogger): request_kwargs, context_fit, retained_pin, - ) + ), + _REQUEST_KWARGS.validate_python(request_kwargs) + if self.config.conversation_header is not None + else _NO_REQUEST_KWARGS, ) routed_response: Final = await self._classify_and_route( @@ -4833,7 +4891,12 @@ class ComplexityRouter(CustomLogger): ), self.config.session_affinity_ttl_seconds, ) - return self._with_session_deployment_affinity(response) + return self._with_session_deployment_affinity( + response, + _REQUEST_KWARGS.validate_python(request_kwargs) + if self.config.conversation_header is not None + else _NO_REQUEST_KWARGS, + ) async def _apply_prompt_cache_routing( self, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 0d958c5be5f..01f29f7f589 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -1667,6 +1667,29 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "idle time for the session's routing decisions rather than total session length" ), ) + conversation_header: str | None = Field( + default=None, + description=( + "Request header whose value names the application conversation for session_affinity, " + "deployment_affinity and user_turn pin reuse, instead of the request's session_id. The name " + "matches case-insensitively and the value is used as-is, so short and case-sensitive ids work. " + "A request without a usable value (header missing, empty, or a redacted credential header, and " + "any SDK call without proxy request headers) gets no conversation-based reuse from this router and " + "never falls back to its session_id. A separately enabled Router-level session_affinity pre-call " + "check keeps its own session_id behavior. Spend logs and traces keep the request's own session_id. " + "Unset keeps the session_id behavior" + ), + ) + + @field_validator("conversation_header") + @classmethod + def _normalize_conversation_header(cls, value: str | None) -> str | None: + if value is None: + return None + name: Final = value.strip().lower() + if not name: + raise ValueError("conversation_header must be a non-empty header name") + return name plugins: list[RoutingPlugin] | None = Field( default=None, diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 19d80f0c0db..1d957ec5db5 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -26,7 +26,11 @@ from litellm.caching.affinity_cache import ( claim_affinity_pin, set_local_affinity_pin, ) -from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY +from litellm.constants import ( + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.routing_common_utils import deployment_ids_failed_in_request from litellm.redis.stores.router_session_pins import ( @@ -324,7 +328,19 @@ class DeploymentAffinityCheck(CustomLogger): @staticmethod def _get_session_id_from_request_kwargs(request_kwargs: Mapping[str, object]) -> str | None: - for metadata in DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs): + metadata_dicts: Final = DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs) + conversation_id: Final = next( + ( + value + for metadata in metadata_dicts + if isinstance(value := metadata.get(SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY), str) + and value + ), + None, + ) + if conversation_id is not None: + return conversation_id + for metadata in metadata_dicts: session_id = DeploymentAffinityCheck._get_session_id_from_metadata_dict(metadata=metadata) if session_id is not None: return session_id diff --git a/litellm/types/router.py b/litellm/types/router.py index fe046a3b305..2c6e2505071 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1428,6 +1428,7 @@ class PreRoutingHookResponse(LiteLLMBaseModel): messages: list[dict[str, Any]] | None routing_decision: StandardLoggingRoutingDecision | None = None session_affinity_ttl_seconds: int | None = None + session_affinity_conversation_id: str | None = None litellm_params: Mapping[str, object] | None = None diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index d0d448cfb57..a4c420fca92 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -45806,6 +45806,11 @@ export interface components { * @default 0.95 */ context_window_escalation_buffer: number; + /** + * Conversation Header + * @description Request header whose value names the application conversation for session_affinity, deployment_affinity and user_turn pin reuse, instead of the request's session_id. The name matches case-insensitively and the value is used as-is, so short and case-sensitive ids work. A request without a usable value (header missing, empty, or a redacted credential header, and any SDK call without proxy request headers) gets no conversation-based reuse from this router and never falls back to its session_id. A separately enabled Router-level session_affinity pre-call check keeps its own session_id behavior. Spend logs and traces keep the request's own session_id. Unset keeps the session_id behavior + */ + conversation_header?: string | null; /** * Custom Dimensions * @description Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, backreferences and lookarounds are rejected. Conservative work limits include alternation paths, repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. Only heuristic, heuristic_first and hybrid accept this field. Uses the existing heuristic tuning quota. diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 35ea64a9c37..7cb861e0e5b 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1592,6 +1592,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, "litellm_gateway_injected_cache": "forged-deployment-id", "_session_deployment_affinity_ttl": 999999, + "_session_deployment_affinity_conversation": "forged-conversation", "internal_call_origin": "autorouter_classifier", "_managed_background_response": True, "_guardrail_pipelines": [{"name": "spoofed"}], @@ -1653,6 +1654,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "routing_decision", "litellm_gateway_injected_cache", "_session_deployment_affinity_ttl", + "_session_deployment_affinity_conversation", "internal_call_origin", "_managed_background_response", "_guardrail_pipelines", diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index dc12775311f..c6b4028d98a 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -7616,7 +7616,7 @@ class _DummyPlugin: class TestChildSessionTierPins: @staticmethod - def _router(session_affinity: bool = False) -> Router: + def _router(session_affinity: bool = False, conversation_header: str | None = None) -> Router: return Router( model_list=[ { @@ -7628,6 +7628,7 @@ class TestChildSessionTierPins: "classification_mode": "user_turn", "session_affinity": session_affinity, "keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}], + **({"conversation_header": conversation_header} if conversation_header else {}), }, }, }, @@ -7674,10 +7675,20 @@ class TestChildSessionTierPins: self, router: Router, surface: str, session_id: str | None, prompt: str | None, expected_tier: str, expected_cause: str, caller: str = "caller-a", child: bool = True, expected_deployment: str | None = None, + ) -> str: + data, bucket = self._normalized_request(surface, session_id, caller, child) + return await self._route_request( + router, surface, data, bucket, prompt, (expected_tier, expected_cause), "parent-session", + expected_deployment, + ) + + @staticmethod + async def _route_request( + router: Router, surface: str, data: dict[str, object], bucket: str, prompt: str | None, + expected_decision: tuple[str, str], expected_log_id: str, expected_deployment: str | None = None, ) -> str: from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck - data, bucket = self._normalized_request(surface, session_id, caller, child) turn: Final[dict[str, object]] = ( {"role": "user", "content": prompt} if prompt is not None else {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "done"}]} @@ -7703,11 +7714,9 @@ class TestChildSessionTierPins: metadata: Final = data[bucket] assert isinstance(metadata, dict) assert (metadata["session_id"], metadata["trace_id"], data["litellm_session_id"], data["litellm_trace_id"]) == ( - "parent-session", "parent-session", "parent-session", "parent-session" - ) - assert (metadata["routing_decision"]["tier"], metadata["routing_decision"]["cause"]) == ( - expected_tier, expected_cause + expected_log_id, expected_log_id, expected_log_id, expected_log_id ) + assert (metadata["routing_decision"]["tier"], metadata["routing_decision"]["cause"]) == expected_decision affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)) if expected_deployment is not None: filtered: Final = await affinity.async_filter_deployments( @@ -7784,6 +7793,196 @@ class TestChildSessionTierPins: router.discard() +class TestConversationHeaderPins: + REASONING: Final = ("REASONING", "literal_keyword_match") + SIMPLE: Final = ("SIMPLE", "heuristic_scorer") + CONTINUED: Final = ("REASONING", "user_turn_continuation") + + @staticmethod + def _request( + surface: str, headers: Mapping[str, str], caller: tuple[str, str] = ("user_api_key_hash", "caller-a") + ) -> tuple[dict[str, object], str]: + from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + metadata_variable_name_for_route, + redact_credential_headers, + ) + + route: Final = { + "chat": "/v1/chat/completions", "responses": "/v1/responses", "messages": "/v1/messages" + }[surface] + bucket: Final = metadata_variable_name_for_route(route) + data: Final[dict[str, object]] = { + bucket: {caller[0]: caller[1], "user_api_key_request_route": route}, + "proxy_server_request": {"headers": redact_credential_headers(headers)}, + } + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=dict(headers), data=data, _metadata_variable_name=bucket + ) + return data, bucket + + async def _route( + self, router: Router, surface: str, headers: Mapping[str, str], prompt: str | None, + expected_decision: tuple[str, str], caller: tuple[str, str] = ("user_api_key_hash", "caller-a"), + expected_deployment: str | None = None, + ) -> str: + data, bucket = self._request(surface, headers, caller) + return await TestChildSessionTierPins._route_request( + router, surface, data, bucket, prompt, expected_decision, + headers.get("x-parent-session-id") or headers["X-Session-Id"], expected_deployment, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("session_affinity", [False, True]) + @pytest.mark.parametrize("surface", ["chat", "responses", "messages"]) + async def test_one_conversation_keeps_its_pins_across_message_ids( + self, surface: str, session_affinity: bool + ) -> None: + router: Final = TestChildSessionTierPins._router(session_affinity, conversation_header=" X-App-Conversation ") + replay: Final = "session_affinity_pin" if session_affinity else "user_turn_continuation" + try: + first: Final = await self._route( + router, surface, {"x-app-conversation": "Ab", "X-Session-Id": "message-0001"}, "reason carefully", + self.REASONING, + ) + await self._route( + router, surface, {"X-App-Conversation": "ab", "X-Session-Id": "message-0001"}, None, self.SIMPLE + ) + assert await self._route( + router, surface, {"X-APP-CONVERSATION": "Ab", "X-Session-Id": "message-0002"}, None, + ("REASONING", replay), expected_deployment=first, + ) == first + finally: + router.discard() + + @pytest.mark.asyncio + @pytest.mark.parametrize("child", [False, True]) + @pytest.mark.parametrize( + "conversation_header,conversation", + [ + ("x-app-conversation", {}), + ("x-app-conversation", {"x-app-conversation": ""}), + ("authorization", {"Authorization": "Bearer test-only"}), + ], + ids=["absent", "empty", "redacted"], + ) + async def test_configured_header_without_a_usable_value_disables_reuse( + self, conversation_header: str, conversation: Mapping[str, str], child: bool + ) -> None: + from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck + + router: Final = TestChildSessionTierPins._router(conversation_header=conversation_header) + headers: Final = { + **conversation, + "X-Session-Id": "message-0001", + **({"x-parent-session-id": "parent-session"} if child else {}), + } + try: + first: Final = await self._route(router, "chat", headers, "reason carefully", self.REASONING) + data, bucket = self._request("chat", headers) + await TestChildSessionTierPins._route_request( + router, "chat", data, bucket, None, self.SIMPLE, "parent-session" if child else "message-0001" + ) + affinity: Final = next( + cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck) + ) + first_group: Final = first.rsplit("-", 1)[0] + filtered: Final = await affinity.async_filter_deployments( + model=first_group, healthy_deployments=router.get_model_list(first_group) or [], + messages=None, request_kwargs=data, + ) + assert sorted(deployment["model_info"]["id"] for deployment in filtered) == [ + f"{first_group}-0", f"{first_group}-1" + ] + finally: + router.discard() + + @pytest.mark.asyncio + async def test_configured_children_stay_inside_their_conversation(self) -> None: + router: Final = TestChildSessionTierPins._router(conversation_header="x-app-conversation") + child: Final = {"x-app-conversation": "c1", "x-parent-session-id": "parent-0001", "X-Session-Id": "child-000a"} + try: + parent: Final = await self._route( + router, "chat", {"x-app-conversation": "c1", "X-Session-Id": "parent-0001"}, "reason carefully", + self.REASONING, + ) + await self._route(router, "chat", child, "Hello!", self.SIMPLE) + await self._route(router, "chat", child, None, ("SIMPLE", "user_turn_continuation")) + assert await self._route( + router, "chat", {"x-app-conversation": "c1", "X-Session-Id": "parent-0002"}, None, self.CONTINUED, + expected_deployment=parent, + ) == parent + await self._route( + router, "chat", {**child, "X-Session-Id": "parent-0001"}, None, self.SIMPLE + ) + finally: + router.discard() + + @pytest.mark.asyncio + @pytest.mark.parametrize("identity", ["user_api_key_hash", "user_api_key_user_id"]) + async def test_callers_sharing_a_conversation_id_never_share_pins(self, identity: str) -> None: + router: Final = TestChildSessionTierPins._router(conversation_header="x-app-conversation") + try: + first: Final = await self._route( + router, "chat", {"x-app-conversation": "shared", "X-Session-Id": "message-0001"}, "reason carefully", + self.REASONING, caller=(identity, "tenant-a"), + ) + await self._route( + router, "chat", {"x-app-conversation": "shared", "X-Session-Id": "message-0002"}, None, self.SIMPLE, + caller=(identity, "tenant-b"), + ) + assert await self._route( + router, "chat", {"x-app-conversation": "shared", "X-Session-Id": "message-0003"}, None, + self.CONTINUED, caller=(identity, "tenant-a"), expected_deployment=first, + ) == first + finally: + router.discard() + + @pytest.mark.asyncio + async def test_unset_header_keeps_session_id_pins_under_the_key_hash(self) -> None: + from litellm.redis.stores.router_session_pins import TierSessionPin + + router: Final = TestChildSessionTierPins._router() + strategy: Final = router.complexity_routers["child-tier-router"][0].strategy + try: + await strategy.ctx.health().session_pins.set( + TierSessionPin(router="child-tier-router", caller="caller-a", session_id="message-0001"), + {"model": "reasoning", "tier": "REASONING"}, + 60, + ) + replayed: Final = await self._route( + router, "chat", {"x-app-conversation": "Ab", "X-Session-Id": "message-0001"}, None, self.CONTINUED + ) + fresh: Final = await self._route( + router, "chat", {"x-app-conversation": "Ab", "X-Session-Id": "message-0002"}, None, self.SIMPLE + ) + assert (replayed.rsplit("-", 1)[0], fresh.rsplit("-", 1)[0]) == ("reasoning", "simple") + finally: + router.discard() + + @pytest.mark.asyncio + async def test_tier_model_pin_follows_the_conversation(self) -> None: + router: Final = TestChildSessionTierPins._router(conversation_header="x-app-conversation") + strategy: Final = router.complexity_routers["child-tier-router"][0].strategy + candidates: Final = ("reasoning", "reasoning-peer") + try: + for headers, offered, pinned in ( + ({"X-Session-Id": "message-0001"}, "reasoning-peer", "reasoning-peer"), + ({"X-Session-Id": "message-0001"}, "reasoning", "reasoning"), + ({"x-app-conversation": "Ab", "X-Session-Id": "message-0001"}, "reasoning", "reasoning"), + ({"x-app-conversation": "Ab", "X-Session-Id": "message-0002"}, "reasoning-peer", "reasoning"), + ({"x-app-conversation": "ab", "X-Session-Id": "message-0001"}, "reasoning-peer", "reasoning-peer"), + ): + request, _ = self._request("chat", headers) + assert await strategy._pin_model_for_tier("REASONING", offered, candidates, request) == pinned + finally: + router.discard() + + def test_conversation_header_must_name_a_header(self) -> None: + with pytest.raises(ValidationError, match="conversation_header"): + ComplexityRouterConfig.model_validate({"conversation_header": " "}) + + class TestClaudeCodeTierPins: SESSION: Final = "11111111-2222-3333-4444-555555555555" AGENT_A: Final = "a7256c04dfd99c9b9" diff --git a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py index 08688bc141a..6b01348fd45 100644 --- a/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py +++ b/tests/unit/router_utils/pre_call_checks/test_session_id_affinity.py @@ -16,6 +16,7 @@ from litellm.redis.stores.router_session_pins import ( TierModelPin, ) from litellm.constants import ( + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY, ) @@ -392,7 +393,9 @@ MOCK_RESPONSES_API_RESPONSE = { } -def _smart_router(session_affinity=True, ttl_seconds=777, deployment_affinity=True): +def _smart_router( + session_affinity=True, ttl_seconds=777, deployment_affinity=True, conversation_header: str | None = None +): return litellm.Router( model_list=[ { @@ -404,6 +407,7 @@ def _smart_router(session_affinity=True, ttl_seconds=777, deployment_affinity=Tr "session_affinity": session_affinity, "deployment_affinity": deployment_affinity, "session_affinity_ttl_seconds": ttl_seconds, + **({"conversation_header": conversation_header} if conversation_header else {}), "tiers": { "SIMPLE": "target-group", "MEDIUM": "target-group", @@ -814,6 +818,82 @@ async def test_pre_routing_hook_stamps_or_clears_the_marker_per_attempt( _cleanup_router_callbacks(router) +async def _conversation_turn( + router: litellm.Router, model: str, headers: Mapping[str, str], metadata: Mapping[str, object] +) -> tuple[list[str], Mapping[str, object]]: + request_kwargs: Final[dict[str, object]] = { + "proxy_server_request": {"headers": headers}, + "metadata": {**metadata, "user_api_key_hash": "key-1"}, + } + await router.async_pre_routing_hook( + model=model, request_kwargs=request_kwargs, messages=[{"role": "user", "content": "Hello"}] + ) + affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)) + filtered: Final = await affinity.async_filter_deployments( + model="target-group", + healthy_deployments=router.get_model_list("target-group") or [], + messages=None, + request_kwargs=request_kwargs, + ) + routed: Final = request_kwargs["metadata"] + assert isinstance(routed, dict) + await affinity.async_pre_call_deployment_hook( + kwargs={ + **request_kwargs, + "model_info": {"id": "deployment-2"}, + "metadata": {**routed, "deployment_model_name": "target-group"}, + }, + call_type=None, + ) + return sorted(str(deployment["model_info"]["id"]) for deployment in filtered), routed + + +@pytest.mark.asyncio +async def test_configured_conversation_header_owns_the_deployment_pin() -> None: + router: Final = _smart_router(conversation_header="x-app-conversation") + both: Final = ["deployment-1", "deployment-2"] + try: + affinity: Final = next(cb for cb in router.optional_callbacks or () if isinstance(cb, DeploymentAffinityCheck)) + await affinity.async_pre_call_deployment_hook( + kwargs={ + "model_info": {"id": "deployment-1"}, + "metadata": { + "deployment_model_name": "target-group", + "session_id": "message-1", + "user_api_key_hash": "key-1", + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 777, + }, + }, + call_type=None, + ) + first, routed = await _conversation_turn( + router, "smart-router", {"x-app-conversation": "Ab"}, {"session_id": "message-1"} + ) + conversation: Final = routed[SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY] + forged: Final = { + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: 777, + SESSION_DEPLOYMENT_AFFINITY_CONVERSATION_METADATA_KEY: conversation, + } + same_conversation, _ = await _conversation_turn( + router, "smart-router", {"x-app-conversation": "Ab"}, {"session_id": "message-2"} + ) + other_conversation, _ = await _conversation_turn( + router, "smart-router", {"x-app-conversation": "ab"}, {"session_id": "message-2"} + ) + unusable_headers: Final = [ + (await _conversation_turn(router, "smart-router", headers, {**forged, "session_id": "message-1"}))[0] + for headers in ({}, {"x-app-conversation": ""}) + ] + rerouted, _ = await _conversation_turn( + router, "target-group", {"x-app-conversation": "Ab"}, {**forged, "session_id": "message-2"} + ) + assert (first, same_conversation, other_conversation, unusable_headers, rerouted) == ( + both, ["deployment-2"], both, [both, both], both + ) + finally: + _cleanup_router_callbacks(router) + + def test_complexity_router_with_deployment_affinity_registers_affinity_callback(): enabled = _smart_router() session_only = _smart_router(session_affinity=True, deployment_affinity=False)