mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(router): keep encrypted reasoning when previous_response_id binds the failed origin
This commit is contained in:
parent
3bc4a58ce7
commit
916e933fd9
2 changed files with 47 additions and 6 deletions
|
|
@ -385,11 +385,20 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
model_id,
|
||||
)
|
||||
|
||||
excluded_ids: Final = frozenset(
|
||||
as_excluded_deployment_ids(typed_request_kwargs.get("_excluded_deployment_ids"))
|
||||
previous_response_id: Final = typed_request_kwargs.get("previous_response_id")
|
||||
previous_response_origin: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(
|
||||
None if previous_response_id is None else str(previous_response_id)
|
||||
)
|
||||
retry_skipped_ids: Final = frozenset(
|
||||
as_retry_skipped_deployment_ids(typed_request_kwargs.get("_retry_skipped_deployment_ids"))
|
||||
previous_response_binds_origin: Final = previous_response_origin == model_id
|
||||
excluded_ids: Final[frozenset[str]] = (
|
||||
frozenset()
|
||||
if previous_response_binds_origin
|
||||
else frozenset(as_excluded_deployment_ids(typed_request_kwargs.get("_excluded_deployment_ids")))
|
||||
)
|
||||
retry_skipped_ids: Final[frozenset[str]] = (
|
||||
frozenset()
|
||||
if previous_response_binds_origin
|
||||
else frozenset(as_retry_skipped_deployment_ids(typed_request_kwargs.get("_retry_skipped_deployment_ids")))
|
||||
)
|
||||
not_excluded_candidates: Final = [
|
||||
candidate for candidate in candidates if self._deployment_model_id(candidate) not in excluded_ids
|
||||
|
|
|
|||
|
|
@ -2633,6 +2633,10 @@ def _reasoning_input_minted_by(origin: str) -> list[dict[str, object]]:
|
|||
]
|
||||
|
||||
|
||||
def _previous_response_id_from(origin: str) -> str:
|
||||
return ResponsesAPIRequestUtils._build_responses_api_response_id("openai", origin, "resp_test")
|
||||
|
||||
|
||||
_ORIGIN_FAILED_THIS_REQUEST: Final = pytest.mark.parametrize(
|
||||
"failed_in_request",
|
||||
[{"_excluded_deployment_ids": ["region-a"]}, {"_retry_skipped_deployment_ids": ("region-a",)}],
|
||||
|
|
@ -2641,8 +2645,9 @@ _ORIGIN_FAILED_THIS_REQUEST: Final = pytest.mark.parametrize(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@_ORIGIN_FAILED_THIS_REQUEST
|
||||
@pytest.mark.parametrize("continuing", [{}, {"previous_response_id": _previous_response_id_from("region-b")}])
|
||||
async def test_affinity_strips_and_dispatches_when_origin_already_failed_this_request(
|
||||
failed_in_request: dict[str, object],
|
||||
failed_in_request: dict[str, object], continuing: dict[str, object]
|
||||
):
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
|
|
@ -2656,13 +2661,40 @@ async def test_affinity_strips_and_dispatches_when_origin_already_failed_this_re
|
|||
model="gpt-5.4",
|
||||
healthy_deployments=group,
|
||||
messages=None,
|
||||
request_kwargs={"litellm_metadata": {}, "input": reasoning_input, **failed_in_request},
|
||||
request_kwargs={"litellm_metadata": {}, "input": reasoning_input, **failed_in_request, **continuing},
|
||||
)
|
||||
|
||||
assert result is group
|
||||
assert reasoning_input[0] == {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_ORIGIN_FAILED_THIS_REQUEST
|
||||
async def test_affinity_keeps_origin_and_reasoning_when_previous_response_id_binds_the_failed_origin(
|
||||
failed_in_request: dict[str, object],
|
||||
):
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
pool: Final = (_region_deployment("region-a"), _region_deployment("region-b"))
|
||||
reasoning_input: Final = _reasoning_input_minted_by("region-a")
|
||||
original_reasoning: Final = dict(reasoning_input[0])
|
||||
|
||||
assert await EncryptedContentAffinityCheck(router=litellm.Router(model_list=list(pool))).async_filter_deployments(
|
||||
model="gpt-5.4",
|
||||
healthy_deployments=list(pool),
|
||||
messages=None,
|
||||
request_kwargs={
|
||||
"litellm_metadata": {},
|
||||
"input": reasoning_input,
|
||||
"previous_response_id": _previous_response_id_from("region-a"),
|
||||
**failed_in_request,
|
||||
},
|
||||
) == [pool[0]]
|
||||
assert reasoning_input[0] == original_reasoning
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("regions", [("region-a",), ("region-a", "region-b")])
|
||||
async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_the_request(regions: tuple[str, ...]):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue