mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(codex): reach encrypted reasoning stripping through the router registry
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
The Codex route read Router.optional_callbacks and imported EncryptedContentAffinityCheck from litellm.router_utils, which the router surface fence forbids outside the router. RouterRegistry gains decryptable_reasoning(model_group, request_input, targets), which keeps the same rule: with encrypted content affinity on for the group, reasoning one target can decrypt stays, otherwise every target must decrypt it
This commit is contained in:
parent
91adba8212
commit
0ad76f27da
3 changed files with 105 additions and 18 deletions
|
|
@ -43,7 +43,6 @@ from litellm.proxy.response_api_endpoints.endpoints import responses_api
|
|||
from litellm.responses.additional_tools import with_additional_tools_hoisted
|
||||
from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
|
@ -339,13 +338,6 @@ def _with_wrapped_reasoning(body: Mapping[str, object]) -> Mapping[str, object]:
|
|||
}
|
||||
|
||||
|
||||
def _router_pins_encrypted_reasoning(model: str, llm_router: "Router") -> bool:
|
||||
return any(
|
||||
isinstance(callback, EncryptedContentAffinityCheck) and callback.is_enabled_for_model_group(model)
|
||||
for callback in llm_router.optional_callbacks or ()
|
||||
)
|
||||
|
||||
|
||||
def _with_reasoning_the_targets_can_decrypt(
|
||||
body: Mapping[str, object], model: str, llm_router: "Router", deployments: tuple[Deployment, ...]
|
||||
) -> Mapping[str, object]:
|
||||
|
|
@ -353,17 +345,9 @@ def _with_reasoning_the_targets_can_decrypt(
|
|||
items: Final = _INPUT_ITEMS.validate_python(body.get("input"))
|
||||
except ValidationError:
|
||||
return body
|
||||
targets: Final = tuple(deployment.model_dump(exclude_none=True) for deployment in deployments)
|
||||
target_groups: Final = (
|
||||
(targets,) if _router_pins_encrypted_reasoning(model, llm_router) else tuple((target,) for target in targets)
|
||||
)
|
||||
kept: Final = list(items)
|
||||
for group in target_groups:
|
||||
EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
|
||||
llm_router, kept, None, group, unmarked_origin=None
|
||||
)
|
||||
kept: Final = llm_router.registry().decryptable_reasoning(model, items, deployments)
|
||||
if body.get("store") is True:
|
||||
return {**body, "input": kept}
|
||||
return {**body, "input": list(kept)}
|
||||
resolvable: Final = (_without_unpersisted_reasoning_reference(item) for item in kept)
|
||||
return {**body, "input": [item for item in resolvable if item is not None]}
|
||||
|
||||
|
|
|
|||
|
|
@ -931,6 +931,34 @@ class RouterRegistry:
|
|||
return r._core_prompt_caching_pin_cache
|
||||
raise ValueError(f"unknown plugin kind {kind!r}")
|
||||
|
||||
def decryptable_reasoning(
|
||||
self, model_group: str, request_input: Sequence[object], targets: "Sequence[Deployment]"
|
||||
) -> tuple[object, ...]:
|
||||
"""
|
||||
``request_input`` without the encrypted reasoning that ``targets``, the deployments a
|
||||
``model_group`` request may go to, cannot decrypt, keeping each item's readable summary.
|
||||
When encrypted content affinity is on for the group, the router sends the request to the
|
||||
reasoning's origin, so reasoning one target can decrypt stays; otherwise every target must
|
||||
decrypt it (was reading ``optional_callbacks`` for ``EncryptedContentAffinityCheck``).
|
||||
"""
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
r: Final = self._router
|
||||
target_dicts: Final = tuple(target.model_dump(exclude_none=True) for target in targets)
|
||||
pinned: Final = any(
|
||||
isinstance(callback, EncryptedContentAffinityCheck) and callback.is_enabled_for_model_group(model_group)
|
||||
for callback in r.optional_callbacks or ()
|
||||
)
|
||||
target_groups: Final = (target_dicts,) if pinned else tuple((target,) for target in target_dicts)
|
||||
kept: Final = list(request_input)
|
||||
for group in target_groups:
|
||||
EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
|
||||
r, kept, None, group, unmarked_origin=None
|
||||
)
|
||||
return tuple(kept)
|
||||
|
||||
|
||||
# Settings a caller may read. Assigning one from outside the router goes through apply(ChangeSet(assign=...)).
|
||||
SETTING_FIELDS: Final[tuple[str, ...]] = (
|
||||
|
|
|
|||
|
|
@ -827,3 +827,78 @@ def test_attribute_errors_about_other_objects_pass_through_unchanged(router):
|
|||
|
||||
other = AttributeError("'dict' object has no attribute 'x'", name="x", obj={})
|
||||
assert _public_attribute_error(router, "_core_x", "x", other) is other
|
||||
|
||||
|
||||
_MINTING_DEPLOYMENT = "dep-minting-org"
|
||||
_OTHER_ORG_DEPLOYMENT = "dep-other-org"
|
||||
_BOTH_ORGS = (_MINTING_DEPLOYMENT, _OTHER_ORG_DEPLOYMENT)
|
||||
_GROUP_AFFINITY = ["encrypted_content_affinity"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"affinity, target_ids, reasoning_stays",
|
||||
[
|
||||
pytest.param({}, _BOTH_ORGS, False, id="no affinity, one target cannot decrypt"),
|
||||
pytest.param({}, (_MINTING_DEPLOYMENT,), True, id="no affinity, the only target minted it"),
|
||||
pytest.param(
|
||||
{"optional_pre_call_checks": _GROUP_AFFINITY}, _BOTH_ORGS, True, id="global affinity, origin among targets"
|
||||
),
|
||||
pytest.param(
|
||||
{"optional_pre_call_checks": _GROUP_AFFINITY},
|
||||
(_OTHER_ORG_DEPLOYMENT,),
|
||||
False,
|
||||
id="global affinity, origin not among targets",
|
||||
),
|
||||
pytest.param(
|
||||
{"optional_pre_call_checks": [], "model_group_affinity_config": {"pair": _GROUP_AFFINITY}},
|
||||
_BOTH_ORGS,
|
||||
True,
|
||||
id="affinity on for this group",
|
||||
),
|
||||
pytest.param(
|
||||
{"optional_pre_call_checks": [], "model_group_affinity_config": {"other-group": _GROUP_AFFINITY}},
|
||||
_BOTH_ORGS,
|
||||
False,
|
||||
id="affinity on for another group only",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_decryptable_reasoning_keeps_encrypted_reasoning_only_when_the_router_sends_it_to_a_deployment_that_decrypts_it(
|
||||
affinity, target_ids, reasoning_stays
|
||||
):
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
|
||||
r = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "pair",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_base": api_base, "api_key": "sk-pair"},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
for api_base, model_id in (
|
||||
("https://minting-org.test/v1", _MINTING_DEPLOYMENT),
|
||||
("https://other-org.test/v1", _OTHER_ORG_DEPLOYMENT),
|
||||
)
|
||||
],
|
||||
**affinity,
|
||||
)
|
||||
summary = [{"type": "summary_text", "text": "multiply 17 by 23"}]
|
||||
user_turn = {"type": "message", "role": "user", "content": "What is 17*23?"}
|
||||
reasoning = {
|
||||
"type": "reasoning",
|
||||
"id": "rs_minted",
|
||||
"encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(
|
||||
"gAAAAA-blob", _MINTING_DEPLOYMENT
|
||||
),
|
||||
"summary": summary,
|
||||
}
|
||||
request_input = [user_turn, reasoning]
|
||||
targets = tuple(r.registry().deployment(model_id) for model_id in target_ids)
|
||||
try:
|
||||
kept = r.registry().decryptable_reasoning("pair", request_input, targets)
|
||||
finally:
|
||||
r.retire()
|
||||
|
||||
readable_only = {"type": "reasoning", "summary": summary}
|
||||
assert kept == (user_turn, reasoning if reasoning_stays else readable_only)
|
||||
assert request_input == [user_turn, reasoning], "the caller's input must not change"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue