mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): drop reasoning ids a Codex turn's LiteLLM target cannot resolve
Some checks are pending
Some checks are pending
A reasoning item another deployment produced, replayed with store false and no blob the target can decrypt, now reaches the target as its readable summary without id or status, and is dropped when nothing readable is left. OpenAI answered 400 on the status field and 404 on the id for a glm-5p3 item replayed whole to an openai deployment. Same-deployment replays and stored conversations keep the item whole
This commit is contained in:
parent
5611997f6a
commit
69ceef4c33
3 changed files with 126 additions and 2 deletions
|
|
@ -41,6 +41,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_thr
|
|||
from litellm.proxy.response_api_endpoints.endpoints import responses_api
|
||||
from litellm.responses.additional_tools import with_additional_tools_hoisted
|
||||
from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import EncryptedContentAffinityCheck
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
from litellm.types.router import Deployment
|
||||
|
|
@ -359,7 +360,16 @@ def _with_reasoning_the_targets_can_decrypt(
|
|||
EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt(
|
||||
llm_router, kept, None, group, unmarked_origin=None
|
||||
)
|
||||
return {**body, "input": kept}
|
||||
if body.get("store") is True:
|
||||
return {**body, "input": kept}
|
||||
resolvable: Final = (_without_unpersisted_reasoning_reference(item) for item in kept)
|
||||
return {**body, "input": [item for item in resolvable if item is not None]}
|
||||
|
||||
|
||||
def _without_unpersisted_reasoning_reference(item: object) -> object | None:
|
||||
if not _is_unpersisted_reasoning_reference(item):
|
||||
return item
|
||||
return ResponsesAPIRequestUtils.readable_reasoning(_JSON_OBJECT.validate_python(item))
|
||||
|
||||
|
||||
def _model_and_provider(deployment: Deployment) -> tuple[str, str] | None:
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from litellm.types.utils import (
|
|||
_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[
|
||||
TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]
|
||||
] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None)
|
||||
_REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES: Final = frozenset({"encrypted_content", "id", "status"})
|
||||
|
||||
|
||||
class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object):
|
||||
|
|
@ -685,13 +686,17 @@ class ResponsesAPIRequestUtils:
|
|||
reasoning: Final = cast(Mapping[str, object], item) # cast-ok: untyped client json
|
||||
if reasoning.get("type") != "reasoning" or not reasoning.get("encrypted_content"):
|
||||
return reasoning
|
||||
return ResponsesAPIRequestUtils.readable_reasoning(reasoning)
|
||||
|
||||
@staticmethod
|
||||
def readable_reasoning(reasoning: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
readable: Final = any(
|
||||
ResponsesAPIRequestUtils._has_readable_text(reasoning.get(key)) for key in ("summary", "content")
|
||||
)
|
||||
if not readable:
|
||||
return None
|
||||
kept: Final[dict[str, object]] = { # mutable-ok: request item rebuilt without the undecryptable keys
|
||||
key: value for key, value in reasoning.items() if key not in ("encrypted_content", "id")
|
||||
key: value for key, value in reasoning.items() if key not in _REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES
|
||||
}
|
||||
return kept
|
||||
|
||||
|
|
|
|||
|
|
@ -1159,6 +1159,115 @@ def test_litellm_model_on_native_route_gets_only_the_reasoning_it_can_decrypt(
|
|||
)
|
||||
|
||||
|
||||
_WHOLE_FIREWORKS_REASONING: Final = {**_FIREWORKS_REASONING, "status": "completed"}
|
||||
_SUMMARY_ONLY_REASONING: Final = {key: value for key, value in _FIREWORKS_REASONING.items() if key != "id"}
|
||||
_OTHER_DEPLOYMENTS_BLOB: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(
|
||||
"gAAAAB-other-deployment-blob", "other-deployment"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "replayed", "store", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"oai-luna", _WHOLE_FIREWORKS_REASONING, False, [_SUMMARY_ONLY_REASONING], id="whole fireworks item to openai"
|
||||
),
|
||||
pytest.param(
|
||||
"oai-luna",
|
||||
{**_WHOLE_FIREWORKS_REASONING, "encrypted_content": _OTHER_DEPLOYMENTS_BLOB},
|
||||
False,
|
||||
[_SUMMARY_ONLY_REASONING],
|
||||
id="whole item with another deployment's blob to openai",
|
||||
),
|
||||
pytest.param(
|
||||
"oai-luna",
|
||||
{**_SUMMARY_ONLY_REASONING, "content": [], "encrypted_content": _OTHER_DEPLOYMENTS_BLOB},
|
||||
False,
|
||||
[{**_SUMMARY_ONLY_REASONING, "content": []}],
|
||||
id="codex's replay of another deployment's item to openai",
|
||||
),
|
||||
pytest.param(
|
||||
"oai-luna",
|
||||
{**_SUMMARY_ONLY_REASONING, "content": None, "encrypted_content": None},
|
||||
False,
|
||||
[_SUMMARY_ONLY_REASONING],
|
||||
id="codex's replay of a blobless item to openai",
|
||||
),
|
||||
pytest.param(
|
||||
"oai-luna",
|
||||
{"type": "reasoning", "id": "rs_unreadable", "summary": []},
|
||||
False,
|
||||
[],
|
||||
id="an unresolvable id with nothing readable is dropped",
|
||||
),
|
||||
pytest.param(
|
||||
"open-weights",
|
||||
{**_SUMMARY_ONLY_REASONING, "id": "rs_openai"},
|
||||
False,
|
||||
[_SUMMARY_ONLY_REASONING],
|
||||
id="openai summary-only item to fireworks",
|
||||
),
|
||||
pytest.param(
|
||||
"oai-luna",
|
||||
{**_LUNA_REASONING, "id": "rs_luna", "content": []},
|
||||
False,
|
||||
[{**_LUNA_REASONING, "id": "rs_luna", "content": [], "encrypted_content": _LUNA_BLOB}],
|
||||
id="the minting deployment gets its item whole",
|
||||
),
|
||||
pytest.param(
|
||||
"oai-luna", _FIREWORKS_REASONING, True, [_FIREWORKS_REASONING], id="a stored conversation keeps the id"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_reasoning_the_target_cannot_resolve_reaches_it_without_its_id(
|
||||
client: TestClient,
|
||||
respx_mock: respx.MockRouter,
|
||||
xai_fireworks_and_openai_router: Router,
|
||||
model: str,
|
||||
replayed: dict[str, object],
|
||||
store: bool,
|
||||
expected: list[dict[str, object]],
|
||||
) -> None:
|
||||
_account_catalog(respx_mock)
|
||||
fireworks: Final = _open_weights_provider(respx_mock)
|
||||
luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply)
|
||||
response: Final = client.post(
|
||||
"/backend-api/codex/responses",
|
||||
json={
|
||||
"model": model,
|
||||
"input": [_USER_TURN, replayed, _ASSISTANT_TURN, _USER_TURN],
|
||||
"stream": False,
|
||||
"store": store,
|
||||
},
|
||||
headers=_ACCOUNT_HEADERS,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
sent: Final = json.loads((fireworks if model == "open-weights" else luna).calls.last.request.content)["input"]
|
||||
assert [item for item in sent if item.get("type") == "reasoning"] == expected
|
||||
|
||||
|
||||
def test_reasoning_the_chat_bridge_target_cannot_resolve_reaches_it_as_its_summary(
|
||||
client: TestClient, respx_mock: respx.MockRouter, xai_fireworks_and_openai_router: Router
|
||||
) -> None:
|
||||
_account_catalog(respx_mock)
|
||||
chat_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/chat/completions").mock(side_effect=_xai_search_call_reply)
|
||||
response: Final = client.post(
|
||||
"/backend-api/codex/responses",
|
||||
json={
|
||||
"model": "grok-4.7",
|
||||
"input": [_USER_TURN, _WHOLE_FIREWORKS_REASONING, _ASSISTANT_TURN, _USER_TURN],
|
||||
"stream": False,
|
||||
"store": False,
|
||||
},
|
||||
headers=_ACCOUNT_HEADERS,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
messages: Final = json.loads(chat_endpoint.calls.last.request.content)["messages"]
|
||||
assert [message.get("reasoning_content") for message in messages if message["role"] == "assistant"] == [
|
||||
"Thinking it over"
|
||||
]
|
||||
|
||||
|
||||
def _namespace(name: str, *members: str) -> dict[str, object]:
|
||||
return {
|
||||
"type": "namespace",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue