scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-31 06:56:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 245

Merge 74d9da7a20 into afec145edc

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-10-30 22:52:48 -07:00 • committed by GitHub
commit ea6e1108ef
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 909 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
litellm/llms/openai/responses/transformation.py
View file

@ -15,7 +15,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
openai_prompt_cache_breakpoint_support,
targets_openai_api,
)
from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, process_response_headers
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
safe_convert_created_field,
@ -44,6 +44,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import (
tools_with_anthropic_tool_search_mapped,
tools_without_anthropic_tool_search,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import *
@ -623,11 +624,45 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
"model": model,
"input": replay_safe_input,
**response_api_optional_request_params,
**self._prompt_cache_options_with_original_comparison_id(request_params),
},
)
return final_request_params
def transform_extra_body(
self,
extra_body: Mapping[str, object],
request: Mapping[str, object],
model: str,
litellm_params: GenericLiteLLMParams,
) -> Mapping[str, object]:
return {**extra_body, **self._prompt_cache_options_with_original_comparison_id(extra_body)}
def _prompt_cache_options_with_original_comparison_id(self, params: Mapping[str, object]) -> Mapping[str, object]:
options: Final = as_str_mapping(params.get("prompt_cache_options"))
comparison_response_id: Final = None if options is None else options.get("comparison_response_id")
if (
self.custom_llm_provider != LlmProviders.OPENAI
or options is None
or not isinstance(comparison_response_id, str)
or not comparison_response_id.startswith("resp_")
):
return {}
decoded_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(comparison_response_id)
decoded_provider: Final = decoded_id.get("custom_llm_provider")
original_response_id: Final = decoded_id.get("response_id", comparison_response_id)
rebuilt_envelope: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
{"id": original_response_id}, decoded_provider, {"model_info": {"id": decoded_id.get("model_id")}}
)
if (
decoded_provider != LlmProviders.OPENAI
or original_response_id == comparison_response_id
or rebuilt_envelope.get("id") != comparison_response_id
):
return {}
return {"prompt_cache_options": {**options, "comparison_response_id": original_response_id}}
def _prepared_input_and_tools(
self,
model: str,

53
litellm/proxy/hooks/responses_id_security.py
View file

@ -14,6 +14,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.proxy._types import LitellmUserRoles
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
@ -23,7 +24,13 @@ from litellm.types.llms.openai import (
BaseLiteLLMOpenAIResponseObject,
ResponsesAPIResponse,
)
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums
from litellm.types.utils import (
ADDRESSED_COMPARISON_RESPONSE_ID_FIELD,
ADDRESSED_RESPONSE_ID_FIELD,
CallTypesLiteral,
LLMResponseTypes,
SpecialEnums,
)
if TYPE_CHECKING:
from litellm.caching.caching import DualCache
@ -135,6 +142,9 @@ class ResponsesIDSecurity(CustomLogger):
addressed_id_field: Final = _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE.get(call_type)
if addressed_id_field is None:
return None
if call_type == "aresponses":
request_data: Final = cast("dict[str, object]", data) # cast-ok: the hook signature leaves the body untyped
request_data.update(self._authorized_comparison_response_id_fields(request_data, user_api_key_dict))
retained_id: Final = data.get(ADDRESSED_RESPONSE_ID_FIELD)
addressed_id: Final = (
retained_id if isinstance(retained_id, str) and retained_id else data.get(addressed_id_field)
@ -146,6 +156,47 @@ class ResponsesIDSecurity(CustomLogger):
data[ADDRESSED_RESPONSE_ID_FIELD] = addressed_id
return data
def _authorized_comparison_response_id_fields(
self,
data: Mapping[str, object],
user_api_key_dict: "UserAPIKeyAuth",
) -> Mapping[str, object]:
extra_body: Final = as_str_mapping(data.get("extra_body"))
overriding_extra_body: Final = (
extra_body if extra_body is not None and "prompt_cache_options" in extra_body else None
)
options: Final = as_str_mapping(
data.get("prompt_cache_options")
if overriding_extra_body is None
else overriding_extra_body.get("prompt_cache_options")
)
requested_id: Final = None if options is None else options.get("comparison_response_id")
if options is None or not isinstance(requested_id, str):
return {}
addressed_id: Final = self._comparison_addressed_id(
requested_id, data.get(ADDRESSED_COMPARISON_RESPONSE_ID_FIELD)
)
authorized_options: Final = {
**options,
"comparison_response_id": self.authorize_response_id(addressed_id, user_api_key_dict),
}
authorized_field: Final[Mapping[str, object]] = (
{"prompt_cache_options": authorized_options}
if overriding_extra_body is None
else {"extra_body": {**overriding_extra_body, "prompt_cache_options": authorized_options}}
)
return {**authorized_field, ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: addressed_id}
def _comparison_addressed_id(self, requested_id: str, retained_id: object) -> str:
if (
not isinstance(retained_id, str)
or self._is_encrypted_response_id(requested_id)
or not self._is_encrypted_response_id(retained_id)
):
return requested_id
original_response_id, _, _ = self._decrypt_response_id(retained_id)
return retained_id if original_response_id == requested_id else requested_id
def authorize_response_id(
self,
response_id: str,

4
litellm/types/litellm_params.py
View file

@ -55,6 +55,7 @@ RoutingStrategyName: TypeAlias = Literal[
TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars"
ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id"
ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: Final = "_litellm_addressed_comparison_response_id"
STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected"
WIRE_NAME: Final = "wire_name"
@ -406,6 +407,9 @@ class ProxyRequestState:
secret_fields: "SecretFields | None" = None
trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD))
addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD))
addressed_comparison_response_id: str | None = field(
default=None, metadata=wire(ADDRESSED_COMPARISON_RESPONSE_ID_FIELD)
)
strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage"))
stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD))
client_side_timeout: bool | None = None

1
litellm/types/llms/openai.py
View file

@ -689,6 +689,7 @@ class PromptCacheBreakpoint(TypedDict):
class PromptCacheOptions(TypedDict, total=False):
mode: ReadOnly[Literal["implicit", "explicit"]]
ttl: ReadOnly[Literal["30m", "12h"]]
comparison_response_id: ReadOnly[str]
class ChatCompletionThinkingBlock(TypedDict, total=False):

1
litellm/types/utils.py
View file

@ -4334,6 +4334,7 @@ bedrock_batch_litellm_params: Final = BEDROCK_BATCH_KWARG_NAMES
TRUSTED_CALLBACK_VARS_FIELD: Final = _litellm_params.TRUSTED_CALLBACK_VARS_FIELD
ADDRESSED_RESPONSE_ID_FIELD: Final = _litellm_params.ADDRESSED_RESPONSE_ID_FIELD
ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: Final = _litellm_params.ADDRESSED_COMPARISON_RESPONSE_ID_FIELD
anthropic_wif_litellm_params: Final = tuple(sorted(ANTHROPIC_WIF_KWARGS_KEYS))
openai_wif_litellm_params: Final = tuple(sorted(OPENAI_WIF_KWARGS_KEYS))

382
tests/integration/authorization/test_responses_id_ownership.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
import json
import os
import uuid
from collections.abc import Callable
from collections.abc import Callable, Iterator, Mapping
from pathlib import Path
from typing import Final, Literal
from urllib.parse import urlsplit
@ -16,7 +16,7 @@ from integration._support.process import owned_proxy
from integration._support.responses_vendor import same_response
from integration._support.wire import Reply, Request, Wire, wire_server
from integration._support.worker_readiness import wait_for_worker_readiness
from openai.types.responses import ResponseInputParam
from openai.types.responses import ResponseCompletedEvent, ResponseInputParam
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.shares_worker_state
@ -51,12 +51,6 @@ class _DeleteResponse(BaseModel):
deleted: bool
def _deployments_in_group(gateway: Gateway, group: str) -> int:
entries: Final = gateway.get("/model/info")["data"]
assert isinstance(entries, list), entries
return sum(object_value(entry).get("model_name") == group for entry in entries)
def _wait_for_workers(gateway: Gateway, group: str, deployments: int) -> None:
wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS)
@ -138,6 +132,378 @@ def _every_id_route(client: openai.OpenAI, model: str, response_id: str) -> tupl
)
def _comparison_body(response_id: str, nested: bool) -> Mapping[str, object]:
options: Final = {"comparison_response_id": response_id}
if nested:
return {
"prompt_cache_options": {"mode": "explicit", "ttl": "12h"},
"extra_body": {"prompt_cache_options": options},
}
return {"prompt_cache_options": options}
@pytest.mark.parametrize(
("prefix", "nested", "stream"),
(("/v1", False, False), ("", True, False), ("/openai/v1", False, True), ("/v1", True, True)),
)
def test_regression_comparison_response_id_owner_repeated_requests_reach_raw_provider_id(
gateway: Gateway, prefix: str, nested: bool, stream: bool
) -> None:
raw_id: Final = f"resp_comparison_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
assert (request.method, request.target) == ("POST", "/v1/responses")
body: Final = _JSON_OBJECT.validate_json(request.body)
if body["input"] == "create comparison response":
assert body == {"model": _MODEL, "input": "create comparison response"}, request.body
return Reply(body=_response(raw_id))
assert body == {
"model": _MODEL,
"input": "compare owned response",
"stream": stream,
"prompt_cache_options": {"comparison_response_id": raw_id},
}, request.body
if stream:
return Reply(
content_type="text/event-stream",
chunks=(
b"data: "
+ json.dumps({"type": "response.completed", "response": json.loads(_response(raw_id))}).encode()
+ b"\n\n",
b"data: [DONE]\n\n",
),
)
return Reply(body=_response(raw_id))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team: Final = scenario.team(models=[model])
user: Final = scenario.member(team)
with _client(gateway, _key(scenario, team, user, model), prefix) as client:
created: Final = client.responses.create(model=model, input="create comparison response")
assert created.id != raw_id, created.model_dump_json()
assert len(_contract_requests(wire)) == 1
def compare_again() -> None:
result: Final = client.responses.create(
model=model,
input="compare owned response",
stream=stream,
extra_body=_comparison_body(created.id, nested),
)
if isinstance(result, openai.Stream):
events: Final = tuple(result)
assert isinstance(events[-1], ResponseCompletedEvent), events
assert events[-1].response.status == "completed", events
else:
assert result.status == "completed", result.model_dump_json()
for _ in range(2):
compare_again()
assert len(_contract_requests(wire)) == 2
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize("stranger", ("user", "team"))
@pytest.mark.parametrize("retained", (False, True), ids=("ordinary", "client-internal-field"))
def test_regression_comparison_response_id_other_identity_is_denied_before_dispatch(
gateway: Gateway, nested: bool, stranger: str, retained: bool
) -> None:
raw_id: Final = f"resp_comparison_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
return Reply(body=_response(raw_id))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team_a: Final = scenario.team(models=[model])
team_b: Final = scenario.team(models=[model])
user_a: Final = scenario.member(team_a)
user_b: Final = scenario.member(team_a)
gateway.post("/team/member_add", {"team_id": team_b, "member": {"role": "user", "user_id": user_a}})
with (
_client(gateway, _key(scenario, team_a, user_a, model), "/v1") as owner,
_client(
gateway,
_key(
scenario, team_a if stranger == "user" else team_b, user_b if stranger == "user" else user_a, model
),
"/v1",
) as other,
):
created: Final = owner.responses.create(model=model, input="create private comparison")
assert created.id != raw_id
assert len(_contract_requests(wire)) == 1
_refused(
lambda: other.responses.create(
model=model,
input="compare another identity",
extra_body={
**_comparison_body(created.id, nested),
**(
{"_litellm_addressed_comparison_response_id": "resp_unmanaged_comparison"}
if retained
else {}
),
},
),
_OTHER_USER_DETAIL if stranger == "user" else _OTHER_TEAM_DETAIL,
)
assert _contract_requests(wire) == (), "A cross-identity comparison reached the provider"
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize("comparison_id", ("resp_unmanaged_comparison", "resp_!!!not-base64", "cmF3X3ZlbmRvcl9pZA=="))
def test_regression_unmanaged_comparison_response_id_is_denied_by_default(
gateway: Gateway, nested: bool, comparison_id: str
) -> None:
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
return Reply(body=_response(f"resp_result_{uuid.uuid4().hex}"))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team: Final = scenario.team(models=[model])
user: Final = scenario.member(team)
with _client(gateway, _key(scenario, team, user, model), "/v1") as client:
_refused(
lambda: client.responses.create(
model=model, input="unmanaged comparison", extra_body=_comparison_body(comparison_id, nested)
),
_UNMANAGED_DETAIL,
)
assert _contract_requests(wire) == ()
def test_comparison_extra_body_owned_id_overrides_foreign_top_level(gateway: Gateway) -> None:
foreign_raw_id: Final = f"resp_ignored_{uuid.uuid4().hex}"
owner_raw_id: Final = f"resp_effective_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
assert (request.method, request.target) == ("POST", "/v1/responses")
body: Final = _JSON_OBJECT.validate_json(request.body)
if body["input"] in ("seed ignored comparison", "seed effective comparison"):
assert body == {"model": _MODEL, "input": body["input"]}, request.body
return Reply(body=_response(foreign_raw_id if body["input"] == "seed ignored comparison" else owner_raw_id))
assert body == {
"model": _MODEL,
"input": "compare effective options",
"prompt_cache_options": {"comparison_response_id": owner_raw_id},
}, request.body
return Reply(body=_response(owner_raw_id))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team: Final = scenario.team(models=[model])
owner_user: Final = scenario.member(team)
foreign_user: Final = scenario.member(team)
with (
_client(gateway, _key(scenario, team, owner_user, model), "/v1") as owner,
_client(gateway, _key(scenario, team, foreign_user, model), "/v1") as foreign,
):
foreign_response: Final = foreign.responses.create(model=model, input="seed ignored comparison")
assert foreign_response.id != foreign_raw_id
owner_response: Final = owner.responses.create(model=model, input="seed effective comparison")
assert owner_response.id != owner_raw_id
assert len(_contract_requests(wire)) == 2
compared: Final = owner.responses.create(
model=model,
input="compare effective options",
extra_body={
"prompt_cache_options": {
"comparison_response_id": foreign_response.id,
"mode": "explicit",
"ttl": "12h",
},
"extra_body": {"prompt_cache_options": {"comparison_response_id": owner_response.id}},
},
)
assert compared.status == "completed", compared.model_dump_json()
assert len(_contract_requests(wire)) == 1
def test_regression_comparison_extra_body_raw_id_overrides_foreign_top_level(gateway: Gateway) -> None:
raw_id: Final = "resp_raw_effective_comparison"
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
return Reply(body=_response("resp_foreign_seed"))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team: Final = scenario.team(models=[model])
owner_user: Final = scenario.member(team)
foreign_user: Final = scenario.member(team)
with (
_client(gateway, _key(scenario, team, owner_user, model), "/v1") as owner,
_client(gateway, _key(scenario, team, foreign_user, model), "/v1") as foreign,
):
foreign_response: Final = foreign.responses.create(model=model, input="seed ignored comparison")
assert foreign_response.id != "resp_foreign_seed"
assert len(_contract_requests(wire)) == 1
_refused(
lambda: owner.responses.create(
model=model,
input="compare effective options",
extra_body={
"prompt_cache_options": {
"comparison_response_id": foreign_response.id,
"mode": "explicit",
"ttl": "12h",
},
"extra_body": {"prompt_cache_options": {"comparison_response_id": raw_id}},
},
),
_UNMANAGED_DETAIL,
)
assert _contract_requests(wire) == ()
@pytest.fixture(scope="module")
def allowed_comparison_gateway(
session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory
) -> Iterator[tuple[Gateway, Wire]]:
directory: Final = tmp_path_factory.mktemp("allowed_comparison")
config: Final = directory / "config.yaml"
config.write_text("general_settings:\n allow_unmanaged_response_ids: true\n")
with wire_server(lambda request: Reply(body=_response("resp_decoy"))) as decoy:
with owned_proxy(
session_gateway,
directory,
{
"INTEGRATION_PROXY_WORKERS": "1",
"OPENAI_BASE_URL": f"{decoy.url}/v1",
"OPENAI_API_KEY": "synthetic-env-fallback-key",
},
config=config,
) as allowed:
yield allowed, decoy
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize("comparison_id", ("resp_unmanaged_comparison", "resp_!!!not-base64", "cmF3X3ZlbmRvcl9pZA=="))
def test_guard_opt_in_forwards_unmanaged_comparison_id(
allowed_comparison_gateway: tuple[Gateway, Wire], nested: bool, comparison_id: str
) -> None:
allowed, decoy = allowed_comparison_gateway
result_id: Final = f"resp_result_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
return Reply(body=_response(result_id))
with wire_server(respond) as wire, allowed.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team: Final = scenario.team(models=[model])
user: Final = scenario.member(team)
with _client(allowed, _key(scenario, team, user, model), "/v1") as client:
compared: Final = client.responses.create(
model=model, input="unmanaged comparison", extra_body=_comparison_body(comparison_id, nested)
)
assert compared.model_dump(mode="json", exclude_none=True) == {
"id": compared.id,
"object": "response",
"created_at": 1.0,
"status": "completed",
"model": model,
"output": [],
"parallel_tool_calls": False,
"tool_choice": "auto",
"tools": [],
}, compared.model_dump_json()
assert compared.id != result_id
requests: Final = _contract_requests(wire)
assert len(requests) == 1
assert (requests[0].method, requests[0].target) == ("POST", "/v1/responses")
assert _JSON_OBJECT.validate_json(requests[0].body) == {
"model": _MODEL,
"input": "unmanaged comparison",
"prompt_cache_options": {"comparison_response_id": comparison_id},
}, requests[0].body
assert decoy.drain() == ()
@pytest.mark.parametrize("stranger", ("user", "team"))
def test_regression_echoed_raw_comparison_id_is_denied_to_other_identity(gateway: Gateway, stranger: str) -> None:
raw_id: Final = f"resp_echo_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.target == "/v1/models":
return _model_discovery_reply()
body: Final = _JSON_OBJECT.validate_json(request.body)
if body["input"] == "create response":
assert body == {"model": _MODEL, "input": "create response"}, request.body
return Reply(body=_response(raw_id))
assert body == {
"model": _MODEL,
"input": "compare owned response",
"prompt_cache_options": {"comparison_response_id": raw_id},
}, request.body
return Reply(
body=json.dumps(
{**json.loads(_response(raw_id)), "prompt_cache_options": {"comparison_response_id": raw_id}}
).encode()
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
team_a: Final = scenario.team(models=[model])
team_b: Final = scenario.team(models=[model])
user_a: Final = scenario.member(team_a)
user_b: Final = scenario.member(team_a)
gateway.post("/team/member_add", {"team_id": team_b, "member": {"role": "user", "user_id": user_a}})
with (
_client(gateway, _key(scenario, team_a, user_a, model), "/v1") as owner,
_client(
gateway,
_key(
scenario, team_a if stranger == "user" else team_b, user_b if stranger == "user" else user_a, model
),
"/v1",
) as other,
):
created: Final = owner.responses.create(model=model, input="create response")
assert created.id != raw_id
assert len(_contract_requests(wire)) == 1
echoed: Final = owner.responses.create(
model=model, input="compare owned response", extra_body=_comparison_body(created.id, False)
)
echo_body: Final = _JSON_OBJECT.validate_json(echoed.model_dump_json(exclude_none=True))
assert echo_body == {
"id": echoed.id,
"object": "response",
"created_at": 1.0,
"status": "completed",
"model": model,
"output": [],
"parallel_tool_calls": False,
"tool_choice": "auto",
"tools": [],
"prompt_cache_options": {"comparison_response_id": raw_id},
}, echo_body
options: Final = object_value(echo_body["prompt_cache_options"])
leaked_id: Final = options["comparison_response_id"]
assert leaked_id == raw_id
assert isinstance(leaked_id, str)
assert len(_contract_requests(wire)) == 1
_refused(
lambda: other.responses.create(
model=model, input="compare owned response", extra_body=_comparison_body(leaked_id, False)
),
_UNMANAGED_DETAIL,
)
assert _contract_requests(wire) == ()
@pytest.mark.parametrize("prefix", _ALIASES)
def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway, prefix: str) -> None:
raw_id: Final = f"resp_owned_{uuid.uuid4().hex}"

189
tests/integration/providers/test_responses_core_wire.py
View file

@ -1,10 +1,12 @@
from __future__ import annotations
import base64
import json
import os
import uuid
from collections.abc import Callable
from typing import Final
from collections.abc import AsyncIterator, Callable, Iterator
from copy import deepcopy
from typing import Final, Protocol, runtime_checkable
import openai
import pytest
@ -14,6 +16,10 @@ from integration._support.worker_readiness import wait_for_worker_readiness
from openai.types.responses import ResponseCompletedEvent
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.types.llms.openai import ResponseCompletedEvent as LiteLLMResponseCompletedEvent
from litellm.types.llms.openai import ResponsesAPIResponse
_MODEL: Final = "gpt-5"
_API_KEY: Final = "synthetic-responses-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
@ -21,6 +27,16 @@ _PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "
_ALIASES: Final = ("/v1", "", "/openai/v1")
@runtime_checkable
class _AsyncResponseEvents(Protocol):
def __aiter__(self) -> AsyncIterator[object]: ...
@runtime_checkable
class _ResponseEvents(Protocol):
def __iter__(self) -> Iterator[object]: ...
def _deployments_in_group(gateway: Gateway, group: str) -> int:
entries: Final = gateway.get("/model/info")["data"]
assert isinstance(entries, list), entries
@ -69,6 +85,175 @@ def _stream(identity: str) -> tuple[bytes, ...]:
) + (b"data: [DONE]\n\n",)
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async"))
@pytest.mark.parametrize("stream", (False, True), ids=("nonstream", "stream"))
@pytest.mark.parametrize("extra", (False, True), ids=("top-level", "extra-body-precedence"))
async def test_regression_sdk_comparison_response_id_is_raw_on_the_wire(
asynchronous: bool, stream: bool, extra: bool
) -> None:
raw_id: Final = f"resp_sdk_comparison_{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
assert (request.method, request.target) == ("POST", "/v1/responses")
body: Final = _JSON_OBJECT.validate_json(request.body)
if body["input"] == "seed comparison":
assert body == {"model": _MODEL, "input": "seed comparison"}, request.body
return Reply(body=_response(raw_id))
assert body == {
"model": _MODEL,
"input": "compare response",
"stream": stream,
"previous_response_id": raw_id,
"prompt_cache_options": {"comparison_response_id": raw_id},
}, request.body
return (
Reply(content_type="text/event-stream", chunks=_stream(raw_id)) if stream else Reply(body=_response(raw_id))
)
with wire_server(respond) as wire:
seeded: Final = litellm.responses(
model=f"openai/{_MODEL}", input="seed comparison", api_base=f"{wire.url}/v1", api_key=_API_KEY
)
assert isinstance(seeded, ResponsesAPIResponse), seeded
assert seeded.id != raw_id, seeded
params: Final = {
"model": f"openai/{_MODEL}",
"input": "compare response",
"stream": stream,
"previous_response_id": seeded.id,
"api_base": f"{wire.url}/v1",
"api_key": _API_KEY,
"num_retries": 0,
**(
{
"prompt_cache_options": {"mode": "explicit", "ttl": "12h"},
"extra_body": {"prompt_cache_options": {"comparison_response_id": seeded.id}},
}
if extra
else {"prompt_cache_options": {"comparison_response_id": seeded.id}}
),
}
original: Final = deepcopy(params)
result: Final = await litellm.aresponses(**params) if asynchronous else litellm.responses(**params)
if stream:
if asynchronous:
assert isinstance(result, _AsyncResponseEvents), result
async_events: Final = tuple([event async for event in result])
async_completed: Final = async_events[-1]
assert isinstance(async_completed, LiteLLMResponseCompletedEvent), async_events
assert async_completed.response.status == "completed", async_events
else:
assert isinstance(result, _ResponseEvents), result
sync_events: Final = tuple(result)
sync_completed: Final = sync_events[-1]
assert isinstance(sync_completed, LiteLLMResponseCompletedEvent), sync_events
assert sync_completed.response.status == "completed", sync_events
else:
assert isinstance(result, ResponsesAPIResponse), result
assert result.status == "completed", result
assert params == original, "The caller's comparison options were mutated"
assert len(wire.drain()) == 2
@pytest.mark.parametrize("extra", (False, True))
@pytest.mark.parametrize(
"comparison_id",
(
"resp_custom_vendor_id",
"resp_!!!not-base64",
"cmF3X3ZlbmRvcl9pZA==",
pytest.param(
"resp_"
+ base64.b64encode(b"litellm:custom_llm_provider:xai;model_id:foreign;response_id:resp_foreign").decode(),
id="foreign-provider-wrapper",
),
pytest.param("resp_" + base64.b64encode(b"custom;id;response").decode(), id="unmanaged-semicolons"),
pytest.param(
"resp_" + base64.b64encode(b"openai;foreign;resp_unmanaged").decode(), id="openai-without-envelope-markers"
),
),
)
def test_guard_sdk_custom_host_preserves_unmanaged_comparison_ids(extra: bool, comparison_id: str) -> None:
def respond(request: Request) -> Reply:
assert (request.method, request.target) == ("POST", "/v1/responses")
assert _JSON_OBJECT.validate_json(request.body) == {
"model": _MODEL,
"input": "custom comparison",
"prompt_cache_options": {"comparison_response_id": comparison_id, "mode": "implicit", "ttl": "30m"},
}, request.body
return Reply(body=_response("resp_custom_result"))
with wire_server(respond) as wire:
options: Final = {
"comparison_response_id": comparison_id,
"mode": "implicit",
"ttl": "30m",
}
params: Final = (
{"extra_body": {"prompt_cache_options": options}} if extra else {"prompt_cache_options": options}
)
original: Final = deepcopy(params)
result: Final = litellm.responses(
model=f"openai/{_MODEL}", input="custom comparison", api_base=f"{wire.url}/v1", api_key=_API_KEY, **params
)
assert isinstance(result, ResponsesAPIResponse), result
assert result.status == "completed", result
assert params == original
assert len(wire.drain()) == 1
@pytest.mark.parametrize("provider", ("azure", "xai"))
@pytest.mark.parametrize("extra", (False, True))
def test_guard_inherited_responses_providers_preserve_comparison_wrappers(provider: str, extra: bool) -> None:
raw_id: Final = f"resp_inherited_{uuid.uuid4().hex}"
def seed(request: Request) -> Reply:
assert _JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": "seed inherited comparison"}
return Reply(body=_response(raw_id))
with wire_server(seed) as seed_wire:
seeded: Final = litellm.responses(
model=f"openai/{_MODEL}",
input="seed inherited comparison",
api_base=f"{seed_wire.url}/v1",
api_key=_API_KEY,
)
assert isinstance(seeded, ResponsesAPIResponse), seeded
assert seeded.id != raw_id
assert len(seed_wire.drain()) == 1
def respond(request: Request) -> Reply:
assert request.method == "POST", request
expected_target: Final = (
"/openai/responses?api-version=2025-03-01-preview" if provider == "azure" else "/v1/responses"
)
assert request.target == expected_target, request.target
assert _JSON_OBJECT.validate_json(request.body) == {
"model": _MODEL,
"input": "inherited comparison",
"prompt_cache_options": {"comparison_response_id": seeded.id},
}, request.body
return Reply(body=_response(raw_id))
with wire_server(respond) as wire:
options: Final = {"comparison_response_id": seeded.id}
params: Final = (
{"extra_body": {"prompt_cache_options": options}} if extra else {"prompt_cache_options": options}
)
result: Final = litellm.responses(
model=f"{provider}/{_MODEL}",
input="inherited comparison",
api_base=wire.url if provider == "azure" else f"{wire.url}/v1",
api_key=_API_KEY,
api_version="2025-03-01-preview" if provider == "azure" else None,
**params,
)
assert isinstance(result, ResponsesAPIResponse), result
assert result.status == "completed", result
assert len(wire.drain()) == 1
def _register_in_group(scenario: Scenario, wire: Wire, group: str, order: int | None = None) -> str:
existing: Final = _deployments_in_group(scenario.gateway, group)
created: Final = scenario.gateway.post(

1
tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -9366,6 +9366,7 @@ PROXY_STAMPED_NAMES: Final = frozenset(
"secret_fields",
"litellm_trusted_callback_vars",
"_litellm_addressed_response_id",
"_litellm_addressed_comparison_response_id",
"_litellm_strip_stream_usage",
"_litellm_stream_usage_injected",
"client_side_timeout",

261
tests/unit/test_responses_id_security.py
View file

@ -4,11 +4,16 @@ Tests for ResponsesIDSecurity hook.
Tests the security hook that prevents user B from seeing response from user A.
"""
from collections.abc import Mapping
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
from fastapi import HTTPException
from pydantic import TypeAdapter
from litellm.caching.caching import DualCache
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.hooks.responses_id_security import (
ResponsesIDSecurity,
_streams_response_ids,
@ -20,7 +25,7 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import SpecialEnums
from litellm.types.utils import ADDRESSED_COMPARISON_RESPONSE_ID_FIELD, SpecialEnums
@pytest.fixture
@ -829,7 +834,9 @@ def salt_key_env(monkeypatch):
return _UNIT_TEST_SALT_KEY
def _hook(general_settings=None, signing_key=_UNIT_TEST_SALT_KEY):
def _hook(
general_settings: Mapping[str, object] | None = None, signing_key: str | None = _UNIT_TEST_SALT_KEY
) -> ResponsesIDSecurity:
settings = general_settings if general_settings is not None else {}
return ResponsesIDSecurity(
general_settings_reader=lambda: settings,
@ -837,21 +844,253 @@ def _hook(general_settings=None, signing_key=_UNIT_TEST_SALT_KEY):
)
def _auth(user_id="owner-user", team_id="owner-team", user_role=None):
from litellm.proxy._types import UserAPIKeyAuth
def _auth(
user_id: str = "owner-user", team_id: str = "owner-team", user_role: LitellmUserRoles | None = None
) -> UserAPIKeyAuth:
return UserAPIKeyAuth(user_id=user_id, team_id=team_id, user_role=user_role)
def _issue_managed_id(hook, owner, provider_response_id=_FABRICATED_PROVIDER_RESPONSE_ID):
def _issue_managed_id(
hook: ResponsesIDSecurity, owner: UserAPIKeyAuth, provider_response_id: str = _FABRICATED_PROVIDER_RESPONSE_ID
) -> str:
"""Mint an id exactly the way the proxy hands one to a client on create."""
issued = hook._encrypt_response_id(
ResponsesAPIResponse(
id=provider_response_id, created_at=1234567890, output=[], status="completed"
),
issued = hook._encrypt_response_id( # pyright: ignore[reportPrivateUsage] # mints ids the way create does
ResponsesAPIResponse(id=provider_response_id, created_at=1234567890, output=[], status="completed"),
owner,
)
return issued.id
return ResponsesAPIResponse.model_validate(issued).id
def _comparison_options(data: object, nested: bool) -> Mapping[str, str]:
request: Final = TypeAdapter(dict[str, object]).validate_python(data)
container: Final = TypeAdapter(dict[str, object]).validate_python(request["extra_body"]) if nested else request
return TypeAdapter(dict[str, str]).validate_python(container["prompt_cache_options"])
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize("stranger", ("user", "team"))
@pytest.mark.parametrize("allow_unmanaged", (False, True))
async def test_regression_comparison_id_reentry_keeps_ownership_after_owner_retry(
salt_key_env: str, nested: bool, stranger: str, allow_unmanaged: bool
) -> None:
hook: Final = _hook({"allow_unmanaged_response_ids": allow_unmanaged}, signing_key=salt_key_env)
owner: Final = UserAPIKeyAuth(user_id="owner-user", team_id="owner-team")
caller: Final = UserAPIKeyAuth(
user_id="stranger-user" if stranger == "user" else owner.user_id,
team_id="stranger-team" if stranger == "team" else owner.team_id,
)
cache: Final = DualCache()
issued: Final = ResponsesAPIResponse.model_validate(
await hook.async_post_call_success_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
data={},
user_api_key_dict=owner,
response=ResponsesAPIResponse(
id=_FABRICATED_PROVIDER_RESPONSE_ID, created_at=1234567890, output=[], status="completed"
),
)
)
assert issued.id != _FABRICATED_PROVIDER_RESPONSE_ID
options: Final = {"comparison_response_id": issued.id}
request: Final = {"extra_body": {"prompt_cache_options": options}} if nested else {"prompt_cache_options": options}
first: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner, cache=cache, data=request, call_type="aresponses"
)
)
retried: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner, cache=cache, data=request, call_type="aresponses"
)
)
with pytest.raises(HTTPException) as refusal:
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=caller, cache=cache, data=request, call_type="aresponses"
)
assert refusal.value.status_code == 403
assert f"not associated with the {stranger}" in refusal.value.detail
assert _comparison_options(first, nested) == {"comparison_response_id": _FABRICATED_PROVIDER_RESPONSE_ID}
assert _comparison_options(retried, nested) == {"comparison_response_id": _FABRICATED_PROVIDER_RESPONSE_ID}
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize("replacement", ("resp_vendor_raw", "resp_!!!not-base64", ""))
async def test_guard_comparison_id_replaced_after_owner_hook_reaches_provider_unchanged(
salt_key_env: str, nested: bool, replacement: str
) -> None:
hook: Final = ResponsesIDSecurity(
general_settings_reader=lambda: {"allow_unmanaged_response_ids": True},
signing_key_reader=lambda: salt_key_env,
)
owner: Final = UserAPIKeyAuth(user_id="owner-user", team_id="owner-team")
cache: Final = DualCache()
issued: Final = ResponsesAPIResponse.model_validate(
await hook.async_post_call_success_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
data={},
user_api_key_dict=owner,
response=ResponsesAPIResponse(
id=_FABRICATED_PROVIDER_RESPONSE_ID, created_at=1234567890, output=[], status="completed"
),
)
)
first_options: Final = {"comparison_response_id": issued.id}
first_request: Final = (
{"extra_body": {"prompt_cache_options": first_options}} if nested else {"prompt_cache_options": first_options}
)
first: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner, cache=cache, data=first_request, call_type="aresponses"
)
)
replaced_options: Final = {"comparison_response_id": replacement, "mode": "implicit"}
replaced: Final = (
{**first, "extra_body": {"prompt_cache_options": replaced_options}}
if nested
else {**first, "prompt_cache_options": replaced_options}
)
rerun: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner, cache=cache, data=replaced, call_type="aresponses"
)
)
assert _comparison_options(rerun, nested) == replaced_options, rerun
def _comparison_request(comparison_response_id: str, nested: bool) -> dict[str, object]:
options: Final = {"comparison_response_id": comparison_response_id, "mode": "implicit"}
return {"extra_body": {"prompt_cache_options": options}} if nested else {"prompt_cache_options": options}
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize("replacement", ("resp_vendor_raw", "resp_!!!not-base64", ""))
async def test_comparison_id_replaced_after_owner_hook_is_rejected_by_default(
salt_key_env: str, nested: bool, replacement: str
) -> None:
hook: Final = _hook()
owner: Final = _auth()
first: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner,
cache=DualCache(),
data=_comparison_request(_issue_managed_id(hook, owner), nested),
call_type="aresponses",
)
)
replaced: Final = {**first, **_comparison_request(replacement, nested)}
with pytest.raises(HTTPException) as refusal:
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner, cache=DualCache(), data=replaced, call_type="aresponses"
)
assert refusal.value.status_code == 403
assert "allow_unmanaged_response_ids" in refusal.value.detail
assert _comparison_options(replaced, nested)["comparison_response_id"] == replacement
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
async def test_raw_comparison_id_is_rejected_by_default(salt_key_env: str, nested: bool) -> None:
request: Final = _comparison_request(_FABRICATED_UNMANAGED_ID, nested)
with pytest.raises(HTTPException) as refusal:
await _hook().async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=_auth(), cache=DualCache(), data=request, call_type="aresponses"
)
assert refusal.value.status_code == 403
assert "allow_unmanaged_response_ids" in refusal.value.detail
assert _comparison_options(request, nested)["comparison_response_id"] == _FABRICATED_UNMANAGED_ID
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
@pytest.mark.parametrize(
("general_settings", "signing_key", "user_role"),
[
({"allow_unmanaged_response_ids": True}, _UNIT_TEST_SALT_KEY, None),
({"disable_responses_id_security": True}, _UNIT_TEST_SALT_KEY, None),
({}, None, None),
({}, _UNIT_TEST_SALT_KEY, LitellmUserRoles.PROXY_ADMIN),
],
)
async def test_raw_comparison_id_follows_previous_response_id_escape_hatches(
salt_key_env: str,
nested: bool,
general_settings: dict[str, bool],
signing_key: str | None,
user_role: LitellmUserRoles | None,
) -> None:
hook: Final = _hook(general_settings=general_settings, signing_key=signing_key)
result: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=_auth(user_role=user_role),
cache=DualCache(),
data=_comparison_request(_FABRICATED_UNMANAGED_ID, nested),
call_type="aresponses",
)
)
assert _comparison_options(result, nested) == {
"comparison_response_id": _FABRICATED_UNMANAGED_ID,
"mode": "implicit",
}
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
async def test_retained_comparison_id_of_another_owner_cannot_authorize_its_raw_id(
salt_key_env: str, nested: bool
) -> None:
hook: Final = _hook()
victim_id: Final = _issue_managed_id(hook, _auth())
request: Final = {
**_comparison_request(_FABRICATED_PROVIDER_RESPONSE_ID, nested),
ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: victim_id,
}
with pytest.raises(HTTPException) as refusal:
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=_auth(user_id="stranger-user", team_id="stranger-team"),
cache=DualCache(),
data=request,
call_type="aresponses",
)
assert refusal.value.status_code == 403
assert "not associated with the user" in refusal.value.detail
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
async def test_retained_own_comparison_id_cannot_carry_a_different_raw_id(salt_key_env: str, nested: bool) -> None:
hook: Final = _hook()
stranger: Final = _auth(user_id="stranger-user", team_id="stranger-team")
stranger_id: Final = _issue_managed_id(hook, stranger, provider_response_id="resp_strangerownprovideridcccccccc")
request: Final = {
**_comparison_request("resp_victimprovideriddddddddddddddddddddd", nested),
ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: stranger_id,
}
with pytest.raises(HTTPException) as refusal:
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=stranger, cache=DualCache(), data=request, call_type="aresponses"
)
assert refusal.value.status_code == 403
assert "allow_unmanaged_response_ids" in refusal.value.detail
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", (False, True))
async def test_newly_supplied_comparison_id_wins_over_the_retained_one(salt_key_env: str, nested: bool) -> None:
hook: Final = _hook()
owner: Final = _auth()
stale_id: Final = _issue_managed_id(hook, owner, provider_response_id="resp_staleownerprovideridbbbbbbbbbb")
fresh_id: Final = _issue_managed_id(hook, owner)
result: Final = TypeAdapter(dict[str, object]).validate_python(
await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary
user_api_key_dict=owner,
cache=DualCache(),
data={**_comparison_request(fresh_id, nested), ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: stale_id},
call_type="aresponses",
)
)
assert _comparison_options(result, nested)["comparison_response_id"] == _FABRICATED_PROVIDER_RESPONSE_ID
assert result[ADDRESSED_COMPARISON_RESPONSE_ID_FIELD] == fresh_id
class TestUnrecognizedResponseIdIsRejected:

1
tests/unit/types/test_litellm_params.py
View file

@ -269,6 +269,7 @@ INTERNAL_STATE_NAMES: Final = (
"github_copilot_user_session",
"litellm_trusted_callback_vars",
"_litellm_addressed_response_id",
"_litellm_addressed_comparison_response_id",
"_litellm_strip_stream_usage",
"_litellm_stream_usage_injected",
"client_side_timeout",

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -41886,6 +41886,8 @@ export interface components {
};
/** PromptCacheOptions */
PromptCacheOptions: {
/** Comparison Response Id */
comparison_response_id?: string;
/**
* Mode
* @enum {string}