From fa84894205458e27369b22de7af039df300ea663 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:05:32 -0700 Subject: [PATCH 1/6] fix(responses): log the encrypted response id on streamed /v1/responses spend rows With response id security on, the stream hook encrypts the resp_ id on every frame, but logging had already received the SDK's own copy of the terminal response, so the spend row, the standard logging payload and callbacks got the managed id (or the chatcmpl id on a bridged stream) the client never saw. The hook now records the exact id it advertises on the request's logging object, and the logging layer stamps that id on the logged streamed response --- litellm/litellm_core_utils/litellm_logging.py | 26 ++- litellm/proxy/hooks/responses_id_security.py | 54 +++-- .../test_bedrock_gpt_responses_native_wire.py | 6 +- .../test_bedrock_model_discovery_wire.py | 9 +- .../test_responses_stream_spend_row_id.py | 198 ++++++++++++++++++ .../test_litellm_logging.py | 27 +++ tests/unit/test_responses_id_security.py | 128 ++++++++++- 7 files changed, 415 insertions(+), 33 deletions(-) create mode 100644 tests/integration/spend/test_responses_stream_spend_row_id.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8b66c27b930..0c84bc2dcde 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -798,6 +798,7 @@ class Logging(LiteLLMLoggingBaseClass): # The model the proxy shows the client on streamed chunks. The logged streamed response carries it # once that response is priced, the same way a non-streamed response is logged self.client_facing_stream_model: str | None = None + self.client_facing_response_id: str | None = None self.zero_cost_warned: bool = False self.llm_caching_handler: LLMCachingHandler | None = None @@ -2922,6 +2923,23 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"id": response_id}) + def _with_client_facing_response_id( + self, + response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: + response_id: Final = self.client_facing_response_id + if response_id is None or getattr(response, "id", None) == response_id: + return response + return response.model_copy(update={"id": response_id}) + + def _client_facing_streaming_response( + self, + response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: + return self._with_client_facing_response_id( + self._with_continued_response_id(self._with_client_facing_stream_model(response)) + ) + def _success_handler_helper_fn( self, result: object = None, @@ -3310,9 +3328,7 @@ class Logging(LiteLLMLoggingBaseClass): ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._with_continued_response_id( - self._with_client_facing_stream_model(complete_streaming_response) - ) + logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) self.model_call_details["complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD if not for_async_request: @@ -3910,9 +3926,7 @@ class Logging(LiteLLMLoggingBaseClass): await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._with_continued_response_id( - self._with_client_facing_stream_model(complete_streaming_response) - ) + logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 27b5924c540..58c44af6652 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -14,6 +14,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import LitellmUserRoles from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, @@ -278,6 +279,14 @@ class ResponsesIDSecurity(CustomLogger): user_api_key_dict: "UserAPIKeyAuth", request_cache: dict[str, str] | None = None, ) -> BaseLiteLLMOpenAIResponseObject: + encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache) + return response if encrypt is None else _rewrite_advertised_id(response, encrypt) + + def _response_id_encrypter( + self, + user_api_key_dict: "UserAPIKeyAuth", + request_cache: dict[str, str] | None, + ) -> Callable[[str], str] | None: signing_key: Final = self._get_signing_key() if signing_key is None: verbose_proxy_logger.debug( @@ -286,7 +295,7 @@ class ResponsesIDSecurity(CustomLogger): "Skipping response ID encryption. " "See: https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" ) - return response + return None def encrypt(original_id: str) -> str: cached: Final = request_cache.get(original_id) if request_cache is not None else None @@ -303,7 +312,28 @@ class ResponsesIDSecurity(CustomLogger): request_cache[original_id] = encrypted_id return encrypted_id - return _rewrite_advertised_id(response, encrypt) + return encrypt + + def _streamed_response_id_advertiser( + self, + user_api_key_dict: "UserAPIKeyAuth", + request_data: Mapping[str, object], + ) -> Callable[[str], str] | None: + if not _is_responses_api_create_route(user_api_key_dict.request_route): + return None + if self._general_settings_reader().get("disable_responses_id_security", False): + return None + encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache={}) + logging_obj: Final = request_data.get("litellm_logging_obj") + if encrypt is None or not isinstance(logging_obj, LiteLLMLoggingObj): + return encrypt + + def advertise(original_id: str) -> str: + advertised_id: Final = encrypt(original_id) + logging_obj.client_facing_response_id = advertised_id + return advertised_id + + return advertise async def async_post_call_success_hook( self, @@ -329,21 +359,15 @@ class ResponsesIDSecurity(CustomLogger): return response async def async_post_call_streaming_iterator_hook( - self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict + self, user_api_key_dict: "UserAPIKeyAuth", response: Any, request_data: dict[str, object] ) -> AsyncGenerator[BaseLiteLLMOpenAIResponseObject, None]: - general_settings: Final = self._general_settings_reader() - - # Create a request-scoped cache for consistent encryption across streaming chunks. - request_encryption_cache: Final[dict[str, str]] = {} - + advertise: Final = self._streamed_response_id_advertiser(user_api_key_dict, request_data) async for chunk in response: - if ( - isinstance(chunk, BaseLiteLLMOpenAIResponseObject) - and _is_responses_api_create_route(user_api_key_dict.request_route) - and not general_settings.get("disable_responses_id_security", False) - ): - chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache) - yield chunk + yield ( + _rewrite_advertised_id(chunk, advertise) + if advertise is not None and isinstance(chunk, BaseLiteLLMOpenAIResponseObject) + else chunk + ) def decrypted_response_id(response_id: str) -> str: diff --git a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py index 3d6eb7fcaff..0788b9ac85a 100644 --- a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py +++ b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py @@ -62,12 +62,12 @@ def _body(request: Request) -> dict[str, JsonValue]: # ciphertext the caller received, because the spend row id is read from response_obj["id"] before the # ResponsesIDSecurity hook rewrites it in place; the row is looked up under both ids until that ordering is fixed on # main -def _spend_row(client_id: str, issued_id: str) -> dict[str, JsonValue]: +def _spend_row(*request_ids: str) -> dict[str, JsonValue]: rows: Final = eventually( lambda: read_rows( 'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" ' "WHERE request_id = ANY(%s)", - ([client_id, issued_id],), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array + (list(request_ids),), # pyright: ignore[reportArgumentType] # psycopg adapts the list to a text array ), lambda found: len(found) == 1, seconds=70, @@ -120,4 +120,4 @@ async def test_async_openai_sdk_responses_stream_is_served_by_the_native_respons assert issued.upstream == f"resp_upstream_{marker}", completed.response.id request: Final = _native_request(wire) assert _body(request) == {"model": GPT, "input": _prompt(marker), "stream": True}, request.body - assert _spend_row(completed.response.id, issued.issued) == _success_row(model) + assert _spend_row(completed.response.id) == _success_row(model) diff --git a/tests/integration/providers/test_bedrock_model_discovery_wire.py b/tests/integration/providers/test_bedrock_model_discovery_wire.py index 3bf4583e736..2871e51de5e 100644 --- a/tests/integration/providers/test_bedrock_model_discovery_wire.py +++ b/tests/integration/providers/test_bedrock_model_discovery_wire.py @@ -301,11 +301,7 @@ def _spend_row_id(rows: Sequence[Mapping[str, JsonValue]], tag: str) -> str: def _landed_once(served: tuple[tuple[str, str, bool], ...], tags: tuple[str, ...]) -> None: - """One spend row per tagged call, carrying the id the client saw. - - FIXME: a streamed /v1/responses row carries the managed `resp_` id LiteLLM built before the proxy encrypted the - advertised one, so that row is matched by tag alone until the spend log reads the id the client received. - """ + """One spend row per tagged call, carrying the id the client saw.""" rows: Final = eventually( lambda: read_rows( 'SELECT request_id, request_tags FROM "LiteLLM_SpendLogs" WHERE request_tags ?| %s', @@ -316,9 +312,6 @@ def _landed_once(served: tuple[tuple[str, str, bool], ...], tags: tuple[str, ... ) for (identity, path, stream), tag in zip(served, tags, strict=True): row_id: Final = _spend_row_id(rows, tag) - if path == "/v1/responses" and stream: - assert row_id.startswith("resp_"), (row_id, identity) - continue assert row_id == identity, (path, stream, row_id, identity) diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py new file mode 100644 index 00000000000..4ff20a8c26b --- /dev/null +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -0,0 +1,198 @@ +import json +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from pathlib import Path +from typing import Final +from uuid import uuid4 + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value, string_value +from integration._support.process import owned_proxy +from integration._support.responses_stream import frame, response_object +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_NATIVE_MODEL: Final = "openai/gpt-5.6" +_BRIDGED_MODEL: Final = "deepseek/deepseek-v4-pro" +_ANSWER: Final = "streamed answer" +_USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 30, "output_tokens": 5, "total_tokens": 35} +_JSON_LIST: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _sse(payload: dict[str, JsonValue]) -> bytes: + return f"data: {json.dumps(payload)}\n\n".encode() + + +def _native_stream(upstream_id: str) -> Reply: + message: Final[dict[str, JsonValue]] = { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": _ANSWER, "annotations": []}], + } + completed: Final = response_object(upstream_id, "completed", output=[message], usage=_USAGE) + return Reply( + content_type="text/event-stream", + chunks=( + frame( + { + "type": "response.created", + "sequence_number": 0, + "response": response_object(upstream_id, "in_progress"), + } + ), + frame( + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": _ANSWER, + } + ), + frame({"type": "response.completed", "sequence_number": 2, "response": completed}), + ), + ) + + +def _chat_stream(upstream_id: str, model: str) -> Reply: + def chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> dict[str, JsonValue]: + return { + "id": upstream_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + return Reply( + content_type="text/event-stream", + chunks=( + _sse(chunk({"role": "assistant", "content": _ANSWER}, None)), + _sse( + { + **chunk({}, "stop"), + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35}, + } + ), + b"data: [DONE]\n\n", + ), + ) + + +def _respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + body: Final = JSON_OBJECT.validate_json(request.body) + assert body.get("stream") is True, body + upstream_id: Final = f"upstream-{uuid4().hex}" + if request.target == "/v1/responses": + return _native_stream(f"resp_{upstream_id}") + assert request.target == "/v1/chat/completions", request.target + return _chat_stream(f"chatcmpl-{upstream_id}", string_value(body["model"])) + + +def _terminal_response_id(gateway: Gateway, key: str, body: dict[str, JsonValue]) -> str: + response: Final = gateway.request("POST", "/v1/responses", body, key=key) + assert response.status_code == 200, response.text + frames: Final = tuple( + JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert frames and frames[-1]["type"] == "response.completed", response.text + advertised: Final = tuple( + dict.fromkeys(string_value(object_value(event["response"])["id"]) for event in frames if "response" in event) + ) + assert len(advertised) == 1, advertised + return advertised[0] + + +def _request_ids_for_key(gateway: Gateway, key: str, count: int) -> list[str]: + rows: Final = eventually( + lambda: gateway.rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s ORDER BY "endTime"', + (sha256(key.encode()).hexdigest(),), + ), + lambda found: len(found) >= count, + seconds=70, + ) + return [string_value(row["request_id"]) for row in rows] + + +def _logs_page_search(gateway: Gateway, client_id: str) -> list[str]: + now: Final = datetime.now(timezone.utc) + page: Final = gateway.get( + "/spend/logs/ui", + params={ + "search": client_id, + "start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + "end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + }, + ) + return [string_value(object_value(row)["request_id"]) for row in list_value(page["data"])] + + +@pytest.mark.timeout(120) +@pytest.mark.parametrize("litellm_model", [_NATIVE_MODEL, _BRIDGED_MODEL], ids=["native", "bridge"]) +def test_streamed_responses_spend_row_is_filed_under_the_id_the_client_saw( + gateway: Gateway, litellm_model: str +) -> None: + with wire_server(_respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=litellm_model, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[model]) + + client_id: Final = _terminal_response_id( + gateway, key, {"model": model, "input": f"marker-{uuid4().hex}", "stream": True} + ) + + assert client_id.startswith("resp_"), client_id + assert _request_ids_for_key(gateway, key, 1) == [client_id] + assert _logs_page_search(gateway, client_id) == [client_id] + + +def _prompt_storing_config(directory: Path) -> Path: + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path: Final = directory / "store-prompts.yaml" + path.write_text( + yaml.safe_dump( + { + **stock, + "general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True}, + } + ) + ) + return path + + +def test_a_follow_up_turn_on_a_streamed_bridged_response_replays_the_first_turn( + gateway: Gateway, tmp_path: Path +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + model: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[model]) + first_id: Final = _terminal_response_id(storing, key, {"model": model, "input": first_prompt, "stream": True}) + assert _request_ids_for_key(storing, key, 1) == [first_id] + wire.drain() + + second_id: Final = _terminal_response_id( + storing, + key, + {"model": model, "input": "second turn", "previous_response_id": first_id, "stream": True}, + ) + + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"]) + assert [(message["role"], message["content"]) for message in replayed] == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ], replayed + assert _request_ids_for_key(storing, key, 2) == [first_id, second_id] diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 299088a6c4f..72a8429d501 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -10613,6 +10613,33 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost() assert logging_obj.response_cost_calculator(result=assembled) == 0.0042 +@pytest.mark.parametrize("client_facing_response_id", ["resp_encryptedidtheclientsaw", None]) +def test_sync_success_handler_logs_a_stream_under_the_id_the_client_saw(client_facing_response_id): + class Recorder(CustomLogger): + def __init__(self): + super().__init__() + self.logged = [] + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.logged.append((response_obj.id, kwargs["standard_logging_object"]["id"])) + + recorder: Final = Recorder() + logging_obj: Final = _responses_stream_logging_obj() + logging_obj.dynamic_success_callbacks = [recorder] + logging_obj.client_facing_response_id = client_facing_response_id + now: Final = datetime.datetime.now() + + logging_obj.success_handler( + result=_completed_responses_event(ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5)), + start_time=now, + end_time=now, + cache_hit=False, + ) + + expected_id: Final = client_facing_response_id or "resp-1" + assert recorder.logged == [(expected_id, expected_id)] + + def test_response_cost_calculator_prices_terminal_responses_event_from_its_response(): logging_obj: Final = _responses_stream_logging_obj() inner_response: Final = ResponsesAPIResponse( diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index c7fd7fe247f..75f0d33d37d 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -4,23 +4,29 @@ Tests for ResponsesIDSecurity hook. Tests the security hook that prevents user B from seeing response from user A. """ +import datetime from unittest.mock import MagicMock, patch import pytest from fastapi import HTTPException +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.responses_id_security import ( ResponsesIDSecurity, _is_responses_api_create_route, ) +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.types.llms.openai import ( GenericEvent, + ResponseAPIUsage, ResponseCompletedEvent, ResponseCreatedEvent, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import SpecialEnums +from litellm.types.utils import Choices, Message, ModelResponse, SpecialEnums, Usage @pytest.fixture @@ -1089,3 +1095,123 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization: assert result["response_id"] == "resp_strangerownprovideridcccccccc" assert result["response_id"] != victim_provider_id + + +_LOGGED_MANAGED_ID = "resp_managedidlitellmbuiltbeforethehookran" + + +def _streamed_response(response_id): + return ResponsesAPIResponse( + id=response_id, + created_at=0, + model="gpt-5.1", + object="response", + output=[], + parallel_tool_calls=False, + tool_choice="auto", + tools=[], + usage=ResponseAPIUsage(input_tokens=3, output_tokens=2, total_tokens=5), + ) + + +def _upstream_frames(): + return ( + ResponseCreatedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_CREATED, response=_streamed_response(_LOGGED_MANAGED_ID) + ), + ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_streamed_response(_LOGGED_MANAGED_ID) + ), + ) + + +def _native_logged_result(): + return ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_streamed_response(_LOGGED_MANAGED_ID) + ) + + +def _bridged_logged_result(): + return ModelResponse( + id="chatcmpl-bridgedchatcompletionid", + model="gpt-5.1", + choices=[Choices(message=Message(role="assistant", content="hi"), finish_reason="stop", index=0)], + usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5), + ) + + +class _SpendRowRecorder(CustomLogger): + def __init__(self): + super().__init__() + self.spend_row_ids = [] + self.standard_logging_ids = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.spend_row_ids.append(get_logging_payload(kwargs, response_obj, start_time, end_time)["request_id"]) + self.standard_logging_ids.append(kwargs["standard_logging_object"]["id"]) + + +async def _stream_logged_mid_stream(hook, route, logged_result): + recorder = _SpendRowRecorder() + logging_obj = LiteLLMLoggingObj( + model="gpt-5.1", + messages=[], + stream=True, + call_type="aresponses", + start_time=datetime.datetime.now(), + litellm_call_id="streamed-responses-call", + function_id="streamed-responses-fn", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gpt-5.1", user="", optional_params={}, litellm_params={"api_base": ""} + ) + upstream = _upstream_frames() + + async def frames(): + for frame in upstream: + yield frame + + client_stream = hook.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(user_id="owner-user", team_id="owner-team", request_route=route), + response=frames(), + request_data={"litellm_logging_obj": logging_obj}, + ) + first_frame = await anext(client_stream) + now = datetime.datetime.now() + await logging_obj.async_success_handler(result=logged_result, start_time=now, end_time=now, cache_hit=False) + client_frames = [first_frame, *[frame async for frame in client_stream]] + return client_frames[-1].response.id, recorder + + +class TestStreamedSpendRowCarriesTheIdTheClientSaw: + @pytest.mark.asyncio + @pytest.mark.parametrize("logged_result", [_native_logged_result, _bridged_logged_result], ids=["native", "bridge"]) + @pytest.mark.parametrize("route", ["/v1/responses", "/responses", "/openai/v1/responses"]) + async def test_spend_row_and_callbacks_log_the_encrypted_id_of_the_terminal_frame( + self, salt_key_env, route, logged_result + ): + hook = _hook() + + client_id, recorder = await _stream_logged_mid_stream(hook, route, logged_result()) + + assert hook._decrypt_response_id(client_id) == (_LOGGED_MANAGED_ID, "owner-user", "owner-team") + assert recorder.spend_row_ids == [client_id] + assert recorder.standard_logging_ids == [client_id] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "hook, route", + [ + (_hook(), "/chat/completions"), + (_hook(general_settings={"disable_responses_id_security": True}), "/v1/responses"), + (_hook(signing_key=None), "/v1/responses"), + ], + ids=["not-a-create-route", "security-disabled", "no-signing-key"], + ) + async def test_stream_the_hook_leaves_alone_keeps_its_logged_id(self, salt_key_env, hook, route): + client_id, recorder = await _stream_logged_mid_stream(hook, route, _native_logged_result()) + + assert client_id == _LOGGED_MANAGED_ID + assert recorder.spend_row_ids == [_LOGGED_MANAGED_ID] + assert recorder.standard_logging_ids == [_LOGGED_MANAGED_ID] From ccbe4332b1e2b16ba4cbb260348088dccabc4e5d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:46:14 -0700 Subject: [PATCH 2/6] fix(responses): return the id the client sent on GET and cancel, and keep read rows off the create row's id Every GET /v1/responses/{id} re-encrypted the response id, so a client that continued a bridged conversation with the id a GET returned referenced a ciphertext no spend row was filed under. The follow-up lost turn 1 and opened a new session. GET and cancel now answer with the exact id the client sent when it is an id this proxy issued and it decodes to the response the provider returned. A create with previous_response_id still gets a fresh id for its own response, and a raw provider id still gets a managed id Because a read now carries the create's id, a GET or cancel spend row is filed as _retrieve_ or _cancel_. That keeps one row per read, as before, and a read row can never take the create row's request_id, so the create row survives whichever row is written first and a follow-up replays turn 1 once. The background settlement row keeps _background_cost, and the cost poll's own status checks keep the response id --- litellm/proxy/hooks/responses_id_security.py | 30 +++++--- .../spend_tracking/spend_tracking_utils.py | 22 +++++- .../test_responses_lifecycle_wire.py | 3 - .../test_responses_stream_spend_row_id.py | 72 +++++++++++++++++- .../test_spend_tracking_utils.py | 47 +++++++++++- tests/unit/test_responses_id_security.py | 73 +++++++++++++++++++ 6 files changed, 226 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 58c44af6652..64336919e97 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -335,27 +335,35 @@ class ResponsesIDSecurity(CustomLogger): return advertise + def _id_the_client_holds(self, response_id: str, addressed_id: object) -> str | None: + if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id): + return None + original_response_id, _, _ = self._decrypt_response_id(addressed_id) + return addressed_id if original_response_id == response_id else None + + def _advertised_response( + self, + response: ResponsesAPIResponse, + user_api_key_dict: "UserAPIKeyAuth", + request_data: Mapping[str, object], + ) -> BaseLiteLLMOpenAIResponseObject: + held_id: Final = self._id_the_client_holds(response.id, request_data.get(ADDRESSED_RESPONSE_ID_FIELD)) + if held_id is None: + return self._encrypt_response_id(response, user_api_key_dict, request_cache=None) + return _rewrite_advertised_id(response, lambda _: held_id) + async def async_post_call_success_hook( self, - data: dict, + data: dict[str, object], user_api_key_dict: "UserAPIKeyAuth", response: LLMResponseTypes, ) -> LLMResponseTypes: - """ - Queue response IDs for batch processing instead of writing directly to DB. - - This method adds response IDs to an in-memory queue, which are then - batch-processed by the DBSpendUpdateWriter during regular database update cycles. - """ general_settings: Final = self._general_settings_reader() if general_settings.get("disable_responses_id_security", False): return response if isinstance(response, ResponsesAPIResponse): - response = cast( - ResponsesAPIResponse, - self._encrypt_response_id(response, user_api_key_dict, request_cache=None), - ) + response = cast(ResponsesAPIResponse, self._advertised_response(response, user_api_key_dict, data)) return response async def async_post_call_streaming_iterator_hook( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..6fd7bc6b052 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -344,14 +344,29 @@ def _cost_row_request_id_suffix( return "" +_RESPONSE_LIFECYCLE_ROW_LABELS: Final[Mapping[str, str]] = MappingProxyType( + {"aget_responses": "retrieve", "acancel_responses": "cancel"} +) + + +def _response_lifecycle_row_suffix( + call_type: str, metadata: Mapping[str, object], resolved_id: str, litellm_call_id: object +) -> str: + label: Final = _RESPONSE_LIFECYCLE_ROW_LABELS.get(call_type) + if label is None or metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) or litellm_call_id == resolved_id: + return "" + return f"_{label}_{litellm_call_id}" + + def get_spend_logs_id( call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING ) -> str | None: standard_logging_payload = kwargs.get("standard_logging_object") + litellm_call_id: Final = kwargs.get("litellm_call_id") candidate_ids: Final = ( response_obj.get("id"), standard_logging_payload.get("id") if isinstance(standard_logging_payload, dict) else None, - kwargs.get("litellm_call_id"), + litellm_call_id, ) resolved_id: Final = next( (candidate for candidate in candidate_ids if isinstance(candidate, str) and candidate), None @@ -361,7 +376,10 @@ def get_spend_logs_id( settles_background_response: Final = ( kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES ) - return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}" + suffix: Final = _cost_row_request_id_suffix( + call_type, metadata, settles_background_response + ) or _response_lifecycle_row_suffix(call_type, metadata, resolved_id, litellm_call_id) + return f"{resolved_id}{suffix}" _MISSING_ATTRIBUTE: Final = object() diff --git a/tests/integration/providers/test_responses_lifecycle_wire.py b/tests/integration/providers/test_responses_lifecycle_wire.py index 6509c8ced96..8280cf684e5 100644 --- a/tests/integration/providers/test_responses_lifecycle_wire.py +++ b/tests/integration/providers/test_responses_lifecycle_wire.py @@ -196,9 +196,6 @@ def test_retrieve_routes_to_the_deployment_that_created_the_response( @pytest.mark.parametrize("prefix", _ALIASES) def test_retrieve_returns_the_id_the_client_holds(gateway: Gateway, prefix: str) -> None: - pytest.skip( - "BUG: GET /v1/responses/{id} returns a re-encrypted id that differs byte-for-byte from the id the client sent" - ) raw_id: Final = f"resp_retrieve_id_{uuid.uuid4().hex}" def respond(request: Request) -> Reply: diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py index 4ff20a8c26b..d4283c03698 100644 --- a/tests/integration/spend/test_responses_stream_spend_row_id.py +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -24,7 +24,7 @@ def _sse(payload: dict[str, JsonValue]) -> bytes: return f"data: {json.dumps(payload)}\n\n".encode() -def _native_stream(upstream_id: str) -> Reply: +def _native_completed(upstream_id: str) -> dict[str, JsonValue]: message: Final[dict[str, JsonValue]] = { "type": "message", "id": "msg_1", @@ -32,7 +32,10 @@ def _native_stream(upstream_id: str) -> Reply: "role": "assistant", "content": [{"type": "output_text", "text": _ANSWER, "annotations": []}], } - completed: Final = response_object(upstream_id, "completed", output=[message], usage=_USAGE) + return response_object(upstream_id, "completed", output=[message], usage=_USAGE) + + +def _native_stream(upstream_id: str) -> Reply: return Reply( content_type="text/event-stream", chunks=( @@ -53,7 +56,7 @@ def _native_stream(upstream_id: str) -> Reply: "delta": _ANSWER, } ), - frame({"type": "response.completed", "sequence_number": 2, "response": completed}), + frame({"type": "response.completed", "sequence_number": 2, "response": _native_completed(upstream_id)}), ), ) @@ -86,9 +89,15 @@ def _chat_stream(upstream_id: str, model: str) -> Reply: def _respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + if request.method == "GET": + stored_id: Final = request.target.removeprefix("/v1/responses/") + assert stored_id.startswith("resp_upstream-"), request.target + return Reply(body=json.dumps(_native_completed(stored_id)).encode()) body: Final = JSON_OBJECT.validate_json(request.body) - assert body.get("stream") is True, body upstream_id: Final = f"upstream-{uuid4().hex}" + if request.target == "/v1/responses" and body.get("stream") is not True: + return Reply(body=json.dumps(_native_completed(f"resp_{upstream_id}")).encode()) + assert body.get("stream") is True, body if request.target == "/v1/responses": return _native_stream(f"resp_{upstream_id}") assert request.target == "/v1/chat/completions", request.target @@ -111,6 +120,18 @@ def _terminal_response_id(gateway: Gateway, key: str, body: dict[str, JsonValue] return advertised[0] +def _created_response_id(gateway: Gateway, key: str, body: dict[str, JsonValue]) -> str: + if body.get("stream") is True: + return _terminal_response_id(gateway, key, body) + return string_value(gateway.post("/v1/responses", body, key=key)["id"]) + + +def _spend_logs_lookup(gateway: Gateway, request_id: str) -> list[JsonValue]: + response: Final = gateway.request("GET", "/spend/logs", params={"request_id": request_id}) + assert response.status_code == 200, response.text + return [row["request_id"] for row in _JSON_LIST.validate_json(response.content)] + + def _request_ids_for_key(gateway: Gateway, key: str, count: int) -> list[str]: rows: Final = eventually( lambda: gateway.rows( @@ -152,6 +173,7 @@ def test_streamed_responses_spend_row_is_filed_under_the_id_the_client_saw( assert client_id.startswith("resp_"), client_id assert _request_ids_for_key(gateway, key, 1) == [client_id] assert _logs_page_search(gateway, client_id) == [client_id] + assert _spend_logs_lookup(gateway, client_id) == [client_id] def _prompt_storing_config(directory: Path) -> Path: @@ -196,3 +218,45 @@ def test_a_follow_up_turn_on_a_streamed_bridged_response_replays_the_first_turn( ("user", "second turn"), ], replayed assert _request_ids_for_key(storing, key, 2) == [first_id, second_id] + + +@pytest.mark.timeout(180) +@pytest.mark.parametrize("stream_first_turn", [True, False], ids=["streamed-first-turn", "non-streamed-first-turn"]) +def test_a_follow_up_through_the_id_a_retrieve_returned_replays_the_first_turn_once( + gateway: Gateway, tmp_path: Path, stream_first_turn: bool +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[native, bridged]) + first_id: Final = _created_response_id( + storing, key, {"model": native, "input": first_prompt, "stream": stream_first_turn} + ) + + retrieved: Final = storing.request("GET", f"/v1/responses/{first_id}", key=key) + + assert retrieved.status_code == 200, retrieved.text + retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"]) + assert retrieved_id == first_id + read_row_id: Final = f"{first_id}_retrieve_{retrieved.headers['x-litellm-call-id']}" + assert sorted(_request_ids_for_key(storing, key, 2)) == sorted([first_id, read_row_id]) + assert _spend_logs_lookup(storing, first_id) == [first_id] + wire.drain() + + _terminal_response_id( + storing, + key, + {"model": bridged, "input": "second turn", "previous_response_id": retrieved_id, "stream": True}, + ) + + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"]) + assert [(message["role"], message["content"]) for message in replayed] == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ], replayed diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..19f7c32437f 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -69,6 +69,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, @@ -5029,7 +5030,51 @@ def test_a_background_response_billing_event_writes_a_row_apart_from_the_read_th for call_id, metadata in (("call-read", read_metadata), ("call-bill", billing_metadata)) ) - assert request_ids == ("resp_shared", "resp_shared_background_cost") + assert request_ids == ("resp_shared_retrieve_call-read", "resp_shared_background_cost") + + +def _response_row_id(call_type: str, call_id: str, metadata: Mapping[str, object]) -> str: + return get_logging_payload( + kwargs={ + "call_type": call_type, + "model": "gpt-5.6", + "litellm_call_id": call_id, + "litellm_params": {"metadata": {"user_api_key": "test-key", **metadata}}, + }, + response_obj={"id": "resp_client_held", "object": "response", "status": "completed"}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), + )["request_id"] + + +def test_reads_and_cancels_of_a_response_never_take_the_row_id_of_the_create_that_made_it(): + """A GET or cancel answers with the id the client holds, which is the create row's id. Spend logs keep one row + per request id and drop the later duplicate, so a read written first would erase the create row a follow-up + turn replays its history from. Each read keeps a row of its own, as every poll is a separate request.""" + calls: Final = ( + ("aresponses", "call-create"), + ("aget_responses", "call-read-1"), + ("aget_responses", "call-read-2"), + ("acancel_responses", "call-cancel"), + ) + + request_ids: Final = tuple(_response_row_id(call_type, call_id, {}) for call_type, call_id in calls) + + assert request_ids[0] == "resp_client_held" + assert len(set(request_ids)) == len(calls), request_ids + assert all(request_id.startswith("resp_client_held_") for request_id in request_ids[1:]), request_ids + + +def test_the_cost_polls_own_status_check_keeps_the_response_id_as_its_row_id(): + """Only a client's read shares the create row's id. The cost poll's status check reads the response under its + managed id, which no create row carries, so it keeps the row id it had.""" + status_poll: Final = {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} + + assert _response_row_id("aget_responses", "call-status-poll", status_poll) == "resp_client_held" + + +def test_a_failed_read_keyed_by_its_own_call_id_keeps_that_id(): + assert get_spend_logs_id("aget_responses", {}, {"litellm_call_id": "call-failed-read"}) == "call-failed-read" @pytest.mark.parametrize("forged_marker", [True, BackgroundResponseSettlement.SETTLES.value]) diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index 75f0d33d37d..a9de17983d0 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -1097,6 +1097,79 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization: assert result["response_id"] != victim_provider_id +def _admin(): + from litellm.proxy._types import LitellmUserRoles + + return _auth(user_id="admin-user", team_id="admin-team", user_role=LitellmUserRoles.PROXY_ADMIN) + + +def _provider_answer(response_id): + return ResponsesAPIResponse(id=response_id, created_at=1234567890, output=[], status="completed") + + +class TestReadsAnswerWithTheIdTheClientHolds: + """A follow-up looks its history up by the id the client holds, so a read must not hand out a new one.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) + @pytest.mark.parametrize("caller", [_auth, _admin], ids=["owner", "proxy-admin"]) + async def test_a_read_or_cancel_answers_with_the_exact_id_the_client_sent( + self, mock_cache, salt_key_env, call_type, caller + ): + hook = _hook() + issued_id = _issue_managed_id(hook, _auth()) + data = await hook.async_pre_call_hook( + user_api_key_dict=caller(), cache=mock_cache, data={"response_id": issued_id}, call_type=call_type + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=caller(), response=_provider_answer(data["response_id"]) + ) + + assert result.id == issued_id + + @pytest.mark.asyncio + async def test_a_follow_up_gets_an_id_for_its_own_response_never_the_previous_one(self, mock_cache, salt_key_env): + hook = _hook() + owner = _auth() + previous_id = _issue_managed_id(hook, owner) + data = await hook.async_pre_call_hook( + user_api_key_dict=owner, + cache=mock_cache, + data={"model": "gpt-fake", "previous_response_id": previous_id}, + call_type="aresponses", + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=owner, response=_provider_answer("resp_followupprovideridzzzzzzzzzzzzzzz") + ) + + assert result.id != previous_id + assert hook._decrypt_response_id(result.id) == ( + "resp_followupprovideridzzzzzzzzzzzzzzz", + "owner-user", + "owner-team", + ) + + @pytest.mark.asyncio + async def test_an_admin_read_by_a_raw_provider_id_gets_a_managed_id_back(self, mock_cache, salt_key_env): + hook = _hook() + admin = _admin() + data = await hook.async_pre_call_hook( + user_api_key_dict=admin, + cache=mock_cache, + data={"response_id": _FABRICATED_UNMANAGED_ID}, + call_type="aget_responses", + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=admin, response=_provider_answer(_FABRICATED_UNMANAGED_ID) + ) + + assert hook._is_encrypted_response_id(result.id) + assert hook._decrypt_response_id(result.id) == (_FABRICATED_UNMANAGED_ID, "admin-user", "admin-team") + + _LOGGED_MANAGED_ID = "resp_managedidlitellmbuiltbeforethehookran" From 93483a056b1571025f5ea60a5c92af6f77fd7ce0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:58:58 -0700 Subject: [PATCH 3/6] fix(responses): log a fresh id for each read and cancel, echo the held id only to its owner A GET or cancel of a stored response now logs a new encrypted id of its own, so callbacks and spend rows never reuse the create's id, while the owner of the id still gets back the exact id they sent. An admin reading someone else's response gets an id bound to the admin, as before. The spend row id suffix for reads and cancels is gone, so with response id security off the rows and session history match the default branch. A streamed GET follows the same rules. --- litellm/proxy/hooks/responses_id_security.py | 41 +++-- .../spend_tracking/spend_tracking_utils.py | 22 +-- .../test_responses_retrieve_e2e.py | 3 - .../test_responses_stream_spend_row_id.py | 145 ++++++++++++++-- .../test_spend_tracking_utils.py | 44 +---- tests/unit/test_responses_id_security.py | 155 ++++++++++++++++-- 6 files changed, 318 insertions(+), 92 deletions(-) diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index d0a6231e7c8..a62b1f1b14e 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -117,6 +117,16 @@ def _streams_response_ids(request_route: str | None) -> bool: return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "") +_NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({}) + + +def _issued_to(user_id: str | None, team_id: str | None, user_api_key_dict: "UserAPIKeyAuth") -> bool: + caller_user_id: Final = user_api_key_dict.user_id or "" + if (user_id or "") != caller_user_id: + return False + return caller_user_id != "" or (team_id or "") == (user_api_key_dict.team_id or "") + + class ResponsesIDSecurity(CustomLogger): def __init__( self, @@ -325,22 +335,26 @@ class ResponsesIDSecurity(CustomLogger): if self._general_settings_reader().get("disable_responses_id_security", False): return None encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache={}) + if encrypt is None: + return None logging_obj: Final = request_data.get("litellm_logging_obj") - if encrypt is None or not isinstance(logging_obj, LiteLLMLoggingObj): - return encrypt + held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD), user_api_key_dict) def advertise(original_id: str) -> str: - advertised_id: Final = encrypt(original_id) - logging_obj.client_facing_response_id = advertised_id - return advertised_id + logged_id: Final = encrypt(original_id) + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.client_facing_response_id = logged_id + return held_ids.get(original_id, logged_id) return advertise - def _id_the_client_holds(self, response_id: str, addressed_id: object) -> str | None: + def _ids_the_caller_holds(self, addressed_id: object, user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, str]: if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id): - return None - original_response_id, _, _ = self._decrypt_response_id(addressed_id) - return addressed_id if original_response_id == response_id else None + return _NO_HELD_IDS + original_response_id, user_id, team_id = self._decrypt_response_id(addressed_id) + if not _issued_to(user_id, team_id, user_api_key_dict): + return _NO_HELD_IDS + return MappingProxyType({original_response_id: addressed_id}) def _advertised_response( self, @@ -348,10 +362,11 @@ class ResponsesIDSecurity(CustomLogger): user_api_key_dict: "UserAPIKeyAuth", request_data: Mapping[str, object], ) -> BaseLiteLLMOpenAIResponseObject: - held_id: Final = self._id_the_client_holds(response.id, request_data.get(ADDRESSED_RESPONSE_ID_FIELD)) - if held_id is None: - return self._encrypt_response_id(response, user_api_key_dict, request_cache=None) - return _rewrite_advertised_id(response, lambda _: held_id) + held_id: Final = self._ids_the_caller_holds( + request_data.get(ADDRESSED_RESPONSE_ID_FIELD), user_api_key_dict + ).get(response.id) + logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None) + return logged_response if held_id is None else logged_response.model_copy(update={"id": held_id}) async def async_post_call_success_hook( self, diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 6fd7bc6b052..94b079163e9 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -344,29 +344,14 @@ def _cost_row_request_id_suffix( return "" -_RESPONSE_LIFECYCLE_ROW_LABELS: Final[Mapping[str, str]] = MappingProxyType( - {"aget_responses": "retrieve", "acancel_responses": "cancel"} -) - - -def _response_lifecycle_row_suffix( - call_type: str, metadata: Mapping[str, object], resolved_id: str, litellm_call_id: object -) -> str: - label: Final = _RESPONSE_LIFECYCLE_ROW_LABELS.get(call_type) - if label is None or metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) or litellm_call_id == resolved_id: - return "" - return f"_{label}_{litellm_call_id}" - - def get_spend_logs_id( call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING ) -> str | None: standard_logging_payload = kwargs.get("standard_logging_object") - litellm_call_id: Final = kwargs.get("litellm_call_id") candidate_ids: Final = ( response_obj.get("id"), standard_logging_payload.get("id") if isinstance(standard_logging_payload, dict) else None, - litellm_call_id, + kwargs.get("litellm_call_id"), ) resolved_id: Final = next( (candidate for candidate in candidate_ids if isinstance(candidate, str) and candidate), None @@ -376,10 +361,7 @@ def get_spend_logs_id( settles_background_response: Final = ( kwargs.get(SETTLES_BACKGROUND_RESPONSE_KWARG) is BackgroundResponseSettlement.SETTLES ) - suffix: Final = _cost_row_request_id_suffix( - call_type, metadata, settles_background_response - ) or _response_lifecycle_row_suffix(call_type, metadata, resolved_id, litellm_call_id) - return f"{resolved_id}{suffix}" + return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}" _MISSING_ATTRIBUTE: Final = object() diff --git a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py index 4a08259c028..0e954af0952 100644 --- a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py +++ b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py @@ -69,9 +69,6 @@ def _retrieve_response(proxy: ProxyClient, key: str, response_id: str) -> Respon class TestResponsesRetrieve: - @pytest.mark.skip( - reason="stage red: product gap (LIT-5446), retrieve returns a different id than the stored response (non-idempotent response-id re-encryption)" - ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.works") @meta( Subject( diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py index d4283c03698..9be82285a3c 100644 --- a/tests/integration/spend/test_responses_stream_spend_row_id.py +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -1,10 +1,13 @@ import json +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from hashlib import sha256 from pathlib import Path +from types import MappingProxyType from typing import Final from uuid import uuid4 +import httpx import pytest import yaml from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value, string_value @@ -93,6 +96,10 @@ def _respond(request: Request) -> Reply: stored_id: Final = request.target.removeprefix("/v1/responses/") assert stored_id.startswith("resp_upstream-"), request.target return Reply(body=json.dumps(_native_completed(stored_id)).encode()) + if request.target.endswith("/cancel"): + cancelled_id: Final = request.target.removeprefix("/v1/responses/").removesuffix("/cancel") + assert cancelled_id.startswith("resp_upstream-"), request.target + return Reply(body=json.dumps(response_object(cancelled_id, "cancelled")).encode()) body: Final = JSON_OBJECT.validate_json(request.body) upstream_id: Final = f"upstream-{uuid4().hex}" if request.target == "/v1/responses" and body.get("stream") is not True: @@ -104,8 +111,10 @@ def _respond(request: Request) -> Reply: return _chat_stream(f"chatcmpl-{upstream_id}", string_value(body["model"])) -def _terminal_response_id(gateway: Gateway, key: str, body: dict[str, JsonValue]) -> str: - response: Final = gateway.request("POST", "/v1/responses", body, key=key) +def _terminal_response_id( + gateway: Gateway, key: str, body: dict[str, JsonValue], headers: Mapping[str, str] | None = None +) -> str: + response: Final = gateway.request("POST", "/v1/responses", body, key=key, headers=headers) assert response.status_code == 200, response.text frames: Final = tuple( JSON_OBJECT.validate_json(line.removeprefix("data: ")) @@ -120,10 +129,29 @@ def _terminal_response_id(gateway: Gateway, key: str, body: dict[str, JsonValue] return advertised[0] -def _created_response_id(gateway: Gateway, key: str, body: dict[str, JsonValue]) -> str: +def _created_response_id( + gateway: Gateway, key: str, body: dict[str, JsonValue], headers: Mapping[str, str] | None = None +) -> str: if body.get("stream") is True: - return _terminal_response_id(gateway, key, body) - return string_value(gateway.post("/v1/responses", body, key=key)["id"]) + return _terminal_response_id(gateway, key, body, headers) + response: Final = gateway.request("POST", "/v1/responses", body, key=key, headers=headers) + assert response.status_code == 200, response.text + return string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + +def _read_and_cancel( + gateway: Gateway, key: str, response_id: str, headers: Mapping[str, str] | None = None +) -> tuple[httpx.Response, ...]: + responses: Final = tuple( + gateway.request(method, path, key=key, headers=headers) + for method, path in ( + ("GET", f"/v1/responses/{response_id}"), + ("GET", f"/v1/responses/{response_id}"), + ("POST", f"/v1/responses/{response_id}/cancel"), + ) + ) + assert [response.status_code for response in responses] == [200, 200, 200], [r.text for r in responses] + return responses def _spend_logs_lookup(gateway: Gateway, request_id: str) -> list[JsonValue]: @@ -176,20 +204,29 @@ def test_streamed_responses_spend_row_is_filed_under_the_id_the_client_saw( assert _spend_logs_lookup(gateway, client_id) == [client_id] -def _prompt_storing_config(directory: Path) -> Path: +def _proxy_config( + directory: Path, + general_settings: Mapping[str, JsonValue], + litellm_settings: Mapping[str, JsonValue] = MappingProxyType({}), +) -> Path: stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) - path: Final = directory / "store-prompts.yaml" + path: Final = directory / "responses-spend.yaml" path.write_text( yaml.safe_dump( { **stock, - "general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True}, + "general_settings": {**object_value(stock["general_settings"]), **general_settings}, + "litellm_settings": {**object_value(stock["litellm_settings"]), **litellm_settings}, } ) ) return path +def _prompt_storing_config(directory: Path) -> Path: + return _proxy_config(directory, {"store_prompts_in_spend_logs": True}) + + def test_a_follow_up_turn_on_a_streamed_bridged_response_replays_the_first_turn( gateway: Gateway, tmp_path: Path ) -> None: @@ -243,8 +280,9 @@ def test_a_follow_up_through_the_id_a_retrieve_returned_replays_the_first_turn_o assert retrieved.status_code == 200, retrieved.text retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"]) assert retrieved_id == first_id - read_row_id: Final = f"{first_id}_retrieve_{retrieved.headers['x-litellm-call-id']}" - assert sorted(_request_ids_for_key(storing, key, 2)) == sorted([first_id, read_row_id]) + stored: Final = _request_ids_for_key(storing, key, 2) + assert stored.count(first_id) == 1, stored + assert len(set(stored)) == 2, stored assert _spend_logs_lookup(storing, first_id) == [first_id] wire.drain() @@ -260,3 +298,90 @@ def test_a_follow_up_through_the_id_a_retrieve_returned_replays_the_first_turn_o ("assistant", _ANSWER), ("user", "second turn"), ], replayed + + +@pytest.mark.timeout(180) +def test_each_read_and_cancel_reaches_callbacks_and_spend_logs_under_an_id_of_its_own( + gateway: Gateway, tmp_path: Path +) -> None: + logged_call_types: Final = frozenset({"aresponses", "aget_responses", "acancel_responses"}) + with ( + wire_server(_respond) as wire, + wire_server(lambda _: Reply()) as sink, + owned_proxy( + gateway, + tmp_path, + {"GENERIC_LOGGER_ENDPOINT": sink.url}, + config=_proxy_config(tmp_path, {}, {"callbacks": ["generic_api"], "DEFAULT_FLUSH_INTERVAL_SECONDS": 1}), + ) as logging_proxy, + logging_proxy.scenario() as scenario, + ): + model: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[model]) + first_id: Final = _created_response_id(logging_proxy, key, {"model": model, "input": f"marker-{uuid4().hex}"}) + + reads: Final = _read_and_cancel(logging_proxy, key, first_id) + + assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3 + batches: Final[list[bytes]] = [] # mutable-ok: drain() consumes, later polls keep earlier batches + + def delivered() -> tuple[tuple[str, str], ...]: + batches.extend(batch.body for batch in sink.drain()) + return tuple( + (string_value(event["call_type"]), string_value(event["id"])) + for event in (object_value(item) for batch in batches for item in _JSON_LIST.validate_json(batch)) + if event["call_type"] in logged_call_types + ) + + events: Final = eventually(delivered, lambda found: len(found) >= 4, seconds=30) + read_ids: Final = [logged_id for call_type, logged_id in events if call_type != "aresponses"] + assert [logged_id for call_type, logged_id in events if call_type == "aresponses"] == [first_id], events + assert len(read_ids) == 3, events + assert first_id not in read_ids, events + assert len(set(read_ids)) == 3, events + assert sorted(_request_ids_for_key(logging_proxy, key, 4)) == sorted([first_id, *read_ids]) + + +@pytest.mark.timeout(240) +def test_with_id_security_off_reads_and_cancels_add_no_rows_and_no_replayed_turns( + gateway: Gateway, tmp_path: Path +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + session: Final = {"x-litellm-session-id": f"session-{uuid4().hex}"} + with ( + wire_server(_respond) as wire, + owned_proxy( + gateway, + tmp_path, + {}, + config=_proxy_config( + tmp_path, {"store_prompts_in_spend_logs": True, "disable_responses_id_security": True} + ), + ) as insecure, + insecure.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[native, bridged]) + first_id: Final = _created_response_id(insecure, key, {"model": native, "input": first_prompt}, session) + + reads: Final = _read_and_cancel(insecure, key, first_id, session) + + assert [string_value(JSON_OBJECT.validate_json(read.content)["id"]) for read in reads] == [first_id] * 3 + later_id: Final = _created_response_id(insecure, key, {"model": native, "input": "a later request"}) + assert _request_ids_for_key(insecure, key, 2) == [first_id, later_id] + wire.drain() + + _terminal_response_id( + insecure, + key, + {"model": bridged, "input": "second turn", "previous_response_id": first_id, "stream": True}, + session, + ) + + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire.drain()[-1].body)["messages"]) + assert [(message["role"], message["content"]) for message in replayed] == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ], replayed diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 19f7c32437f..a4bd23b7989 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -69,7 +69,6 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, - BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, @@ -5030,51 +5029,24 @@ def test_a_background_response_billing_event_writes_a_row_apart_from_the_read_th for call_id, metadata in (("call-read", read_metadata), ("call-bill", billing_metadata)) ) - assert request_ids == ("resp_shared_retrieve_call-read", "resp_shared_background_cost") + assert request_ids == ("resp_shared", "resp_shared_background_cost") -def _response_row_id(call_type: str, call_id: str, metadata: Mapping[str, object]) -> str: - return get_logging_payload( +@pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) +def test_a_read_or_cancel_row_carries_the_id_its_callbacks_logged(call_type: str): + row: Final = get_logging_payload( kwargs={ "call_type": call_type, "model": "gpt-5.6", - "litellm_call_id": call_id, - "litellm_params": {"metadata": {"user_api_key": "test-key", **metadata}}, + "litellm_call_id": "call-read", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, }, - response_obj={"id": "resp_client_held", "object": "response", "status": "completed"}, + response_obj={"id": "resp_logged_by_callbacks", "object": "response", "status": "completed"}, start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), - )["request_id"] - - -def test_reads_and_cancels_of_a_response_never_take_the_row_id_of_the_create_that_made_it(): - """A GET or cancel answers with the id the client holds, which is the create row's id. Spend logs keep one row - per request id and drop the later duplicate, so a read written first would erase the create row a follow-up - turn replays its history from. Each read keeps a row of its own, as every poll is a separate request.""" - calls: Final = ( - ("aresponses", "call-create"), - ("aget_responses", "call-read-1"), - ("aget_responses", "call-read-2"), - ("acancel_responses", "call-cancel"), ) - request_ids: Final = tuple(_response_row_id(call_type, call_id, {}) for call_type, call_id in calls) - - assert request_ids[0] == "resp_client_held" - assert len(set(request_ids)) == len(calls), request_ids - assert all(request_id.startswith("resp_client_held_") for request_id in request_ids[1:]), request_ids - - -def test_the_cost_polls_own_status_check_keeps_the_response_id_as_its_row_id(): - """Only a client's read shares the create row's id. The cost poll's status check reads the response under its - managed id, which no create row carries, so it keeps the row id it had.""" - status_poll: Final = {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} - - assert _response_row_id("aget_responses", "call-status-poll", status_poll) == "resp_client_held" - - -def test_a_failed_read_keyed_by_its_own_call_id_keeps_that_id(): - assert get_spend_logs_id("aget_responses", {}, {"litellm_call_id": "call-failed-read"}) == "call-failed-read" + assert row["request_id"] == "resp_logged_by_callbacks" @pytest.mark.parametrize("forged_marker", [True, BackgroundResponseSettlement.SETTLES.value]) diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index e51707ce0db..95f54384ccd 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -12,7 +12,6 @@ from fastapi import HTTPException from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.responses_id_security import ( ResponsesIDSecurity, _streams_response_ids, @@ -1121,22 +1120,60 @@ class TestReadsAnswerWithTheIdTheClientHolds: @pytest.mark.asyncio @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) - @pytest.mark.parametrize("caller", [_auth, _admin], ids=["owner", "proxy-admin"]) - async def test_a_read_or_cancel_answers_with_the_exact_id_the_client_sent( - self, mock_cache, salt_key_env, call_type, caller + @pytest.mark.parametrize( + "issuer, reader", + [ + (_auth(), _auth()), + (_auth(team_id=None), _auth(team_id="another-team")), + (_auth(user_id=None), _auth(user_id=None)), + ], + ids=["same-key", "same-user-new-team", "same-team-no-user"], + ) + async def test_the_owner_gets_back_the_exact_id_they_sent( + self, mock_cache, salt_key_env, call_type, issuer, reader ): hook = _hook() - issued_id = _issue_managed_id(hook, _auth()) + issued_id = _issue_managed_id(hook, issuer) data = await hook.async_pre_call_hook( - user_api_key_dict=caller(), cache=mock_cache, data={"response_id": issued_id}, call_type=call_type + user_api_key_dict=reader, cache=mock_cache, data={"response_id": issued_id}, call_type=call_type ) result = await hook.async_post_call_success_hook( - data=data, user_api_key_dict=caller(), response=_provider_answer(data["response_id"]) + data=data, user_api_key_dict=reader, response=_provider_answer(data["response_id"]) ) assert result.id == issued_id + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) + @pytest.mark.parametrize( + "issuer, admin", + [ + (_auth(), _admin()), + (_auth(user_id=None), _auth(user_id=None, team_id="admin-team", user_role=_admin().user_role)), + ], + ids=["admin-user", "admin-without-user"], + ) + async def test_an_admin_reading_someone_elses_response_gets_an_id_of_their_own( + self, mock_cache, salt_key_env, call_type, issuer, admin + ): + hook = _hook() + issued_id = _issue_managed_id(hook, issuer) + data = await hook.async_pre_call_hook( + user_api_key_dict=admin, cache=mock_cache, data={"response_id": issued_id}, call_type=call_type + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=admin, response=_provider_answer(data["response_id"]) + ) + + assert result.id != issued_id + assert hook._decrypt_response_id(result.id) == ( + _FABRICATED_PROVIDER_RESPONSE_ID, + admin.user_id or "", + "admin-team", + ) + @pytest.mark.asyncio async def test_a_follow_up_gets_an_id_for_its_own_response_never_the_previous_one(self, mock_cache, salt_key_env): hook = _hook() @@ -1233,7 +1270,8 @@ class _SpendRowRecorder(CustomLogger): self.standard_logging_ids.append(kwargs["standard_logging_object"]["id"]) -async def _stream_logged_mid_stream(hook, route, logged_result): +async def _stream_logged_mid_stream(hook, route, logged_result, caller=None, request_data=None): + streaming_caller = caller if caller is not None else _auth() recorder = _SpendRowRecorder() logging_obj = LiteLLMLoggingObj( model="gpt-5.1", @@ -1255,9 +1293,9 @@ async def _stream_logged_mid_stream(hook, route, logged_result): yield frame client_stream = hook.async_post_call_streaming_iterator_hook( - user_api_key_dict=UserAPIKeyAuth(user_id="owner-user", team_id="owner-team", request_route=route), + user_api_key_dict=streaming_caller.model_copy(update={"request_route": route}), response=frames(), - request_data={"litellm_logging_obj": logging_obj}, + request_data={**(request_data or {}), "litellm_logging_obj": logging_obj}, ) first_frame = await anext(client_stream) now = datetime.datetime.now() @@ -1297,3 +1335,100 @@ class TestStreamedSpendRowCarriesTheIdTheClientSaw: assert client_id == _LOGGED_MANAGED_ID assert recorder.spend_row_ids == [_LOGGED_MANAGED_ID] assert recorder.standard_logging_ids == [_LOGGED_MANAGED_ID] + + +class TestStreamedReadAnswersWithTheIdTheClientHolds: + @pytest.mark.asyncio + @pytest.mark.parametrize("route_prefix", ["/v1/responses", "/responses", "/openai/v1/responses"]) + async def test_the_owner_sees_their_id_while_callbacks_log_one_of_the_reads_own( + self, mock_cache, salt_key_env, route_prefix + ): + hook = _hook() + owner = _auth() + issued_id = _issue_managed_id(hook, owner, provider_response_id=_LOGGED_MANAGED_ID) + data = await hook.async_pre_call_hook( + user_api_key_dict=owner, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses" + ) + + client_id, recorder = await _stream_logged_mid_stream( + hook, f"{route_prefix}/{issued_id}", _native_logged_result(), caller=owner, request_data=data + ) + + assert client_id == issued_id + assert recorder.spend_row_ids == recorder.standard_logging_ids + assert len(recorder.standard_logging_ids) == 1 + assert recorder.standard_logging_ids[0] != issued_id + assert hook._decrypt_response_id(recorder.standard_logging_ids[0]) == ( + _LOGGED_MANAGED_ID, + "owner-user", + "owner-team", + ) + + @pytest.mark.asyncio + async def test_an_admin_streamed_read_of_someone_elses_response_gets_and_logs_an_id_of_their_own( + self, mock_cache, salt_key_env + ): + hook = _hook() + admin = _admin() + issued_id = _issue_managed_id(hook, _auth(), provider_response_id=_LOGGED_MANAGED_ID) + data = await hook.async_pre_call_hook( + user_api_key_dict=admin, cache=mock_cache, data={"response_id": issued_id}, call_type="aget_responses" + ) + + client_id, recorder = await _stream_logged_mid_stream( + hook, f"/v1/responses/{issued_id}", _native_logged_result(), caller=admin, request_data=data + ) + + assert hook._decrypt_response_id(client_id) == (_LOGGED_MANAGED_ID, "admin-user", "admin-team") + assert recorder.spend_row_ids == [client_id] + assert recorder.standard_logging_ids == [client_id] + + +async def _read_logged_by_callbacks(hook, caller, issued_id, call_type, cache): + recorder = _SpendRowRecorder() + now = datetime.datetime.now() + logging_obj = LiteLLMLoggingObj( + model="gpt-5.1", + messages=[], + stream=False, + call_type=call_type, + start_time=now, + litellm_call_id=f"{call_type}-call", + function_id=f"{call_type}-fn", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gpt-5.1", user="", optional_params={}, litellm_params={"api_base": ""} + ) + data = await hook.async_pre_call_hook( + user_api_key_dict=caller, cache=cache, data={"response_id": issued_id}, call_type=call_type + ) + provider_answer = _provider_answer(data["response_id"]) + + client_response = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=caller, response=provider_answer + ) + await logging_obj.async_success_handler(result=provider_answer, start_time=now, end_time=now, cache_hit=False) + return client_response.id, recorder + + +class TestEveryReadLogsAnIdOfItsOwn: + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) + async def test_callbacks_and_the_spend_row_log_a_new_id_per_read_never_the_one_the_client_holds( + self, mock_cache, salt_key_env, call_type + ): + hook = _hook() + owner = _auth() + issued_id = _issue_managed_id(hook, owner) + + reads = [await _read_logged_by_callbacks(hook, owner, issued_id, call_type, mock_cache) for _ in range(2)] + + assert [client_id for client_id, _ in reads] == [issued_id, issued_id] + logged_ids = [id_ for _, recorder in reads for id_ in recorder.standard_logging_ids] + assert [recorder.spend_row_ids for _, recorder in reads] == [[logged_id] for logged_id in logged_ids] + assert len(set(logged_ids)) == 2 + assert issued_id not in logged_ids + assert {hook._decrypt_response_id(logged_id) for logged_id in logged_ids} == { + (_FABRICATED_PROVIDER_RESPONSE_ID, "owner-user", "owner-team") + } From 4f3d193758aa6451d0e4582bd2134be836ad0b75 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:10:51 -0700 Subject: [PATCH 4/6] fix(responses): echo the sent id to every reader that passes the response access check A GET, streamed GET or cancel of a response now answers with the id the caller sent whenever the caller passed the access check, not only when the caller is the user or team the id was issued to. An admin, the master key or a teammate who read someone else's streamed response got a fresh id back, and the session lookup could not reach the first turn from it, so a bridged follow-up through that id lost the conversation history Every read and cancel still logs a fresh id bound to the reader, never the create's id --- litellm/proxy/hooks/responses_id_security.py | 19 ++---- .../test_responses_stream_spend_row_id.py | 47 +++++++++++++ tests/unit/test_responses_id_security.py | 67 +++++++------------ 3 files changed, 77 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index a62b1f1b14e..83d3f8d870c 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -120,13 +120,6 @@ def _streams_response_ids(request_route: str | None) -> bool: _NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({}) -def _issued_to(user_id: str | None, team_id: str | None, user_api_key_dict: "UserAPIKeyAuth") -> bool: - caller_user_id: Final = user_api_key_dict.user_id or "" - if (user_id or "") != caller_user_id: - return False - return caller_user_id != "" or (team_id or "") == (user_api_key_dict.team_id or "") - - class ResponsesIDSecurity(CustomLogger): def __init__( self, @@ -338,7 +331,7 @@ class ResponsesIDSecurity(CustomLogger): if encrypt is None: return None logging_obj: Final = request_data.get("litellm_logging_obj") - held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD), user_api_key_dict) + held_ids: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD)) def advertise(original_id: str) -> str: logged_id: Final = encrypt(original_id) @@ -348,12 +341,10 @@ class ResponsesIDSecurity(CustomLogger): return advertise - def _ids_the_caller_holds(self, addressed_id: object, user_api_key_dict: "UserAPIKeyAuth") -> Mapping[str, str]: + def _ids_the_caller_holds(self, addressed_id: object) -> Mapping[str, str]: if not isinstance(addressed_id, str) or not self._is_encrypted_response_id(addressed_id): return _NO_HELD_IDS - original_response_id, user_id, team_id = self._decrypt_response_id(addressed_id) - if not _issued_to(user_id, team_id, user_api_key_dict): - return _NO_HELD_IDS + original_response_id, _, _ = self._decrypt_response_id(addressed_id) return MappingProxyType({original_response_id: addressed_id}) def _advertised_response( @@ -362,9 +353,7 @@ class ResponsesIDSecurity(CustomLogger): user_api_key_dict: "UserAPIKeyAuth", request_data: Mapping[str, object], ) -> BaseLiteLLMOpenAIResponseObject: - held_id: Final = self._ids_the_caller_holds( - request_data.get(ADDRESSED_RESPONSE_ID_FIELD), user_api_key_dict - ).get(response.id) + held_id: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD)).get(response.id) logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None) return logged_response if held_id is None else logged_response.model_copy(update={"id": held_id}) diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py index 9be82285a3c..557d4b329ed 100644 --- a/tests/integration/spend/test_responses_stream_spend_row_id.py +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -300,6 +300,53 @@ def test_a_follow_up_through_the_id_a_retrieve_returned_replays_the_first_turn_o ], replayed +def _replayed_turns(wire_body: bytes) -> list[tuple[JsonValue, JsonValue]]: + replayed: Final = _JSON_LIST.validate_python(JSON_OBJECT.validate_json(wire_body)["messages"]) + return [(message["role"], message["content"]) for message in replayed] + + +@pytest.mark.timeout(180) +@pytest.mark.parametrize("reader", ["proxy-admin", "master-key"]) +def test_a_reader_other_than_the_creator_continues_through_the_id_its_retrieve_returned( + gateway: Gateway, tmp_path: Path, reader: str +) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + creator: Final = scenario.key(user_id=scenario.user(), models=[native, bridged]) + reader_key: Final = ( + scenario.key(user_id=scenario.user(user_role="proxy_admin"), models=[native, bridged]) + if reader == "proxy-admin" + else storing.key + ) + first_id: Final = _terminal_response_id( + storing, creator, {"model": native, "input": first_prompt, "stream": True} + ) + assert _request_ids_for_key(storing, creator, 1) == [first_id] + + retrieved: Final = storing.request("GET", f"/v1/responses/{first_id}", key=reader_key) + + assert retrieved.status_code == 200, retrieved.text + retrieved_id: Final = string_value(JSON_OBJECT.validate_json(retrieved.content)["id"]) + wire.drain() + _terminal_response_id( + storing, + reader_key, + {"model": bridged, "input": "second turn", "previous_response_id": retrieved_id, "stream": True}, + ) + assert _replayed_turns(wire.drain()[-1].body) == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ] + assert retrieved_id == first_id + + @pytest.mark.timeout(180) def test_each_read_and_cancel_reaches_callbacks_and_spend_logs_under_an_id_of_its_own( gateway: Gateway, tmp_path: Path diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index 95f54384ccd..a44fb0d9427 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -1126,10 +1126,20 @@ class TestReadsAnswerWithTheIdTheClientHolds: (_auth(), _auth()), (_auth(team_id=None), _auth(team_id="another-team")), (_auth(user_id=None), _auth(user_id=None)), + (_auth(user_id=None), _auth(user_id="teammate-user")), + (_auth(), _admin()), + (_auth(user_id=None), _auth(user_id=None, team_id="admin-team", user_role=_admin().user_role)), + ], + ids=[ + "same-key", + "same-user-new-team", + "same-team-no-user", + "teammate-reading-a-team-key-id", + "admin-user", + "admin-without-user", ], - ids=["same-key", "same-user-new-team", "same-team-no-user"], ) - async def test_the_owner_gets_back_the_exact_id_they_sent( + async def test_a_caller_with_access_gets_back_the_exact_id_they_sent( self, mock_cache, salt_key_env, call_type, issuer, reader ): hook = _hook() @@ -1144,36 +1154,6 @@ class TestReadsAnswerWithTheIdTheClientHolds: assert result.id == issued_id - @pytest.mark.asyncio - @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) - @pytest.mark.parametrize( - "issuer, admin", - [ - (_auth(), _admin()), - (_auth(user_id=None), _auth(user_id=None, team_id="admin-team", user_role=_admin().user_role)), - ], - ids=["admin-user", "admin-without-user"], - ) - async def test_an_admin_reading_someone_elses_response_gets_an_id_of_their_own( - self, mock_cache, salt_key_env, call_type, issuer, admin - ): - hook = _hook() - issued_id = _issue_managed_id(hook, issuer) - data = await hook.async_pre_call_hook( - user_api_key_dict=admin, cache=mock_cache, data={"response_id": issued_id}, call_type=call_type - ) - - result = await hook.async_post_call_success_hook( - data=data, user_api_key_dict=admin, response=_provider_answer(data["response_id"]) - ) - - assert result.id != issued_id - assert hook._decrypt_response_id(result.id) == ( - _FABRICATED_PROVIDER_RESPONSE_ID, - admin.user_id or "", - "admin-team", - ) - @pytest.mark.asyncio async def test_a_follow_up_gets_an_id_for_its_own_response_never_the_previous_one(self, mock_cache, salt_key_env): hook = _hook() @@ -1365,7 +1345,7 @@ class TestStreamedReadAnswersWithTheIdTheClientHolds: ) @pytest.mark.asyncio - async def test_an_admin_streamed_read_of_someone_elses_response_gets_and_logs_an_id_of_their_own( + async def test_an_admin_streamed_read_of_someone_elses_response_sees_the_id_sent_and_logs_one_of_its_own( self, mock_cache, salt_key_env ): hook = _hook() @@ -1379,9 +1359,14 @@ class TestStreamedReadAnswersWithTheIdTheClientHolds: hook, f"/v1/responses/{issued_id}", _native_logged_result(), caller=admin, request_data=data ) - assert hook._decrypt_response_id(client_id) == (_LOGGED_MANAGED_ID, "admin-user", "admin-team") - assert recorder.spend_row_ids == [client_id] - assert recorder.standard_logging_ids == [client_id] + assert client_id == issued_id + assert recorder.spend_row_ids == recorder.standard_logging_ids + assert len(recorder.standard_logging_ids) == 1 + assert hook._decrypt_response_id(recorder.standard_logging_ids[0]) == ( + _LOGGED_MANAGED_ID, + "admin-user", + "admin-team", + ) async def _read_logged_by_callbacks(hook, caller, issued_id, call_type, cache): @@ -1415,14 +1400,14 @@ async def _read_logged_by_callbacks(hook, caller, issued_id, call_type, cache): class TestEveryReadLogsAnIdOfItsOwn: @pytest.mark.asyncio @pytest.mark.parametrize("call_type", ["aget_responses", "acancel_responses"]) + @pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"]) async def test_callbacks_and_the_spend_row_log_a_new_id_per_read_never_the_one_the_client_holds( - self, mock_cache, salt_key_env, call_type + self, mock_cache, salt_key_env, call_type, reader ): hook = _hook() - owner = _auth() - issued_id = _issue_managed_id(hook, owner) + issued_id = _issue_managed_id(hook, _auth()) - reads = [await _read_logged_by_callbacks(hook, owner, issued_id, call_type, mock_cache) for _ in range(2)] + reads = [await _read_logged_by_callbacks(hook, reader, issued_id, call_type, mock_cache) for _ in range(2)] assert [client_id for client_id, _ in reads] == [issued_id, issued_id] logged_ids = [id_ for _, recorder in reads for id_ in recorder.standard_logging_ids] @@ -1430,5 +1415,5 @@ class TestEveryReadLogsAnIdOfItsOwn: assert len(set(logged_ids)) == 2 assert issued_id not in logged_ids assert {hook._decrypt_response_id(logged_id) for logged_id in logged_ids} == { - (_FABRICATED_PROVIDER_RESPONSE_ID, "owner-user", "owner-team") + (_FABRICATED_PROVIDER_RESPONSE_ID, reader.user_id, reader.team_id) } From 3ae300a7bc529585bbe270460ab253293235f1f6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:52:02 -0700 Subject: [PATCH 5/6] fix(responses): keep a read or cancel row in session history only when it brings new output A GET, streamed GET or cancel sent with a session header joins the create's session, and its stored response repeats the create's output. The replay then sent the same assistant turn once per read. Session history now keeps an aget_responses or acancel_responses row only when its stored Responses output has an item id that no earlier kept row carried. A polled background response keeps the first poll that returns the finished output, since its queued create row has none, and later polls of the same output are dropped. Every other row keeps today's rule --- .../session_handler.py | 56 +++++++++++++++++-- .../test_responses_stream_spend_row_id.py | 31 ++++++++++ .../test_session_handler.py | 52 +++++++++++++++++ 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index ce2396355a2..157375b4d89 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -2,6 +2,7 @@ import asyncio import json from collections.abc import Mapping, Sequence from dataclasses import dataclass +from functools import reduce from itertools import chain from typing import TYPE_CHECKING, Any, Final, cast @@ -69,6 +70,51 @@ def _is_a_conversation_turn(spend_log: "SpendLogsPayload") -> bool: return spend_log.get("call_type") not in _MCP_TOOL_CALL_TYPES and not _is_a_failed_turn(spend_log) +_RESPONSE_READ_CALL_TYPES: Final = frozenset(("aget_responses", "acancel_responses")) + + +def _output_item_id(item: object) -> str | None: + match item: + case {"id": str() as item_id}: + return item_id + case _: + return None + + +def _is_a_response_read(spend_log: "SpendLogsPayload") -> bool: + return "call_type" in spend_log and spend_log["call_type"] in _RESPONSE_READ_CALL_TYPES + + +def _stored_response(spend_log: "SpendLogsPayload") -> object: + return spend_log.get("response") + + +def _output_item_ids(spend_log: "SpendLogsPayload") -> frozenset[str]: + output: Final = _stored_responses_api_output(_stored_response(spend_log)) or [] + return frozenset(item_id for item_id in map(_output_item_id, output) if item_id is not None) + + +@dataclass(frozen=True, slots=True) +class _SessionTurns: + spend_logs: tuple["SpendLogsPayload", ...] = () + output_item_ids: frozenset[str] = frozenset() + + +def _with_conversation_turn(turns: _SessionTurns, spend_log: "SpendLogsPayload") -> _SessionTurns: + if not _is_a_conversation_turn(spend_log): + return turns + output_item_ids: Final = _output_item_ids(spend_log) + if _is_a_response_read(spend_log) and output_item_ids <= turns.output_item_ids: + return turns + return _SessionTurns( + spend_logs=(*turns.spend_logs, spend_log), output_item_ids=turns.output_item_ids | output_item_ids + ) + + +def _conversation_turns(spend_logs: Sequence["SpendLogsPayload"]) -> tuple["SpendLogsPayload", ...]: + return reduce(_with_conversation_turn, spend_logs, _SessionTurns()).spend_logs + + _MCP_AUTO_EXECUTE_OUTPUT_TYPES: Final = frozenset(("mcp_tools_fetched", "tool_execution_results")) @@ -101,7 +147,7 @@ class StoredTurns: def _compaction_point( index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None ) -> _ReplayStart | None: - if response_holds_compaction_block(spend_log.get("response")): + if response_holds_compaction_block(_stored_response(spend_log)): return _ReplayStart(index=index, at_output=True) if proxy_server_request is not None and holds_compaction_summary(proxy_server_request.get("input")): return _ReplayStart(index=index, at_output=False) @@ -266,12 +312,10 @@ class ResponsesSessionHandler: previous_response_id: str, client_response_ids: tuple[str, ...] = (), ) -> StoredTurns: - spend_logs: Final = tuple( - spend_log - for spend_log in await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( + spend_logs: Final = _conversation_turns( + await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( previous_response_id, client_response_ids ) - if _is_a_conversation_turn(spend_log) ) return StoredTurns( spend_logs=spend_logs, @@ -369,7 +413,7 @@ class ResponsesSessionHandler: ############################################################ # Add Output messages for this Spend Log ############################################################ - _response_output: Final[object] = spend_log.get("response", "{}") + _response_output: Final = _stored_response(spend_log) if (responses_api_output := _stored_responses_api_output(_response_output)) is not None: chat_completion_message_history.extend( LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( diff --git a/tests/integration/spend/test_responses_stream_spend_row_id.py b/tests/integration/spend/test_responses_stream_spend_row_id.py index 557d4b329ed..58ed9f7785f 100644 --- a/tests/integration/spend/test_responses_stream_spend_row_id.py +++ b/tests/integration/spend/test_responses_stream_spend_row_id.py @@ -347,6 +347,37 @@ def test_a_reader_other_than_the_creator_continues_through_the_id_its_retrieve_r assert retrieved_id == first_id +@pytest.mark.timeout(180) +def test_reads_and_cancels_sent_with_a_session_header_add_no_replayed_turns(gateway: Gateway, tmp_path: Path) -> None: + first_prompt: Final = f"first turn {uuid4().hex}" + session: Final = {"x-litellm-session-id": f"session-{uuid4().hex}"} + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storing_config(tmp_path)) as storing, + storing.scenario() as scenario, + ): + native: Final = scenario.model(model=_NATIVE_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + bridged: Final = scenario.model(model=_BRIDGED_MODEL, api_base=f"{wire.url}/v1", api_key="synthetic-key") + key: Final = scenario.key(models=[native, bridged]) + first_id: Final = _created_response_id(storing, key, {"model": native, "input": first_prompt}, session) + + _read_and_cancel(storing, key, first_id, session) + + assert len(_request_ids_for_key(storing, key, 4)) == 4 + wire.drain() + _terminal_response_id( + storing, + key, + {"model": bridged, "input": "second turn", "previous_response_id": first_id, "stream": True}, + session, + ) + assert _replayed_turns(wire.drain()[-1].body) == [ + ("user", first_prompt), + ("assistant", _ANSWER), + ("user", "second turn"), + ] + + @pytest.mark.timeout(180) def test_each_read_and_cancel_reaches_callbacks_and_spend_logs_under_an_id_of_its_own( gateway: Gateway, tmp_path: Path diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 8d5befd9443..1c97574d7f5 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1081,6 +1081,58 @@ async def test_previous_response_history_starts_at_a_compaction_item_in_a_stored ] +def _stored_response_turn( + request_id: str, call_type: str, status: str, output: list[dict[str, object]], prompt: str | None = None +) -> dict[str, object]: + return { + "request_id": request_id, + "call_type": call_type, + "session_id": "session-with-reads", + "proxy_server_request": json.dumps({} if prompt is None else {"input": [{"role": "user", "content": prompt}]}), + "response": {"id": request_id, "object": "response", "status": status, "output": output}, + } + + +def _identified_output_message(item_id: str, text: str) -> dict[str, object]: + return {**_output_message(text), "id": item_id} + + +def _replayed_roles_and_texts(session: ChatCompletionSession) -> list[tuple[object, object]]: + return [ + (role, _text_of(content)) for role, content in (_role_and_content(message) for message in session["messages"]) + ] + + +@pytest.mark.asyncio +async def test_reads_and_cancels_of_a_response_the_session_already_holds_replay_no_turns(): + answer: Final = [_identified_output_message("msg_answer", "OK")] + spend_logs: Final = ( + _stored_response_turn("resp_create", "aresponses", "completed", answer, prompt="Reply with OK"), + _stored_response_turn("resp_read", "aget_responses", "completed", answer), + _stored_response_turn("resp_streamed_read", "aget_responses", "completed", answer), + _stored_response_turn("resp_cancel", "acancel_responses", "completed", answer), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_create") + + assert _replayed_roles_and_texts(session) == [("user", "Reply with OK"), ("assistant", "OK")] + + +@pytest.mark.asyncio +async def test_a_polled_background_response_replays_its_finished_output_once(): + answer: Final = [_identified_output_message("msg_answer", "One, two, three")] + spend_logs: Final = ( + _stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to three"), + _stored_response_turn("resp_poll_running", "aget_responses", "in_progress", []), + _stored_response_turn("resp_poll_done", "aget_responses", "completed", answer), + _stored_response_turn("resp_poll_again", "aget_responses", "completed", answer), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_queued") + + assert _replayed_roles_and_texts(session) == [("user", "Count to three"), ("assistant", "One, two, three")] + + def _tool_call_ids(message: object) -> tuple[str, ...]: tool_calls: Final = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) return tuple(call["id"] if isinstance(call, dict) else call.id for call in tool_calls or ()) From eaa19d89f62856708e53839f076e8aca1e267ad1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:31:14 -0700 Subject: [PATCH 6/6] fix(responses): replay each history item once, echo the sent id on delete, keep background cost files apart A create row now owns its output items in session history, and a read or cancel row adds only items no create and no earlier read carried, in one linear pass, so a background response polled while it grows, or a read that ends before its streamed create, replays the answer once and after its prompt DELETE /v1/responses/{id} answers with the id the caller sent, on a copy, and logs a fresh encrypted id like a read does Azure Data Lake Storage per-event files and unbatched GCS objects are named after the event's spend row id, so a streamed background create and its cost event, which now share the client's id, keep separate files The integration tests that decoded the managed id now match the exact id the client received --- .../azure_storage/azure_storage.py | 50 +++- litellm/integrations/gcs_bucket/gcs_bucket.py | 38 +-- litellm/proxy/hooks/responses_id_security.py | 42 ++- .../spend_tracking/spend_tracking_utils.py | 17 +- .../session_handler.py | 62 ++-- .../test_openai_sdk_client_paths.py | 15 +- .../test_langfuse_otel_metadata_chaos.py | 6 +- .../test_langfuse_otel_metadata_clients.py | 9 +- .../test_otel_conversation_id.py | 44 +-- .../observability/test_signoz_delivery.py | 37 +-- .../test_bedrock_mantle_output_limit_wire.py | 2 +- ..._bedrock_runtime_chat_completions_chaos.py | 6 +- .../azure_storage/test_azure_storage.py | 170 ++++++++++- .../gcs_bucket/test_gcs_bucket.py | 98 ++++++- .../test_session_handler.py | 29 ++ tests/unit/test_responses_id_security.py | 271 +++++++++--------- 16 files changed, 612 insertions(+), 284 deletions(-) diff --git a/litellm/integrations/azure_storage/azure_storage.py b/litellm/integrations/azure_storage/azure_storage.py index 37b4d2d3937..dd2e99f8a59 100644 --- a/litellm/integrations/azure_storage/azure_storage.py +++ b/litellm/integrations/azure_storage/azure_storage.py @@ -1,7 +1,8 @@ import asyncio import os import time -from collections.abc import Callable +from collections.abc import Callable, Mapping +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from functools import cache from typing import Final @@ -47,6 +48,21 @@ def adls_safe_file_name(payload_id: str | None) -> str: return f"{name.translate(_ADLS_SAFE_NAME)}.json" +@dataclass(frozen=True, slots=True) +class AzureStorageLogQueueItem: + payload: StandardLoggingPayload + file_name: str + + +def _queued_log(kwargs: Mapping[str, object], payload: StandardLoggingPayload) -> AzureStorageLogQueueItem: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logged_event_spend_logs_id + + return AzureStorageLogQueueItem( + payload=payload, + file_name=adls_safe_file_name(get_logged_event_spend_logs_id(kwargs, payload.get("id"))), + ) + + @cache def _cached_credential_chain_token_provider() -> Callable[[], str]: return get_azure_ad_token_provider( @@ -99,7 +115,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() - self.log_queue: list[StandardLoggingPayload] = [] + self.log_queue: list[AzureStorageLogQueueItem] = [] super().__init__(**kwargs, flush_lock=self.flush_lock) except Exception as e: verbose_logger.exception( @@ -115,7 +131,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): def azure_storage_blob_endpoint(self) -> str: return f"https://{self.azure_storage_account_name}.blob.{self.azure_storage_endpoint_suffix}" - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime): """ Async Log success events to Azure Blob Storage @@ -133,12 +149,12 @@ class AzureBlobStorageLogger(CustomBatchLogger): if standard_logging_payload is None: raise ValueError("standard_logging_payload is not set") - self.log_queue.append(standard_logging_payload) + self.log_queue.append(_queued_log(kwargs, standard_logging_payload)) except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event(self, kwargs, response_obj: object, start_time: datetime, end_time: datetime): """ Async Log failure events to Azure Blob Storage @@ -156,7 +172,7 @@ class AzureBlobStorageLogger(CustomBatchLogger): if standard_logging_payload is None: raise ValueError("standard_logging_payload is not set") - self.log_queue.append(standard_logging_payload) + self.log_queue.append(_queued_log(kwargs, standard_logging_payload)) except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Layer Error - %s", e) @@ -195,13 +211,15 @@ class AzureBlobStorageLogger(CustomBatchLogger): len(self.log_queue), ) - for payload in self.log_queue: - await self.async_upload_payload_to_azure_blob_storage(payload=payload) + for item in self.log_queue: + await self.async_upload_payload_to_azure_blob_storage(payload=item.payload, file_name=item.file_name) except Exception as e: verbose_logger.exception("AzureBlobStorageLogger Error sending batch API - %s", e) - async def async_upload_payload_to_azure_blob_storage(self, payload: StandardLoggingPayload): + async def async_upload_payload_to_azure_blob_storage( + self, payload: StandardLoggingPayload, file_name: str | None = None + ): """ Uploads the payload to Azure Blob Storage using a 3-step process: 1. Create file resource @@ -210,14 +228,14 @@ class AzureBlobStorageLogger(CustomBatchLogger): """ try: if self.azure_storage_account_key: - await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload) + await self.upload_to_azure_data_lake_with_azure_account_key(payload=payload, file_name=file_name) else: # Get a valid token instead of always requesting a new one await self.set_valid_azure_ad_token() async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) json_payload: Final = safe_dumps(payload) + "\n" # Add newline for each log entry payload_bytes: Final = json_payload.encode("utf-8") - filename: Final = adls_safe_file_name(payload.get("id")) + filename: Final = file_name or adls_safe_file_name(payload.get("id")) base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}" # Execute the 3-step upload process @@ -377,7 +395,9 @@ class AzureBlobStorageLogger(CustomBatchLogger): self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS return self._service_client - async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload): + async def upload_to_azure_data_lake_with_azure_account_key( + self, payload: StandardLoggingPayload, file_name: str | None = None + ): """ Uploads the payload to Azure Data Lake using the Azure SDK @@ -386,11 +406,11 @@ class AzureBlobStorageLogger(CustomBatchLogger): try: today: Final = datetime.now().strftime("%Y-%m-%d") - file_name: Final = adls_safe_file_name(payload.get("id")) + resolved_file_name: Final = file_name or adls_safe_file_name(payload.get("id")) await self._upload_bytes_with_account_key( - file_path=f"{today}/{file_name}", content=safe_dumps(payload).encode("utf-8") + file_path=f"{today}/{resolved_file_name}", content=safe_dumps(payload).encode("utf-8") ) - verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, file_name) + verbose_logger.debug("Successfully uploaded and wrote to %s/%s", today, resolved_file_name) except Exception as e: verbose_logger.exception("Error occurred: %s", e) diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index 99cd8b31b63..558839b56c8 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -296,30 +296,30 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): else: await self._send_individual_logs(items_to_process) - def _get_object_name(self, kwargs: dict, logging_payload: StandardLoggingPayload, response_obj: Any) -> str: + def _get_object_name( + self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any + ) -> str: """ Get the object name to use for the current payload """ + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logged_event_spend_logs_id + current_date: Final = self._get_object_date_from_datetime(datetime.now(timezone.utc)) + metadata: Final = as_str_mapping((as_str_mapping(kwargs.get("litellm_params")) or {}).get("metadata")) or {} + gcs_log_id: Final = metadata.get("gcs_log_id") + safe_log_id: Final = sanitize_cloud_object_component( + gcs_log_id if isinstance(gcs_log_id, str) else None, fallback="" + ) + if safe_log_id: + return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}" if logging_payload.get("error_str", None) is not None: - object_name = self._generate_failure_object_name( - request_date_str=current_date, - ) - else: - object_name = self._generate_success_object_name( - request_date_str=current_date, - response_id=response_obj.get("id", ""), - ) - - # used for testing - _litellm_params: Final = kwargs.get("litellm_params", None) or {} - _metadata: Final = _litellm_params.get("metadata", None) or {} - if "gcs_log_id" in _metadata: - safe_log_id: Final = sanitize_cloud_object_component(_metadata.get("gcs_log_id"), fallback="") - if safe_log_id: - object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}" - - return object_name + return self._generate_failure_object_name(request_date_str=current_date) + raw_response_id: Final[object] = response_obj.get("id", "") + response_id: Final = raw_response_id if isinstance(raw_response_id, str) else "" + return self._generate_success_object_name( + request_date_str=current_date, + response_id=get_logged_event_spend_logs_id(kwargs, response_id or None) or response_id, + ) async def get_request_response_payload( self, diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 83d3f8d870c..6d12ccf803d 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -7,7 +7,7 @@ instead of writing immediately on each request. from collections.abc import AsyncGenerator, Callable, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeVar from fastapi import HTTPException from pydantic import TypeAdapter, ValidationError @@ -20,10 +20,12 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, ResponsesAPIResponse, ) +from litellm.types.responses.main import DeleteResponseResult from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums if TYPE_CHECKING: @@ -68,6 +70,8 @@ def _proxy_signing_key() -> str | None: _RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_ResponseObjectT = TypeVar("_ResponseObjectT", bound=BaseLiteLLMOpenAIResponseObject) +_AddressedResultT = TypeVar("_AddressedResultT", bound=ResponsesAPIResponse | DeleteResponseResult) def _response_payload(response_obj: object) -> Mapping[str, object] | None: @@ -78,9 +82,9 @@ def _response_payload(response_obj: object) -> Mapping[str, object] | None: def _rewrite_advertised_id( - event: BaseLiteLLMOpenAIResponseObject, + event: _ResponseObjectT, rewrite: Callable[[str], str], -) -> BaseLiteLLMOpenAIResponseObject: +) -> _ResponseObjectT: event_id: Final = getattr(event, "id", None) if isinstance(event_id, str) and event_id.startswith("resp_"): setattr(event, "id", rewrite(event_id)) @@ -120,6 +124,14 @@ def _streams_response_ids(request_route: str | None) -> bool: _NO_HELD_IDS: Final[Mapping[str, str]] = MappingProxyType({}) +def _upstream_response_id(response_id: str) -> str: + return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id) + + +def _names_the_same_response(response_id: str | None, original_response_id: str) -> bool: + return response_id is not None and _upstream_response_id(response_id) == _upstream_response_id(original_response_id) + + class ResponsesIDSecurity(CustomLogger): def __init__( self, @@ -279,10 +291,10 @@ class ResponsesIDSecurity(CustomLogger): def _encrypt_response_id( self, - response: BaseLiteLLMOpenAIResponseObject, + response: _ResponseObjectT, user_api_key_dict: "UserAPIKeyAuth", request_cache: dict[str, str] | None = None, - ) -> BaseLiteLLMOpenAIResponseObject: + ) -> _ResponseObjectT: encrypt: Final = self._response_id_encrypter(user_api_key_dict, request_cache) return response if encrypt is None else _rewrite_advertised_id(response, encrypt) @@ -347,13 +359,23 @@ class ResponsesIDSecurity(CustomLogger): original_response_id, _, _ = self._decrypt_response_id(addressed_id) return MappingProxyType({original_response_id: addressed_id}) + def _id_the_caller_holds(self, response_id: str | None, addressed_id: object) -> str | None: + return next( + ( + held_id + for original_response_id, held_id in self._ids_the_caller_holds(addressed_id).items() + if _names_the_same_response(response_id, original_response_id) + ), + None, + ) + def _advertised_response( self, - response: ResponsesAPIResponse, + response: _AddressedResultT, user_api_key_dict: "UserAPIKeyAuth", request_data: Mapping[str, object], - ) -> BaseLiteLLMOpenAIResponseObject: - held_id: Final = self._ids_the_caller_holds(request_data.get(ADDRESSED_RESPONSE_ID_FIELD)).get(response.id) + ) -> _AddressedResultT: + held_id: Final = self._id_the_caller_holds(response.id, request_data.get(ADDRESSED_RESPONSE_ID_FIELD)) logged_response: Final = self._encrypt_response_id(response, user_api_key_dict, request_cache=None) return logged_response if held_id is None else logged_response.model_copy(update={"id": held_id}) @@ -367,8 +389,8 @@ class ResponsesIDSecurity(CustomLogger): if general_settings.get("disable_responses_id_security", False): return response - if isinstance(response, ResponsesAPIResponse): - response = cast(ResponsesAPIResponse, self._advertised_response(response, user_api_key_dict, data)) + if isinstance(response, ResponsesAPIResponse | DeleteResponseResult): + return self._advertised_response(response, user_api_key_dict, data) return response async def async_post_call_streaming_iterator_hook( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..f0c01b04f41 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -45,6 +45,7 @@ from litellm.litellm_core_utils.classifier_logging import ( without_classifier_audit, ) from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, get_litellm_metadata_from_kwargs, proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, @@ -345,7 +346,10 @@ def _cost_row_request_id_suffix( def get_spend_logs_id( - call_type: str, response_obj: dict, kwargs: Mapping[str, object], metadata: Mapping[str, object] = EMPTY_MAPPING + call_type: str, + response_obj: Mapping[str, object], + kwargs: Mapping[str, object], + metadata: Mapping[str, object] = EMPTY_MAPPING, ) -> str | None: standard_logging_payload = kwargs.get("standard_logging_object") candidate_ids: Final = ( @@ -364,6 +368,17 @@ def get_spend_logs_id( return f"{resolved_id}{_cost_row_request_id_suffix(call_type, metadata, settles_background_response)}" +def get_logged_event_spend_logs_id(kwargs: Mapping[str, object], event_id: str | None) -> str | None: + call_type: Final = kwargs.get("call_type") + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {} + return get_spend_logs_id( + call_type if isinstance(call_type, str) and call_type else "acompletion", + {"id": event_id} if event_id else {}, + kwargs, + as_str_mapping(litellm_params.get("litellm_metadata")) or as_str_mapping(litellm_params.get("metadata")) or {}, + ) + + _MISSING_ATTRIBUTE: Final = object() diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 157375b4d89..2c3a8530d12 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -1,15 +1,16 @@ import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass -from functools import reduce from itertools import chain +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, cast import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.responses.litellm_completion_transformation.client_tool_search import ( bridge_client_tool_search, client_tool_search_loaded_tools, @@ -94,25 +95,52 @@ def _output_item_ids(spend_log: "SpendLogsPayload") -> frozenset[str]: return frozenset(item_id for item_id in map(_output_item_id, output) if item_id is not None) -@dataclass(frozen=True, slots=True) -class _SessionTurns: - spend_logs: tuple["SpendLogsPayload", ...] = () - output_item_ids: frozenset[str] = frozenset() - - -def _with_conversation_turn(turns: _SessionTurns, spend_log: "SpendLogsPayload") -> _SessionTurns: - if not _is_a_conversation_turn(spend_log): - return turns - output_item_ids: Final = _output_item_ids(spend_log) - if _is_a_response_read(spend_log) and output_item_ids <= turns.output_item_ids: - return turns - return _SessionTurns( - spend_logs=(*turns.spend_logs, spend_log), output_item_ids=turns.output_item_ids | output_item_ids +def _first_index_holding_each_item(indexed_turns: Iterable[tuple[int, "SpendLogsPayload"]]) -> Mapping[str, int]: + return dict( + chain.from_iterable( + ((item_id, index) for item_id in _output_item_ids(turn)) for index, turn in reversed(tuple(indexed_turns)) + ) ) +def _turn_replaying_each_item(turns: Sequence["SpendLogsPayload"]) -> Mapping[str, int]: + indexed_turns: Final = tuple(enumerate(turns)) + first_read_holding: Final = _first_index_holding_each_item( + (index, turn) for index, turn in indexed_turns if _is_a_response_read(turn) + ) + first_other_turn_holding: Final = _first_index_holding_each_item( + (index, turn) for index, turn in indexed_turns if not _is_a_response_read(turn) + ) + return MappingProxyType({**first_read_holding, **first_other_turn_holding}) + + +def _replayed_by(item: object, index: int, turn_replaying: Mapping[str, int]) -> bool: + item_id: Final = _output_item_id(item) + return item_id is not None and turn_replaying.get(item_id) == index + + +def _with_only_new_output( + index: int, turn: "SpendLogsPayload", turn_replaying: Mapping[str, int] +) -> "SpendLogsPayload | None": + if not _is_a_response_read(turn): + return turn + stored_response: Final = as_str_mapping(_stored_response(turn)) + new_output: Final = [ + item + for item in _stored_responses_api_output(stored_response) or [] + if _replayed_by(item, index, turn_replaying) + ] + if stored_response is None or not new_output: + return None + replayed_turn: Final[SpendLogsPayload] = {**turn, "response": {**stored_response, "output": new_output}} + return replayed_turn + + def _conversation_turns(spend_logs: Sequence["SpendLogsPayload"]) -> tuple["SpendLogsPayload", ...]: - return reduce(_with_conversation_turn, spend_logs, _SessionTurns()).spend_logs + turns: Final = tuple(filter(_is_a_conversation_turn, spend_logs)) + turn_replaying: Final = _turn_replaying_each_item(turns) + replayed: Final = (_with_only_new_output(index, turn, turn_replaying) for index, turn in enumerate(turns)) + return tuple(turn for turn in replayed if turn is not None) _MCP_AUTO_EXECUTE_OUTPUT_TYPES: Final = frozenset(("mcp_tools_fetched", "tool_execution_results")) diff --git a/tests/integration/compatibility/test_openai_sdk_client_paths.py b/tests/integration/compatibility/test_openai_sdk_client_paths.py index f8d7449038a..d6b6d1d460d 100644 --- a/tests/integration/compatibility/test_openai_sdk_client_paths.py +++ b/tests/integration/compatibility/test_openai_sdk_client_paths.py @@ -630,19 +630,6 @@ def _spend_row(identity: str) -> Mapping[str, JsonValue]: return rows[0] -def _spend_row_keyed_by_any(identities: frozenset[str]) -> Mapping[str, JsonValue]: - keys: Final = tuple(sorted(identities)) - placeholders: Final = ", ".join("%s" for _ in keys) - rows: Final = eventually( - lambda: read_rows( - f'SELECT request_id, call_type, model FROM "LiteLLM_SpendLogs" WHERE request_id IN ({placeholders})', keys - ), - lambda rows: len(rows) == 1, - seconds=70, - ) - return rows[0] - - def _spend_request_ids(identities: tuple[str, str, str]) -> frozenset[str]: rows: Final = eventually( lambda: read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id IN (%s, %s, %s)', identities), @@ -815,7 +802,7 @@ def test_r07_responses_stream_through_the_async_openai_sdk(gateway: Gateway) -> served: Final = _served(wire) assert _routes(served) == (("POST", "/v1/responses"),), served assert _body(served[0])["stream"] is True, served[0].body - assert _spend_row_keyed_by_any(response_identities(last.response.id))["model"] == DEPLOYMENT_MODEL + assert _spend_row(last.response.id)["model"] == DEPLOYMENT_MODEL def test_r08_completions_non_stream_through_the_sync_openai_sdk(gateway: Gateway) -> None: diff --git a/tests/integration/observability/test_langfuse_otel_metadata_chaos.py b/tests/integration/observability/test_langfuse_otel_metadata_chaos.py index bb386f6d41e..15bc076b8ba 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata_chaos.py +++ b/tests/integration/observability/test_langfuse_otel_metadata_chaos.py @@ -107,9 +107,11 @@ def test_langfuse_otel_sink_outage_mid_burst_lands_every_response_id_once(gatewa ) payloads: Final = tuple(json.loads(line) for line in events) found: Final = next( - payload.get("response_id") or (payload.get("message") or {}).get("id") or payload.get("id") + (payload.get("response") or {}).get("id") + or (payload.get("message") or {}).get("id") + or payload.get("id") for payload in payloads - if payload.get("response_id") or payload.get("message") or payload.get("id") + if payload.get("response") or payload.get("message") or payload.get("id") ) return str(found) response: Final = rig.candidate.client.post(path, json=body, headers={"Authorization": f"Bearer {key}"}) diff --git a/tests/integration/observability/test_langfuse_otel_metadata_clients.py b/tests/integration/observability/test_langfuse_otel_metadata_clients.py index c2e0573b48a..fd2b9758449 100644 --- a/tests/integration/observability/test_langfuse_otel_metadata_clients.py +++ b/tests/integration/observability/test_langfuse_otel_metadata_clients.py @@ -162,8 +162,13 @@ def test_langfuse_otel_metadata_reaches_langfuse_for_responses_sse_stream(gatewa for line in stream.iter_lines() if line.startswith("data: ") and line != "data: [DONE]" ) - response_id: Final = next( - event["response_id"] for event in events if event.get("type") == "response.output_text.delta" + advertised: Final = frozenset( + event["response"]["id"] + for event in events + if event.get("type") in ("response.created", "response.completed") ) + assert len(advertised) == 1, events + response_id: Final = next(iter(advertised)) attrs: Final = exported_llm_span(rig.collector, response_id) + assert attrs["gen_ai.response.id"] == response_id, attrs assert_identity(attrs, chat_identity_fields(identity)) diff --git a/tests/integration/observability/test_otel_conversation_id.py b/tests/integration/observability/test_otel_conversation_id.py index d7f4de2961b..cd095991e53 100644 --- a/tests/integration/observability/test_otel_conversation_id.py +++ b/tests/integration/observability/test_otel_conversation_id.py @@ -1,5 +1,4 @@ import asyncio -import base64 import json import re import signal @@ -118,17 +117,6 @@ def _responses_reply(identity: str, stream: bool) -> Reply: ) -def _decoded_responses_id(identity: str) -> str: - try: - return base64.b64decode(identity.removeprefix("resp_").encode()).decode() - except (ValueError, UnicodeDecodeError): - return identity - - -def _canonical_id(identity: str) -> str: - return _decoded_responses_id(identity).rpartition("response_id:")[2] - - def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]: return tuple( json.loads(line[6:]) for line in text.splitlines() if line.startswith("data: ") and line != "data: [DONE]" @@ -170,8 +158,7 @@ class Collector: return tuple( attributes for attributes in self.attributes() - if isinstance(logged := attributes.get("gen_ai.response.id", {}).get("stringValue"), str) - and _canonical_id(logged) == _canonical_id(response_id) + if attributes.get("gen_ai.response.id", {}).get("stringValue") == response_id ) def conversation_ids(self, response_id: str) -> tuple[str | None, ...]: @@ -183,10 +170,6 @@ class Collector: def single_span(self, response_id: str) -> str | None: return eventually(lambda: self.conversation_ids(response_id), lambda values: len(values) == 1, seconds=30)[0] - def logged_id(self, response_id: str) -> str: - spans: Final = eventually(lambda: self.spans(response_id), lambda values: len(values) == 1, seconds=30) - return str(spans[0]["gen_ai.response.id"]["stringValue"]) - @pytest.fixture(scope="session") def collector() -> Iterator[Collector]: @@ -437,12 +420,13 @@ def test_responses_stream_raw_metadata_session_id_lands_as_conversation_id(rig: completed: Final = tuple(event for event in events if event["type"] == "response.completed") assert len(completed) == 1, events assert completed[0]["response"]["output"][0]["id"] == f"msg_resp_{marker}", completed - assert str(completed[0]["response"]["id"]).startswith("resp_"), completed + client_id: Final = str(completed[0]["response"]["id"]) + assert client_id.startswith("resp_"), completed assert rig.upstream_bodies(marker) == ( {"model": "gpt-4o-mini", "input": marker, "metadata": {"session_id": session}, "stream": True}, ) - assert rig.sink.single_span(f"resp_{marker}") == session - assert rig.spend_session(rig.sink.logged_id(f"resp_{marker}")) == session + assert rig.sink.single_span(client_id) == session + assert rig.spend_session(client_id) == session @pytest.mark.covers("other.observability.otel.conversation_id_from_metadata_chat_raw") @@ -689,15 +673,11 @@ def _burst(rig: Rig, count: int, session_for: Mapping[int, str]) -> tuple[tuple[ return tuple(pool.map(one, range(count))) -def _is_encrypted_responses_id(identity: str) -> bool: - return identity.startswith("resp_") and _decoded_responses_id(identity) == identity - - def _landed(rig: Rig, expected: Mapping[str, str]) -> dict[str, tuple[str, ...]]: spans: Final = rig.sink.attributes() return { session: tuple( - _canonical_id(str(attributes["gen_ai.response.id"]["stringValue"])) + str(attributes["gen_ai.response.id"]["stringValue"]) for attributes in spans if attributes.get(CONVERSATION, {}).get("stringValue") == session and "gen_ai.response.id" in attributes ) @@ -707,21 +687,13 @@ def _landed(rig: Rig, expected: Mapping[str, str]) -> dict[str, tuple[str, ...]] def _assert_exactly_once(rig: Rig, expected: Mapping[str, str], landed: Mapping[str, tuple[str, ...]]) -> None: spend: Final = eventually( - lambda: { - session: tuple(_canonical_id(identity) for identity in rig.spend_request_ids(session)) - for session in expected.values() - }, + lambda: {session: rig.spend_request_ids(session) for session in expected.values()}, lambda rows: all(len(values) >= 1 for values in rows.values()), seconds=70, ) assert landed == spend, (landed, spend) assert all(len(values) == 1 for values in landed.values()), landed - caller_visible: Final = { - session: (_canonical_id(identity),) - for identity, session in expected.items() - if not _is_encrypted_responses_id(identity) - } - assert {session: landed[session] for session in caller_visible} == caller_visible, landed + assert landed == {session: (identity,) for identity, session in expected.items()}, landed @pytest.mark.covers("other.observability.otel.conversation_id_sink_outage_recovers_exactly_once") diff --git a/tests/integration/observability/test_signoz_delivery.py b/tests/integration/observability/test_signoz_delivery.py index e25126e4fa8..93539d7641b 100644 --- a/tests/integration/observability/test_signoz_delivery.py +++ b/tests/integration/observability/test_signoz_delivery.py @@ -1,5 +1,4 @@ import asyncio -import base64 import json import re import signal @@ -9,7 +8,6 @@ from collections import deque from collections.abc import Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from functools import partial from pathlib import Path from typing import Final @@ -137,17 +135,6 @@ def _upstream(request: Request) -> Reply: return _chat_reply(f"chatcmpl-{marker}", stream) -def _decoded_responses_id(identity: str) -> str: - try: - return base64.b64decode(identity.removeprefix("resp_").encode()).decode() - except (ValueError, UnicodeDecodeError): - return identity - - -def _canonical_id(identity: str) -> str: - return _decoded_responses_id(identity).rpartition("response_id:")[2] - - def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]: return tuple( object_value(JSON.validate_json(line[6:])) @@ -232,8 +219,7 @@ class Collector: return tuple( span for span in self.spans() - if RESPONSE_ID in span.attributes - and _canonical_id(span.attributes[RESPONSE_ID]) == _canonical_id(response_id) + if RESPONSE_ID in span.attributes and span.attributes[RESPONSE_ID] == response_id ) def single_span(self, response_id: str, *, elsewhere: "Collector | None" = None) -> Span: @@ -249,11 +235,8 @@ class Collector: def landed(self, response_ids: Sequence[str]) -> dict[str, int]: spans: Final = self.spans() return { - _canonical_id(identity): sum( - 1 - for span in spans - if RESPONSE_ID in span.attributes - and _canonical_id(span.attributes[RESPONSE_ID]) == _canonical_id(identity) + identity: sum( + 1 for span in spans if RESPONSE_ID in span.attributes and span.attributes[RESPONSE_ID] == identity ) for identity in response_ids } @@ -448,7 +431,7 @@ def _assert_operator_span(rig: Rig, response_id: str, marker: str) -> Span: span: Final = rig.sink.single_span(response_id) assert span.target == "/v1/traces", span assert span.ingestion_key == OPERATOR_KEY, span - assert rig.tenant_sink.landed((response_id,)) == {_canonical_id(response_id): 0} + assert rig.tenant_sink.landed((response_id,)) == {response_id: 0} bodies: Final = rig.upstream_bodies(marker) assert len(bodies) == 1, bodies assert "signoz" not in json.dumps(bodies[0]).replace(marker, ""), bodies[0] @@ -522,7 +505,7 @@ def test_responses_stream_raw_httpx_span_lands_once_after_the_stream_is_consumed marker: Final = _marker() response: Final = rig.proxy.request("POST", "/v1/responses", {"model": rig.model, "input": marker, "stream": True}) assert response.status_code == 200, response.text - _assert_operator_span(rig, _responses_id(response, marker), marker) + _assert_operator_span(rig, _responses_id(response), marker) def test_v2_flag_on_still_delivers_the_operator_span_with_the_ingestion_key(v2_rig: Rig) -> None: @@ -821,7 +804,7 @@ def _chat_id(response: httpx.Response) -> str: return next(iter(identities)) -def _responses_id(response: httpx.Response, marker: str) -> str: +def _responses_id(response: httpx.Response) -> str: if not _is_event_stream(response): return _body_id(response) completed: Final = tuple( @@ -830,7 +813,7 @@ def _responses_id(response: httpx.Response, marker: str) -> str: if event.get("type") == "response.completed" ) assert len(completed) == 1 and completed[0].startswith("resp_"), response.text - return f"resp_{marker}" + return completed[0] def _message_id(response: httpx.Response) -> str: @@ -852,7 +835,7 @@ def _burst(rig: Rig, count: int) -> tuple[tuple[int, str, str | None], ...]: stream: Final = index % 2 == 0 path, body, identity_of = ( ("/v1/chat/completions", {"model": rig.model, "messages": [{"role": "user", "content": marker}]}, _chat_id), - ("/v1/responses", {"model": rig.model, "input": marker}, partial(_responses_id, marker=marker)), + ("/v1/responses", {"model": rig.model, "input": marker}, _responses_id), ( "/v1/messages", {"model": rig.model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}]}, @@ -874,8 +857,8 @@ def _assert_exactly_once(rig: Rig, identities: Sequence[str]) -> None: landed: Final = eventually( lambda: rig.sink.landed(identities), lambda seen: all(count >= 1 for count in seen.values()), seconds=80 ) - assert landed == {_canonical_id(identity): 1 for identity in identities}, landed - charged: Final = frozenset(identity for identity in identities if not identity.startswith("resp_")) + assert landed == {identity: 1 for identity in identities}, landed + charged: Final = frozenset(identities) spend: Final = eventually( lambda: read_rows( 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s::text[])', diff --git a/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py b/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py index 89ad226761e..5aede070896 100644 --- a/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py @@ -516,7 +516,7 @@ def test_responses_max_output_tokens_is_forwarded_natively_on_both_legs(gateway: for attempt in attempts: assert attempt.body["max_output_tokens"] == 1000 and _limits(attempt.body) == (None, None), attempt.body _assert_logged(issued, model, "success") - _assert_logged(_managed_response_id(streamed_issued), model, "success") + _assert_logged(streamed_issued, model, "success") def test_region_prefixed_deployment_on_a_v1_base_sends_the_bare_model_with_max_completion_tokens( diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py index 3cadc1b5ccf..42950fb18c3 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py @@ -172,7 +172,11 @@ def _upstream_id_inside(row_id: str) -> str | None: def _row_belongs_to(row_id: str, served: _Served) -> bool: if row_id == _response_id(served): return True - return served.call.endpoint == "responses" and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}" + return ( + served.call.endpoint == "responses" + and not served.call.stream + and _upstream_id_inside(row_id) == f"resp_upstream_{served.call.marker}" + ) def _assert_each_success_landed_once(rows: list[dict[str, JsonValue]], served: tuple[_Served, ...]) -> None: diff --git a/tests/unit/integrations/azure_storage/test_azure_storage.py b/tests/unit/integrations/azure_storage/test_azure_storage.py index 0f0ce24ff84..93490171e2c 100644 --- a/tests/unit/integrations/azure_storage/test_azure_storage.py +++ b/tests/unit/integrations/azure_storage/test_azure_storage.py @@ -5,19 +5,28 @@ import re import sys import threading from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest -from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS +from litellm.constants import _DEFAULT_TTL_FOR_HTTPX_CLIENTS, INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.azure_storage import azure_storage as azure_storage_module from litellm.integrations.azure_storage.azure_storage import ( AzureBlobStorageLogger, _cached_credential_chain_token_provider, adls_safe_file_name, ) +from litellm.types.passthrough_endpoints.pass_through_endpoints import ( + SETTLES_BACKGROUND_RESPONSE_KWARG, + BackgroundResponseSettlement, +) from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType -from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload +from litellm.types.utils import ( + BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + StandardAuditLogPayload, + StandardLoggingPayload, +) @pytest.fixture @@ -648,3 +657,160 @@ async def test_audit_log_is_uploaded_to_a_dated_audit_logs_path_with_an_account_ assert json.loads(uploaded) == audit_log assert file_client.append_data.await_args.kwargs == {"data": uploaded, "offset": 0, "length": len(uploaded)} file_client.flush_data.assert_awaited_once_with(position=len(uploaded), offset=0) + + +BACKGROUND_RESPONSE_ID: Final = "resp_YWJjZA==" +BACKGROUND_COST_POLL_KWARGS: Final = { + "call_type": "aget_responses", + "litellm_params": { + "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}, + }, +} +PASSTHROUGH_SETTLEMENT_KWARGS: Final = { + "call_type": "pass_through_endpoint", + "litellm_params": {"metadata": {}}, + SETTLES_BACKGROUND_RESPONSE_KWARG: BackgroundResponseSettlement.SETTLES, +} + + +def _background_create_cost_and_ordinary_events( + cost_kwargs: dict[str, object], +) -> tuple[tuple[dict[str, object], StandardLoggingPayload], ...]: + create: Final = StandardLoggingPayload( + id=BACKGROUND_RESPONSE_ID, + call_type="aresponses", + messages=[{"role": "user", "content": "background create input"}], + ) + cost: Final = StandardLoggingPayload(id=BACKGROUND_RESPONSE_ID, call_type="aget_responses", response_cost=0.42) + ordinary: Final = StandardLoggingPayload(id="chatcmpl-ordinary", call_type="acompletion") + return ( + ({"call_type": "aresponses", "litellm_params": {"metadata": {}}, "standard_logging_object": create}, create), + ({**cost_kwargs, "standard_logging_object": cost}, cost), + ( + {"call_type": "acompletion", "litellm_params": {"metadata": {}}, "standard_logging_object": ordinary}, + ordinary, + ), + ) + + +def _expected_files( + events: tuple[tuple[dict[str, object], StandardLoggingPayload], ...], +) -> dict[str, StandardLoggingPayload]: + create, cost, ordinary = (payload for _, payload in events) + return { + "resp_YWJjZA.json": create, + "resp_YWJjZA_background_cost.json": cost, + "chatcmpl-ordinary.json": ordinary, + } + + +async def _log_and_flush( + logger: AzureBlobStorageLogger, events: tuple[tuple[dict[str, object], StandardLoggingPayload], ...] +) -> None: + for kwargs, payload in events: + await logger.async_log_success_event(kwargs, {"id": payload["id"]}, None, None) + await logger.async_send_batch() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cost_kwargs", + ( + pytest.param(BACKGROUND_COST_POLL_KWARGS, id="cost-poll"), + pytest.param(PASSTHROUGH_SETTLEMENT_KWARGS, id="passthrough-settlement"), + ), +) +async def test_background_cost_event_keeps_its_own_file_beside_the_create_on_the_entra_path( + mock_env_vars, monkeypatch, cost_kwargs +): + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + events: Final = _background_create_cost_and_ordinary_events(cost_kwargs) + with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger() + logger.azure_auth_token = "mock-azure-ad-token" + logger.token_expiry = None + await _log_and_flush(logger, events) + + base_url: Final = "https://test-account.dfs.core.windows.net/test-container/" + appends: Final = [call for call in mock_http_client.patch.call_args_list if "action=append" in call.args[0]] + uploaded: Final = { + call.args[0].removeprefix(base_url).split("?")[0]: json.loads(call.kwargs["data"]) for call in appends + } + assert uploaded == _expected_files(events), ( + "the create, its background cost event and an ordinary event must each keep their own file" + ) + + +@pytest.mark.asyncio +async def test_failure_event_is_uploaded_to_a_file_named_after_its_id(mock_env_vars, monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + failure: Final = StandardLoggingPayload(id="chatcmpl-failed", call_type="acompletion", error_str="provider failure") + with patch( # test-quality-ok: REST client is created inside the method; assert emitted requests + "litellm.integrations.azure_storage.azure_storage.get_async_httpx_client" + ) as mock_get_client: + mock_http_client = AsyncMock() + mock_http_client.put.return_value = MagicMock() + mock_http_client.patch.return_value = MagicMock() + mock_get_client.return_value = mock_http_client + + logger = AzureBlobStorageLogger() + logger.azure_auth_token = "mock-azure-ad-token" + logger.token_expiry = None + await logger.async_log_failure_event( + {"call_type": "acompletion", "litellm_params": {"metadata": {}}, "standard_logging_object": failure}, + None, + None, + None, + ) + await logger.async_send_batch() + + base_url: Final = "https://test-account.dfs.core.windows.net/test-container/" + appends: Final = [call for call in mock_http_client.patch.call_args_list if "action=append" in call.args[0]] + assert {call.args[0].removeprefix(base_url).split("?")[0]: json.loads(call.kwargs["data"]) for call in appends} == { + "chatcmpl-failed.json": failure + } + + +@pytest.mark.asyncio +async def test_background_cost_event_keeps_its_own_file_beside_the_create_with_an_account_key( + workload_identity_env_vars, monkeypatch +): + monkeypatch.setenv("AZURE_STORAGE_ACCOUNT_KEY", "dGVzdC1rZXk=") + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + events: Final = _background_create_cost_and_ordinary_events(BACKGROUND_COST_POLL_KWARGS) + + uploaded: Final[dict[str, StandardLoggingPayload]] = {} + + def file_client_for(file_path: str) -> MagicMock: + file_client = MagicMock() + file_client.create_file = AsyncMock() + file_client.flush_data = AsyncMock() + + async def append_data(data: bytes, offset: int, length: int) -> None: + uploaded[file_path] = json.loads(data) + + file_client.append_data = append_data + return file_client + + file_system_client = MagicMock() + file_system_client.get_file_client = MagicMock(side_effect=file_client_for) + service_client = MagicMock() + service_client.get_file_system_client = MagicMock(return_value=file_system_client) + fake_aio_module = MagicMock() + fake_aio_module.DataLakeServiceClient = MagicMock(return_value=service_client) + + with patch.dict(sys.modules, {"azure.storage.filedatalake.aio": fake_aio_module}): + logger = AzureBlobStorageLogger() + await _log_and_flush(logger, events) + + today: Final = datetime.now().strftime("%Y-%m-%d") + assert uploaded == {f"{today}/{name}": payload for name, payload in _expected_files(events).items()}, ( + "the create, its background cost event and an ordinary event must each keep their own file" + ) diff --git a/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py b/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py index 6f143a244f6..a12ffd61b3a 100644 --- a/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py +++ b/tests/unit/integrations/gcs_bucket/test_gcs_bucket.py @@ -13,9 +13,11 @@ import litellm import pytest import respx +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN mock_response_data: Final = { @@ -172,9 +174,7 @@ async def test_get_payload_not_found(monkeypatch): assert payload is None -def _gcs_logger_with_upload_status( - monkeypatch: pytest.MonkeyPatch, status_code: int -) -> tuple[GCSBucketLogger, Mock]: +def _gcs_logger_with_upload_status(monkeypatch: pytest.MonkeyPatch, status_code: int) -> tuple[GCSBucketLogger, Mock]: monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) monkeypatch.setattr(google.auth, "default", _google_default_credentials) monkeypatch.setenv("GCS_FLUSH_INTERVAL", "3600") @@ -394,3 +394,95 @@ async def test_batch_line_items_are_serialized_off_the_event_loop(monkeypatch: p (thread,) = threads assert thread is not threading.main_thread() + + +@pytest.mark.asyncio +async def test_unbatched_background_cost_event_keeps_its_own_object_readable_by_its_spend_log_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("GCS_USE_BATCHED_LOGGING", "false") + logger, _ = _gcs_logger_with_upload_status(monkeypatch, 200) + object_prefix: Final = "/storage/v1/b/test-bucket/o/" + stored: Final[dict[str, object]] = {} + + def bucket(request: httpx.Request) -> httpx.Response: + if request.method == "POST": + stored[request.url.params["name"]] = json.loads(request.content) + return httpx.Response(200, json={"name": request.url.params["name"]}) + name: Final = request.url.path.removeprefix(object_prefix) + if name not in stored: + return httpx.Response(404, text="No such object") + return httpx.Response(200, content=json.dumps(stored[name]).encode("utf-8")) + + logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(bucket)) + response_id: Final = "resp_YmFja2dyb3VuZA" + create: Final = { + "id": response_id, + "messages": [{"role": "user", "content": "background create input"}], + "error_str": None, + } + cost: Final = {"id": response_id, "response_cost": 0.42, "error_str": None} + ordinary: Final = {"id": "chatcmpl-ordinary", "error_str": None} + events: Final = ( + ({"call_type": "aresponses", "litellm_params": {"metadata": {}}}, create), + ( + { + "call_type": "aget_responses", + "litellm_params": { + "metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN} + }, + }, + cost, + ), + ({"call_type": "acompletion", "litellm_params": {"metadata": {}}}, ordinary), + ) + + for kwargs, payload in events: + await logger.async_log_success_event( + {**kwargs, "standard_logging_object": payload}, {"id": payload["id"]}, None, None + ) + await logger.flush_queue() + + assert {name.split("/", maxsplit=1)[1]: payload for name, payload in stored.items()} == { + response_id: create, + f"{response_id}_background_cost": cost, + "chatcmpl-ordinary": ordinary, + }, "the create, its background cost event and an ordinary event must each keep their own object" + now: Final = datetime.now(timezone.utc) + assert await logger.get_request_response_payload(f"{response_id}_background_cost", now, None) == cost + assert await logger.get_request_response_payload(response_id, now, None) == create + + +@pytest.mark.asyncio +async def test_unbatched_gcs_log_id_still_names_a_background_cost_event_object(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GCS_USE_BATCHED_LOGGING", "false") + logger, _ = _gcs_logger_with_upload_status(monkeypatch, 200) + stored: Final[dict[str, object]] = {} + + def bucket(request: httpx.Request) -> httpx.Response: + stored[request.url.params["name"]] = json.loads(request.content) + return httpx.Response(200, json={"name": request.url.params["name"]}) + + logger.async_httpx_client = AsyncHTTPHandler(transport=httpx.MockTransport(bucket)) + cost: Final = {"id": "resp_YmFja2dyb3VuZA", "response_cost": 0.42, "error_str": None} + await logger.async_log_success_event( + { + "call_type": "aget_responses", + "litellm_params": { + "metadata": { + INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, + "gcs_log_id": "qa-run-7", + } + }, + "standard_logging_object": cost, + }, + {"id": cost["id"]}, + None, + None, + ) + await logger.flush_queue() + + ((name, payload),) = stored.items() + object_name: Final = name.split("/", maxsplit=1)[1] + assert object_name.startswith("custom-") and object_name.endswith("-qa-run-7"), object_name + assert payload == cost diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 1c97574d7f5..b843ab84051 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1133,6 +1133,35 @@ async def test_a_polled_background_response_replays_its_finished_output_once(): assert _replayed_roles_and_texts(session) == [("user", "Count to three"), ("assistant", "One, two, three")] +@pytest.mark.asyncio +async def test_a_background_response_that_grows_between_polls_replays_each_output_item_once(): + first: Final = _identified_output_message("msg_first", "One") + second: Final = _identified_output_message("msg_second", "Two") + spend_logs: Final = ( + _stored_response_turn("resp_queued", "aresponses", "queued", [], prompt="Count to two"), + _stored_response_turn("resp_poll_partial", "aget_responses", "in_progress", [first]), + _stored_response_turn("resp_poll_done", "aget_responses", "completed", [first, second]), + _stored_response_turn("resp_poll_again", "aget_responses", "completed", [first, second]), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_queued") + + assert _replayed_roles_and_texts(session) == [("user", "Count to two"), ("assistant", "One"), ("assistant", "Two")] + + +@pytest.mark.asyncio +async def test_a_read_that_ends_before_its_streamed_create_replays_the_answer_once_after_the_prompt(): + answer: Final = [_identified_output_message("msg_answer", "OK")] + spend_logs: Final = ( + _stored_response_turn("resp_read_mid_stream", "aget_responses", "completed", answer), + _stored_response_turn("resp_create", "aresponses", "completed", answer, prompt="Say OK"), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_create") + + assert _replayed_roles_and_texts(session) == [("user", "Say OK"), ("assistant", "OK")] + + def _tool_call_ids(message: object) -> tuple[str, ...]: tool_calls: Final = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) return tuple(call["id"] if isinstance(call, dict) else call.id for call in tool_calls or ()) diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index a44fb0d9427..9aede2e1170 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -5,18 +5,22 @@ Tests the security hook that prevents user B from seeing response from user A. """ import datetime +from collections.abc import Callable, Mapping from unittest.mock import MagicMock, patch import pytest from fastapi import HTTPException +from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.responses_id_security import ( ResponsesIDSecurity, _streams_response_ids, ) from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( GenericEvent, ResponseAPIUsage, @@ -25,7 +29,8 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import Choices, Message, ModelResponse, SpecialEnums, Usage +from litellm.types.responses.main import DeleteResponseResult +from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse, SpecialEnums, Usage @pytest.fixture @@ -60,12 +65,12 @@ class TestIsEncryptedResponseId: import litellm.proxy.hooks.responses_id_security as responses_module with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt: - mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456" - - result = responses_id_security._is_encrypted_response_id( - "resp_encrypted_value" + mock_decrypt.return_value = ( + f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_123;user_id:user-456" ) + result = responses_id_security._is_encrypted_response_id("resp_encrypted_value") + assert result is True mock_decrypt.assert_called_once() @@ -93,9 +98,7 @@ class TestDecryptResponseId: with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt: mock_decrypt.return_value = f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}response_id:resp_original_123;user_id:user-456;team_id:team-789" - original_id, user_id, team_id = responses_id_security._decrypt_response_id( - "resp_encrypted_value" - ) + original_id, user_id, team_id = responses_id_security._decrypt_response_id("resp_encrypted_value") assert original_id == "resp_original_123" assert user_id == "user-456" @@ -109,9 +112,7 @@ class TestDecryptResponseId: with patch.object(responses_module, "decrypt_value_helper") as mock_decrypt: mock_decrypt.return_value = None - original_id, user_id, team_id = responses_id_security._decrypt_response_id( - "resp_plain_value" - ) + original_id, user_id, team_id = responses_id_security._decrypt_response_id("resp_plain_value") assert original_id == "resp_plain_value" assert user_id is None @@ -121,9 +122,7 @@ class TestDecryptResponseId: class TestCheckUserAccessToResponseId: """Test check_user_access_to_response_id function""" - def test_check_user_access_same_user( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_same_user(self, responses_id_security, mock_user_api_key_dict): """Test that same user has access to their response ID""" result = responses_id_security.check_user_access_to_response_id( response_id_user_id="test-user-123", @@ -133,9 +132,7 @@ class TestCheckUserAccessToResponseId: assert result is True - def test_check_user_access_different_user_raises_exception( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_different_user_raises_exception(self, responses_id_security, mock_user_api_key_dict): """Test that different user is denied access to response ID""" with patch("litellm.proxy.proxy_server.general_settings", {}): with pytest.raises(HTTPException) as exc_info: @@ -148,9 +145,7 @@ class TestCheckUserAccessToResponseId: assert exc_info.value.status_code == 403 assert "Forbidden" in exc_info.value.detail - def test_check_user_access_different_team_raises_exception( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_different_team_raises_exception(self, responses_id_security, mock_user_api_key_dict): """Test that different team is denied access to response ID""" with patch("litellm.proxy.proxy_server.general_settings", {}): with pytest.raises(HTTPException) as exc_info: @@ -163,9 +158,7 @@ class TestCheckUserAccessToResponseId: assert exc_info.value.status_code == 403 assert "Forbidden" in exc_info.value.detail - def test_check_user_access_team_a_to_team_b_without_user_id( - self, responses_id_security - ): + def test_check_user_access_team_a_to_team_b_without_user_id(self, responses_id_security): """Test that key from team A (without user_id) cannot access response from team B (without user_id)""" # Create a mock user from team A without user_id mock_auth_team_a = MagicMock() @@ -184,9 +177,7 @@ class TestCheckUserAccessToResponseId: assert exc_info.value.status_code == 403 assert "team" in exc_info.value.detail.lower() - def test_check_user_access_team_a_to_team_b_with_user_id( - self, responses_id_security - ): + def test_check_user_access_team_a_to_team_b_with_user_id(self, responses_id_security): """Test that key from team A (without user_id) cannot access response from team B (with user_id)""" # Create a mock user from team A without user_id mock_auth_team_a = MagicMock() @@ -222,9 +213,7 @@ class TestCheckUserAccessToResponseId: assert result is True - def test_check_user_access_admin_can_access_any_response( - self, responses_id_security - ): + def test_check_user_access_admin_can_access_any_response(self, responses_id_security): """Test that proxy admin can access any response ID""" from litellm.proxy._types import LitellmUserRoles @@ -243,9 +232,7 @@ class TestCheckUserAccessToResponseId: assert result is True - def test_check_user_access_security_disabled( - self, responses_id_security, mock_user_api_key_dict - ): + def test_check_user_access_security_disabled(self, responses_id_security, mock_user_api_key_dict): """Test that when security is disabled, any user can access any response""" with patch( "litellm.proxy.proxy_server.general_settings", @@ -271,9 +258,7 @@ class TestAsyncPreCallHook: """Test pre-call hook decrypts previous_response_id for aresponses call""" data = {"previous_response_id": "resp_encrypted_value"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -289,15 +274,11 @@ class TestAsyncPreCallHook: assert result["previous_response_id"] == "resp_original_123" @pytest.mark.asyncio - async def test_async_pre_call_hook_aget_responses( - self, responses_id_security, mock_user_api_key_dict, mock_cache - ): + async def test_async_pre_call_hook_aget_responses(self, responses_id_security, mock_user_api_key_dict, mock_cache): """Test pre-call hook decrypts response_id for aget_responses call""" data = {"response_id": "resp_encrypted_456"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -313,9 +294,7 @@ class TestAsyncPreCallHook: assert result["response_id"] == "resp_original_456" @pytest.mark.asyncio - async def test_async_pre_call_hook_team_a_accessing_team_b_response( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_team_a_accessing_team_b_response(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A from accessing team B response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -325,9 +304,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -346,9 +323,7 @@ class TestAsyncPreCallHook: assert "team" in exc_info.value.detail.lower() @pytest.mark.asyncio - async def test_async_pre_call_hook_team_a_accessing_team_b_with_user( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_team_a_accessing_team_b_with_user(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A (no user) from accessing team B response (with user)""" # Create a mock user from team A without user_id mock_auth_team_a = MagicMock() @@ -358,9 +333,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b_with_user"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -380,9 +353,7 @@ class TestAsyncPreCallHook: assert "forbidden" in exc_info.value.detail.lower() @pytest.mark.asyncio - async def test_async_pre_call_hook_same_team_access( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_same_team_access(self, responses_id_security, mock_cache): """Test pre-call hook allows team A to access their own team's response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -392,9 +363,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_a"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -410,9 +379,7 @@ class TestAsyncPreCallHook: assert result["response_id"] == "resp_original_team_a" @pytest.mark.asyncio - async def test_async_pre_call_hook_adelete_responses_team_security( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_adelete_responses_team_security(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A from deleting team B's response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -422,9 +389,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -443,9 +408,7 @@ class TestAsyncPreCallHook: assert "team" in exc_info.value.detail.lower() @pytest.mark.asyncio - async def test_async_pre_call_hook_acancel_responses_team_security( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_acancel_responses_team_security(self, responses_id_security, mock_cache): """Test pre-call hook prevents team A from canceling team B's response""" # Create a mock user from team A mock_auth_team_a = MagicMock() @@ -455,9 +418,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -475,16 +436,13 @@ class TestAsyncPreCallHook: assert exc_info.value.status_code == 403 assert "team" in exc_info.value.detail.lower() - @pytest.mark.asyncio async def test_async_pre_call_hook_alist_input_items_decrypts_response_id( self, responses_id_security, mock_user_api_key_dict, mock_cache ): data = {"response_id": "resp_encrypted_789"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -501,9 +459,7 @@ class TestAsyncPreCallHook: assert result["response_id"] == "resp_original_789" @pytest.mark.asyncio - async def test_async_pre_call_hook_alist_input_items_team_security( - self, responses_id_security, mock_cache - ): + async def test_async_pre_call_hook_alist_input_items_team_security(self, responses_id_security, mock_cache): mock_auth_team_a = MagicMock() mock_auth_team_a.user_id = None mock_auth_team_a.team_id = "team-a" @@ -511,9 +467,7 @@ class TestAsyncPreCallHook: data = {"response_id": "resp_encrypted_team_b"} - with patch.object( - responses_id_security, "_is_encrypted_response_id", return_value=True - ): + with patch.object(responses_id_security, "_is_encrypted_response_id", return_value=True): with patch.object( responses_id_security, "_decrypt_response_id", @@ -625,9 +579,7 @@ class TestAsyncPostCallStreamingIteratorHook: "/openai/v1/responses/resp_rawprovider123", ], ) - async def test_streamed_id_encrypted_on_all_responses_routes( - self, responses_id_security, route, monkeypatch - ): + async def test_streamed_id_encrypted_on_all_responses_routes(self, responses_id_security, route, monkeypatch): streamed_id = await self._drain_streamed_id(responses_id_security, route, monkeypatch) assert streamed_id != "resp_rawprovider123" @@ -639,12 +591,8 @@ class TestAsyncPostCallStreamingIteratorHook: ) @pytest.mark.asyncio - async def test_streamed_id_untouched_on_non_responses_route( - self, responses_id_security, monkeypatch - ): - streamed_id = await self._drain_streamed_id( - responses_id_security, "/chat/completions", monkeypatch - ) + async def test_streamed_id_untouched_on_non_responses_route(self, responses_id_security, monkeypatch): + streamed_id = await self._drain_streamed_id(responses_id_security, "/chat/completions", monkeypatch) assert streamed_id == "resp_rawprovider123" assert not responses_id_security._is_encrypted_response_id(streamed_id) @@ -703,12 +651,8 @@ class TestStreamedGenericEventIdEncryption: def _advertised_ids(events): nested = (getattr(event, "response", None) for event in events) return [ - payload["id"] if isinstance(payload, dict) else payload.id - for payload in nested - if payload is not None - ] + [ - event.id for event in events if isinstance(getattr(event, "id", None), str) - ] + payload["id"] if isinstance(payload, dict) else payload.id for payload in nested if payload is not None + ] + [event.id for event in events if isinstance(getattr(event, "id", None), str)] async def _drain(self, responses_id_security, monkeypatch): monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij") @@ -728,9 +672,7 @@ class TestStreamedGenericEventIdEncryption: ] @pytest.mark.asyncio - async def test_every_event_advertises_the_same_encrypted_id( - self, responses_id_security, monkeypatch - ): + async def test_every_event_advertises_the_same_encrypted_id(self, responses_id_security, monkeypatch): events = await self._drain(responses_id_security, monkeypatch) advertised = self._advertised_ids(events) @@ -747,22 +689,14 @@ class TestStreamedGenericEventIdEncryption: ) @pytest.mark.asyncio - async def test_raw_provider_id_never_reaches_the_client( - self, responses_id_security, monkeypatch - ): + async def test_raw_provider_id_never_reaches_the_client(self, responses_id_security, monkeypatch): events = await self._drain(responses_id_security, monkeypatch) - assert [self.RAW_ID in event.model_dump_json() for event in events] == [ - False - ] * len(events) + assert [self.RAW_ID in event.model_dump_json() for event in events] == [False] * len(events) @pytest.mark.asyncio - async def test_sibling_fields_survive_the_rewrite( - self, responses_id_security, monkeypatch - ): - _, queued, keepalive, later, _ = await self._drain( - responses_id_security, monkeypatch - ) + async def test_sibling_fields_survive_the_rewrite(self, responses_id_security, monkeypatch): + _, queued, keepalive, later, _ = await self._drain(responses_id_security, monkeypatch) assert queued.response["status"] == "queued" assert later.response["status"] == "in_progress" @@ -774,27 +708,19 @@ class TestAsyncPostCallSuccessHook: """Test async_post_call_success_hook function""" @pytest.mark.asyncio - async def test_async_post_call_success_hook_encrypts_response( - self, responses_id_security, mock_user_api_key_dict - ): + async def test_async_post_call_success_hook_encrypts_response(self, responses_id_security, mock_user_api_key_dict): """Test post-call hook encrypts ResponsesAPIResponse""" - mock_response = ResponsesAPIResponse( - id="resp_789", created_at=1234567890, output=[], status="completed" - ) + mock_response = ResponsesAPIResponse(id="resp_789", created_at=1234567890, output=[], status="completed") data = {} - with patch.object( - responses_id_security, "_encrypt_response_id", return_value=mock_response - ) as mock_encrypt: + with patch.object(responses_id_security, "_encrypt_response_id", return_value=mock_response) as mock_encrypt: result = await responses_id_security.async_post_call_success_hook( data=data, user_api_key_dict=mock_user_api_key_dict, response=mock_response, ) - mock_encrypt.assert_called_once_with( - mock_response, mock_user_api_key_dict, request_cache=None - ) + mock_encrypt.assert_called_once_with(mock_response, mock_user_api_key_dict, request_cache=None) assert result == mock_response @pytest.mark.asyncio @@ -851,9 +777,7 @@ def _auth(user_id="owner-user", team_id="owner-team", user_role=None): def _issue_managed_id(hook, owner, provider_response_id=_FABRICATED_PROVIDER_RESPONSE_ID): """Mint an id exactly the way the proxy hands one to a client on create.""" issued = hook._encrypt_response_id( - ResponsesAPIResponse( - id=provider_response_id, created_at=1234567890, output=[], status="completed" - ), + ResponsesAPIResponse(id=provider_response_id, created_at=1234567890, output=[], status="completed"), owner, ) return issued.id @@ -1085,9 +1009,7 @@ class TestClientSuppliedRetainedIdCannotBypassAuthorization: assert exc_info.value.status_code == 403 @pytest.mark.asyncio - async def test_stranger_forging_their_own_id_never_reaches_someone_elses_response( - self, mock_cache, salt_key_env - ): + async def test_stranger_forging_their_own_id_never_reaches_someone_elses_response(self, mock_cache, salt_key_env): hook = _hook() stranger = _auth(user_id="stranger-user", team_id="stranger-team") stranger_id = _issue_managed_id(hook, stranger, provider_response_id="resp_strangerownprovideridcccccccc") @@ -1111,7 +1033,7 @@ def _admin(): return _auth(user_id="admin-user", team_id="admin-team", user_role=LitellmUserRoles.PROXY_ADMIN) -def _provider_answer(response_id): +def _provider_answer(response_id: str) -> ResponsesAPIResponse: return ResponsesAPIResponse(id=response_id, created_at=1234567890, output=[], status="completed") @@ -1244,13 +1166,22 @@ class _SpendRowRecorder(CustomLogger): super().__init__() self.spend_row_ids = [] self.standard_logging_ids = [] + self.callback_response_ids = [] async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): self.spend_row_ids.append(get_logging_payload(kwargs, response_obj, start_time, end_time)["request_id"]) - self.standard_logging_ids.append(kwargs["standard_logging_object"]["id"]) + self.callback_response_ids.append(response_obj.id) + if "standard_logging_object" in kwargs: + self.standard_logging_ids.append(kwargs["standard_logging_object"]["id"]) -async def _stream_logged_mid_stream(hook, route, logged_result, caller=None, request_data=None): +async def _stream_logged_mid_stream( + hook: ResponsesIDSecurity, + route: str, + logged_result: ResponseCompletedEvent | ModelResponse, + caller: UserAPIKeyAuth | None = None, + request_data: Mapping[str, object] | None = None, +) -> tuple[str, _SpendRowRecorder]: streaming_caller = caller if caller is not None else _auth() recorder = _SpendRowRecorder() logging_obj = LiteLLMLoggingObj( @@ -1369,7 +1300,14 @@ class TestStreamedReadAnswersWithTheIdTheClientHolds: ) -async def _read_logged_by_callbacks(hook, caller, issued_id, call_type, cache): +async def _read_logged_by_callbacks( + hook: ResponsesIDSecurity, + caller: UserAPIKeyAuth, + issued_id: str, + call_type: CallTypesLiteral, + cache: DualCache, + answer: Callable[[str], ResponsesAPIResponse | DeleteResponseResult] = _provider_answer, +) -> tuple[str, _SpendRowRecorder]: recorder = _SpendRowRecorder() now = datetime.datetime.now() logging_obj = LiteLLMLoggingObj( @@ -1388,7 +1326,7 @@ async def _read_logged_by_callbacks(hook, caller, issued_id, call_type, cache): data = await hook.async_pre_call_hook( user_api_key_dict=caller, cache=cache, data={"response_id": issued_id}, call_type=call_type ) - provider_answer = _provider_answer(data["response_id"]) + provider_answer = answer(data["response_id"]) client_response = await hook.async_post_call_success_hook( data=data, user_api_key_dict=caller, response=provider_answer @@ -1417,3 +1355,68 @@ class TestEveryReadLogsAnIdOfItsOwn: assert {hook._decrypt_response_id(logged_id) for logged_id in logged_ids} == { (_FABRICATED_PROVIDER_RESPONSE_ID, reader.user_id, reader.team_id) } + + +_RAW_UPSTREAM_RESPONSE_ID = "resp_06de59f3upstreamiddddddddddddddddd" + + +def _routed_response_id(upstream_response_id: str) -> str: + return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + _provider_answer(upstream_response_id), "openai", {"model_info": {"id": "deployment-1"}} + ).id + + +def _deletion_of(response_id: str) -> DeleteResponseResult: + return DeleteResponseResult( + id=ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response_id), + object="response.deleted", + deleted=True, + ) + + +class TestDeleteAnswersWithTheIdTheClientHolds: + @pytest.mark.asyncio + @pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"]) + @pytest.mark.parametrize( + "stored_id", + [_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID), _RAW_UPSTREAM_RESPONSE_ID], + ids=["routed-deployment", "plain-id"], + ) + async def test_a_caller_with_access_gets_back_the_exact_id_they_sent( + self, mock_cache, salt_key_env, reader, stored_id + ): + hook = _hook() + issued_id = _issue_managed_id(hook, _auth(), provider_response_id=stored_id) + data = await hook.async_pre_call_hook( + user_api_key_dict=reader, cache=mock_cache, data={"response_id": issued_id}, call_type="adelete_responses" + ) + + result = await hook.async_post_call_success_hook( + data=data, user_api_key_dict=reader, response=_deletion_of(data["response_id"]) + ) + + assert (result.id, result.object, result.deleted) == (issued_id, "response.deleted", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize("reader", [_auth(), _admin()], ids=["owner", "admin"]) + async def test_callbacks_and_the_spend_row_log_an_id_of_its_own_never_the_one_the_client_holds( + self, mock_cache, salt_key_env, reader + ): + hook = _hook() + issued_id = _issue_managed_id( + hook, _auth(), provider_response_id=_routed_response_id(_RAW_UPSTREAM_RESPONSE_ID) + ) + + client_id, recorder = await _read_logged_by_callbacks( + hook, reader, issued_id, "adelete_responses", mock_cache, answer=_deletion_of + ) + + assert client_id == issued_id + assert recorder.spend_row_ids == recorder.callback_response_ids + assert len(recorder.callback_response_ids) == 1 + assert recorder.callback_response_ids[0] != issued_id + assert hook._decrypt_response_id(recorder.callback_response_ids[0]) == ( + _RAW_UPSTREAM_RESPONSE_ID, + reader.user_id, + reader.team_id, + )