From d295f76c4ad7204de6ab388a7da2e74300ca9ef1 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 5 Nov 2026 16:09:49 -0800 Subject: [PATCH] fix(types): define the responses compact call types for during_call guardrails and deployment hooks (#50274) * fix(types): define the responses compact call types so during_call guardrails and deployment hooks see them * test: regenerate dashboard api types and skip compact in the azure sdk client sweep --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/types/utils.py | 4 + ...sponses_lifecycle_during_call_guardrail.py | 114 ++++++++++++++++++ .../llms/azure/test_azure_common_utils.py | 1 + .../test_unified_guardrail.py | 18 +++ tests/unit/test_utils.py | 23 ++++ tests/unit/types/test_utils.py | 12 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 7 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ba29a67a36c..ca434154f68 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -703,6 +703,8 @@ class CallTypes(str, Enum): adelete_responses = "adelete_responses" cancel_responses = "cancel_responses" acancel_responses = "acancel_responses" + compact_responses = "compact_responses" + acompact_responses = "acompact_responses" alist_input_items = "alist_input_items" llm_passthrough_route = "llm_passthrough_route" allm_passthrough_route = "allm_passthrough_route" @@ -806,6 +808,8 @@ CallTypesLiteral = Literal[ "send_message", "aresponses", "responses", + "compact_responses", + "acompact_responses", "acreate_skill", "acreate_realtime_client_secret", "arealtime_calls", diff --git a/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py b/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py new file mode 100644 index 00000000000..3227caaa7f5 --- /dev/null +++ b/tests/integration/authorization/test_responses_lifecycle_during_call_guardrail.py @@ -0,0 +1,114 @@ +"""GET, compact, DELETE and cancel on /v1/responses answer 200 when a during_call guardrail is on by default +(LIT-9770): the hook used to raise `'acompact_responses' is not a valid CallTypes` and the proxy returned 500""" + +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, string_value +from integration._support.process import owned_proxy +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MODEL: Final = "openai/gpt-5.4-mini" + + +def _input() -> list[JsonValue]: + return [ + {"role": "user", "content": f"Remember the number {uuid.uuid4().hex}."}, + {"role": "assistant", "content": "Got it."}, + ] + + +def _response(response_id: str) -> Reply: + body: Final[dict[str, JsonValue]] = { + "id": response_id, + "object": "response", + "created_at": 1, + "status": "completed", + "model": _MODEL.removeprefix("openai/"), + "output": [ + { + "type": "message", + "id": f"msg_{uuid.uuid4().hex}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Got it.", "annotations": []}], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 12, "output_tokens": 5, "total_tokens": 17}, + } + return Reply(body=json.dumps(body).encode()) + + +def _deleted(response_id: str) -> Reply: + return Reply(body=json.dumps({"id": response_id, "object": "response.deleted", "deleted": True}).encode()) + + +def _guarded_config(path: Path) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + guardrail: Final[dict[str, JsonValue]] = { + "guardrail_name": f"content-filter-{uuid.uuid4().hex}", + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "during_call", + "default_on": True, + "blocked_words": [{"keyword": "forbiddenword", "action": "BLOCK"}], + }, + } + path.write_text(yaml.safe_dump({**base, "guardrails": [guardrail]})) + return path + + +@pytest.fixture(scope="module") +def guarded(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("during-call-guardrail") + with owned_proxy(session_gateway, directory, {}, config=_guarded_config(directory / "config.yaml")) as candidate: + yield candidate + + +def _create(guarded: Gateway, provider: SharedProvider, upstream_id: str) -> str: + provider.expect(_response(upstream_id)) + created: Final = guarded.request("POST", "/v1/responses", {"model": _MODEL, "input": _input(), "store": True}) + assert created.status_code == 200, created.text + (upstream,) = provider.received() + assert upstream.target.endswith("/responses"), upstream.target + return string_value(_JSON_OBJECT.validate_json(created.content)["id"]) + + +def _lifecycle(guarded: Gateway, provider: SharedProvider, operation: str) -> tuple[httpx.Response, str]: + if operation == "compact": + provider.expect(_response(f"resp_{uuid.uuid4().hex}")) + return guarded.request( + "POST", "/v1/responses/compact", {"model": _MODEL, "input": _input()} + ), "/responses/compact" + upstream_id: Final = f"resp_{uuid.uuid4().hex}" + managed_id: Final = _create(guarded, provider, upstream_id) + provider.expect(_deleted(upstream_id) if operation == "delete" else _response(upstream_id)) + if operation == "get": + return guarded.request("GET", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}" + if operation == "delete": + return guarded.request("DELETE", f"/v1/responses/{managed_id}"), f"/responses/{upstream_id}" + return guarded.request("POST", f"/v1/responses/{managed_id}/cancel"), f"/responses/{upstream_id}/cancel" + + +@pytest.mark.parametrize("operation", ["get", "compact", "delete", "cancel"]) +def test_responses_lifecycle_answers_200_with_a_during_call_guardrail_on( + guarded: Gateway, provider: SharedProvider, operation: str +) -> None: + response, upstream_target = _lifecycle(guarded, provider, operation) + assert response.status_code == 200, response.text + (upstream,) = provider.received() + assert upstream.target.endswith(upstream_target), upstream.target diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index d66ba3f324e..65f8400e53c 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -477,6 +477,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "aget_responses", "adelete_responses", "acancel_responses", + "acompact_responses", "aresponses_websocket", "alist_input_items", "acreate_fine_tuning_job", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 84f4d7a4173..de3ced0c583 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -3291,3 +3291,21 @@ async def test_pass_through_guardrail_text_is_the_client_body_without_the_inject texts: Final = [call["inputs"]["texts"] for call in guardrail.apply_calls] assert [[json.loads(text) for text in call_texts] for call_texts in texts] == [[body]], texts assert data["litellm_metadata"]["user_api_key_user_email"] == "someone@example.com", data["litellm_metadata"] + + +@pytest.mark.asyncio +async def test_moderation_hook_lets_responses_compact_through_without_a_translation() -> None: + handler: Final = UnifiedLLMGuardrails() + guardrail: Final = RecordingGuardrail() + data: Final = { + "guardrail_to_apply": guardrail, + "model": "gpt-5.4-mini", + "input": [{"role": "user", "content": "Remember the number 42."}], + } + + result: Final = await handler.async_moderation_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), call_type="acompact_responses" + ) + + assert result is data + assert guardrail.apply_calls == [] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 924c3738638..c41c4db2c02 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -104,6 +104,7 @@ from litellm.utils import ( _snapshot_exception_for_hook, async_post_call_failure_deployment_hook, async_post_call_success_deployment_hook, + async_pre_call_deployment_hook, calculate_max_parallel_requests, client, get_non_default_completion_params, @@ -11626,3 +11627,25 @@ async def test_success_enqueue_is_tracked_without_starting_callbacks_synchronous assert GLOBAL_LOGGING_WORKER.pending_count > 0, "Tracked enqueue must finish before worker draining starts" await GLOBAL_LOGGING_WORKER.flush() assert logging_obj.model_call_details["has_logged_async_success"] is True + + +class _RecordingPreCallDeploymentHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_call_types: tuple[CallTypes | None, ...] = () + + async def async_pre_call_deployment_hook( + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict[str, object] | None: + self.seen_call_types = (*self.seen_call_types, call_type) + return None + + +@pytest.mark.asyncio +async def test_pre_call_deployment_hook_hands_responses_compact_its_call_type(monkeypatch: pytest.MonkeyPatch) -> None: + hook: Final = _RecordingPreCallDeploymentHook() + monkeypatch.setattr(litellm, "callbacks", [hook]) + + await async_pre_call_deployment_hook({"model": "gpt-5.4-mini"}, "acompact_responses") + + assert hook.seen_call_types == (CallTypes.acompact_responses,) diff --git a/tests/unit/types/test_utils.py b/tests/unit/types/test_utils.py index 32479c62372..bdbcbb698d4 100644 --- a/tests/unit/types/test_utils.py +++ b/tests/unit/types/test_utils.py @@ -1,7 +1,12 @@ import pytest from litellm.litellm_core_utils.initialize_dynamic_callback_params import validate_excluded_services_value -from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content +from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VALUES, + CallTypes, + CaptureMessageContent, + captures_span_content, +) @pytest.mark.parametrize("value", ["", " ", "auth", "none,redis", "redis,", ",postgres"]) @@ -45,3 +50,8 @@ def test_capture_message_content_constructor_accepts_any_string() -> None: ) def test_captures_span_content(mode: str | None, captures: bool) -> None: assert captures_span_content(mode) is captures + + +@pytest.mark.parametrize("call_type", ["compact_responses", "acompact_responses"]) +def test_responses_compact_call_types_round_trip_their_public_names(call_type: str) -> None: + assert CallTypes(call_type).value == call_type diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d14e490f606..a9af3f2b719 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29442,7 +29442,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "compact_responses" | "acompact_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */