From 3e94f7d71e386550648146e056c04b23f50c2b97 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:47:00 -0700 Subject: [PATCH 01/21] fix(interactions): map step and turn input to Responses API roles and content types --- .../transformation.py | 159 ++++++++---------- .../test_litellm_responses_bridge.py | 72 ++++++++ 2 files changed, 146 insertions(+), 85 deletions(-) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index 2a71c3e8977..4209f1538e0 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -2,10 +2,11 @@ Transformation utilities for bridging Interactions API to Responses API. This module handles transforming between: -- Interactions API format (Google's format with Turn[], system_instruction, etc.) +- Interactions API format (Google's format with Step[]/Turn[], system_instruction, etc.) - Responses API format (OpenAI's format with input[], instructions, etc.) """ +from types import MappingProxyType from typing import Any, Final, cast from litellm.types.interactions import ( @@ -19,6 +20,8 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ) +_STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"}) + class LiteLLMResponsesInteractionsConfig: """Configuration class for transforming between Interactions API and Responses API.""" @@ -91,112 +94,98 @@ class LiteLLMResponsesInteractionsConfig: Interactions API input can be: - string: "Hello" - - Turn[]: [{"role": "user", "content": [...]}] - - Content object + - Step[]: [{"type": "user_input", "content": [...]}, {"type": "model_output", "content": [...]}] + - Turn[] (legacy): [{"role": "user", "content": [...]}] + - Content | Content[]: one user message worth of content parts Responses API input is: - string: "Hello" - - Message[]: [{"role": "user", "content": [...]}] + - Message[]: [{"role": "user", "content": [{"type": "input_text", ...}]}] """ if isinstance(input, str): - # ResponseInputParam accepts str return cast(ResponseInputParam, input) if isinstance(input, list): - # Turn[] format - convert to Responses API Message[] format - messages: Final = [] - for turn in input: - if isinstance(turn, dict): - role = turn.get("role", "user") - content = turn.get("content", []) - - # Transform content array - transformed_content = LiteLLMResponsesInteractionsConfig._transform_content_array(content) - - messages.append( - { - "role": role, - "content": transformed_content, - } - ) - elif isinstance(turn, Turn): - # Pydantic model - role = turn.role if hasattr(turn, "role") else "user" - content = turn.content if hasattr(turn, "content") else [] - - # Ensure content is a list for _transform_content_array - # Cast to List[Any] to handle various content types - if isinstance(content, list): - content_list: list[Any] = list(content) - elif content is not None: - content_list = [content] - else: - content_list = [] - - transformed_content = LiteLLMResponsesInteractionsConfig._transform_content_array(content_list) - - messages.append( - { - "role": role, - "content": transformed_content, - } - ) - - return cast(ResponseInputParam, messages) - - # Single content object - wrap in message - if isinstance(input, dict): + if any(LiteLLMResponsesInteractionsConfig._is_history_item(item) for item in input): + return cast( + ResponseInputParam, + [ + LiteLLMResponsesInteractionsConfig._transform_history_item(item) + for item in input + if LiteLLMResponsesInteractionsConfig._is_history_item(item) + ], + ) return cast( ResponseInputParam, [ { "role": "user", - "content": LiteLLMResponsesInteractionsConfig._transform_content_array( - input.get("content", []) if isinstance(input.get("content"), list) else [input] - ), + "content": LiteLLMResponsesInteractionsConfig._transform_content_array(list(input), "user"), + } + ], + ) + + if isinstance(input, dict): + raw_content: Final = input.get("content") + content_items: Final = raw_content if isinstance(raw_content, list) else [input] + return cast( + ResponseInputParam, + [ + { + "role": "user", + "content": LiteLLMResponsesInteractionsConfig._transform_content_array(content_items, "user"), } ], ) - # Fallback: convert to string return cast(ResponseInputParam, str(input)) @staticmethod - def _transform_content_array(content: list[Any]) -> list[dict[str, Any]]: - """Transform Interactions API content array to Responses API format.""" - if not isinstance(content, list): - # Single content item - wrap in array - content = [content] + def _is_history_item(item: Any) -> bool: + if isinstance(item, Turn): + return True + return isinstance(item, dict) and ("role" in item or item.get("type") in _STEP_TYPE_ROLES) - transformed: Final[list[dict[str, Any]]] = [] - for item in content: - if isinstance(item, dict): - # Already in dict format, pass through - transformed.append(item) - elif isinstance(item, str): - # Plain string - wrap in text format - transformed.append({"type": "text", "text": item}) - else: - # Pydantic model or other - convert to dict - if hasattr(item, "model_dump"): - dumped = item.model_dump() - if isinstance(dumped, dict): - transformed.append(dumped) - else: - # Fallback: wrap in text format - transformed.append({"type": "text", "text": str(dumped)}) - elif hasattr(item, "dict"): - dumped = item.dict() - if isinstance(dumped, dict): - transformed.append(dumped) - else: - # Fallback: wrap in text format - transformed.append({"type": "text", "text": str(dumped)}) - else: - # Fallback: wrap in text format - transformed.append({"type": "text", "text": str(item)}) + @staticmethod + def _transform_history_item(item: "Turn | dict[str, Any]") -> dict[str, Any]: + raw: Final = item.model_dump(exclude_none=True) if isinstance(item, Turn) else item + role: Final = LiteLLMResponsesInteractionsConfig._responses_role(raw) + raw_content: Final = raw.get("content") + content_items: Final = ( + raw_content if isinstance(raw_content, list) else [] if raw_content is None else [raw_content] + ) + return { + "role": role, + "content": LiteLLMResponsesInteractionsConfig._transform_content_array(content_items, role), + } - return transformed + @staticmethod + def _responses_role(item: dict[str, Any]) -> str: + step_role: Final = _STEP_TYPE_ROLES.get(str(item.get("type", ""))) + if step_role is not None: + return step_role + raw_role: Final = str(item.get("role") or "user") + return "assistant" if raw_role == "model" else raw_role + + @staticmethod + def _transform_content_array(content: list[Any], role: str) -> list[dict[str, Any]]: + """Transform Interactions API content parts to Responses API parts for the given role.""" + return [LiteLLMResponsesInteractionsConfig._transform_content_item(item, role) for item in content] + + @staticmethod + def _transform_content_item(item: Any, role: str) -> dict[str, Any]: + text_type: Final = "output_text" if role == "assistant" else "input_text" + if isinstance(item, str): + return {"type": text_type, "text": item} + if isinstance(item, dict): + if item.get("type") == "text": + return {"type": text_type, "text": str(item.get("text", ""))} + return item + if hasattr(item, "model_dump"): + dumped: Final = item.model_dump(exclude_none=True) + if isinstance(dumped, dict): + return LiteLLMResponsesInteractionsConfig._transform_content_item(dumped, role) + return {"type": text_type, "text": str(item)} @staticmethod def transform_responses_response_to_interactions_response( diff --git a/tests/test_litellm/interactions/test_litellm_responses_bridge.py b/tests/test_litellm/interactions/test_litellm_responses_bridge.py index 17e7f9fc4ff..8400f2c4840 100644 --- a/tests/test_litellm/interactions/test_litellm_responses_bridge.py +++ b/tests/test_litellm/interactions/test_litellm_responses_bridge.py @@ -7,6 +7,10 @@ the litellm_responses bridge provider, which calls litellm.responses() internall import os +from litellm.interactions.litellm_responses_transformation.transformation import ( + LiteLLMResponsesInteractionsConfig, +) +from litellm.types.interactions import Turn from tests.test_litellm.interactions.base_interactions_test import ( BaseInteractionsTest, ) @@ -26,3 +30,71 @@ class TestLiteLLMResponsesBridge(BaseInteractionsTest): def get_api_key(self) -> str: """Return the OpenAI API key from environment.""" return os.getenv("OPENAI_API_KEY", "") + + +class TestBridgeInputTransformation: + """Regression tests for translating Interactions input into Responses API input. + + The bridge used to pass Google content parts through raw ({"type": "text"}), + which the Responses API rejects with a 400, and it dropped the role encoded + in step types and in the legacy "model" turn role. + """ + + def test_step_input_maps_roles_and_content_types(self): + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [ + {"type": "user_input", "content": [{"type": "text", "text": "I like apples."}]}, + {"type": "model_output", "content": [{"type": "text", "text": "I like oranges."}]}, + {"type": "user_input", "content": [{"type": "text", "text": "What did you say?"}]}, + ] + ) + assert transformed == [ + {"role": "user", "content": [{"type": "input_text", "text": "I like apples."}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "I like oranges."}]}, + {"role": "user", "content": [{"type": "input_text", "text": "What did you say?"}]}, + ] + + def test_legacy_turn_input_maps_model_role_to_assistant(self): + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [ + {"role": "user", "content": [{"type": "text", "text": "I like apples."}]}, + {"role": "model", "content": [{"type": "text", "text": "I like oranges."}]}, + ] + ) + assert transformed == [ + {"role": "user", "content": [{"type": "input_text", "text": "I like apples."}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "I like oranges."}]}, + ] + + def test_turn_pydantic_model_with_string_content(self): + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [Turn(role="model", content="I like oranges.")] + ) + assert transformed == [ + {"role": "assistant", "content": [{"type": "output_text", "text": "I like oranges."}]} + ] + + def test_string_input_passes_through(self): + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input("Hello") + assert transformed == "Hello" + + def test_content_list_input_becomes_single_user_message(self): + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [{"type": "text", "text": "Hello"}, "world"] + ) + assert transformed == [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Hello"}, + {"type": "input_text", "text": "world"}, + ], + } + ] + + def test_non_text_content_passes_through_unchanged(self): + image_part = {"type": "image", "data": "base64data", "mime_type": "image/png"} + transformed = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + [{"type": "user_input", "content": [image_part]}] + ) + assert transformed == [{"role": "user", "content": [image_part]}] From 0aeed161259aaec87e6e7e2275191bce16c11d8d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:40:32 -0700 Subject: [PATCH 02/21] test(interactions): follow Google spec drift replacing Turn with typed steps --- .../interactions/test_openapi_compliance.py | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 1fe343ca6ee..2665f8703a6 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -167,17 +167,39 @@ class TestRequestCompliance: assert text_schema["properties"]["type"].get("const") == "text" print("✓ TextContent schema is correct") - def test_turn_schema(self, spec_dict): - """Verify Turn schema for multi-turn conversations.""" - turn_schema = spec_dict["components"]["schemas"]["Turn"] + def test_step_schema(self, spec_dict): + """Verify step-based multi-turn input. - assert "role" in turn_schema["properties"] - assert "content" in turn_schema["properties"] + Google replaced the role-carrying `Turn` schema with typed steps + (spec update of Aug 13, 2026): conversation history is now a `Step[]` + where `UserInputStep`/`ModelOutputStep` pin `type` values that our + transformations read to recover the role. Assert exactly what our code + depends on: `InteractionsInput` accepts a Step array, both step kinds + are part of the `Step` union, each pins its `type` const, and each + carries a `Content[]` content field. + """ + input_schema = spec_dict["components"]["schemas"]["InteractionsInput"] + step_array_items = [ + option["items"]["$ref"].split("/")[-1] + for option in input_schema["oneOf"] + if option.get("type") == "array" and "$ref" in option.get("items", {}) + ] + assert "Step" in step_array_items, f"InteractionsInput should accept Step[], got arrays of {step_array_items}" - # Content can be string or Content[] - content_prop = turn_schema["properties"]["content"] - assert "oneOf" in content_prop - print("✓ Turn schema supports role + content") + step_variants = { + option["$ref"].split("/")[-1] + for option in spec_dict["components"]["schemas"]["Step"]["oneOf"] + if "$ref" in option + } + assert {"UserInputStep", "ModelOutputStep"} <= step_variants, f"Step union is missing role steps: {step_variants}" + + for step_name, type_value in [("UserInputStep", "user_input"), ("ModelOutputStep", "model_output")]: + step_schema = spec_dict["components"]["schemas"][step_name] + assert step_schema["properties"]["type"].get("const") == type_value + assert "type" in step_schema["required"] + content_items = step_schema["properties"]["content"]["items"] + assert content_items["$ref"].split("/")[-1] == "Content" + print(f"✓ {step_name} pins type '{type_value}' with Content[] content") class TestResponseCompliance: From 432ea8644b838b44685dbe5929fc45a47205eda7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:47:30 -0700 Subject: [PATCH 03/21] test(interactions): send step and content-list input to the live Gemini API --- .../test_google_interactions_integration.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/test_litellm/interactions/test_google_interactions_integration.py index 9c651cc94f5..41f0fa0d7fb 100644 --- a/tests/test_litellm/interactions/test_google_interactions_integration.py +++ b/tests/test_litellm/interactions/test_google_interactions_integration.py @@ -55,17 +55,10 @@ class TestGoogleInteractionsCreate: print(f"Usage: {response.usage}") def test_create_with_content_list(self, api_key): - """Test creating an interaction with a structured content list (Turn format).""" + """Test creating an interaction with a structured content list (Content[] input).""" response = interactions.create( model="gemini/gemini-2.5-flash", - input=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is the capital of France?"} - ], - } - ], + input=[{"type": "text", "text": "What is the capital of France?"}], api_key=api_key, ) @@ -169,25 +162,25 @@ class TestGoogleInteractionsStreaming: class TestGoogleInteractionsMultiTurn: - """Tests for multi-turn conversations using Turn[] input.""" + """Tests for multi-turn conversations using Step[] input.""" def test_multi_turn_conversation(self, api_key): - """Test a multi-turn conversation per OpenAPI spec (Turn[] format).""" + """Test a multi-turn conversation per OpenAPI spec (Step[] format).""" response = interactions.create( model="gemini/gemini-2.5-flash", input=[ { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "My name is Alice."}], }, { - "role": "model", + "type": "model_output", "content": [ {"type": "text", "text": "Hello Alice! Nice to meet you."} ], }, { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "What is my name?"}], }, ], From 2a3b54394fcbbaa1992fb4d2b3085a0920aff150 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:03:33 -0700 Subject: [PATCH 04/21] refactor(interactions): drop Any and extra casts from bridge input helpers --- .../transformation.py | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py index 4209f1538e0..9657b444969 100644 --- a/litellm/interactions/litellm_responses_transformation/transformation.py +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -6,9 +6,12 @@ This module handles transforming between: - Responses API format (OpenAI's format with input[], instructions, etc.) """ +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Any, Final, cast +from pydantic import BaseModel + from litellm.types.interactions import ( InteractionInput, InteractionsAPIOptionalRequestParams, @@ -106,24 +109,21 @@ class LiteLLMResponsesInteractionsConfig: return cast(ResponseInputParam, input) if isinstance(input, list): - if any(LiteLLMResponsesInteractionsConfig._is_history_item(item) for item in input): - return cast( - ResponseInputParam, - [ - LiteLLMResponsesInteractionsConfig._transform_history_item(item) - for item in input - if LiteLLMResponsesInteractionsConfig._is_history_item(item) - ], - ) - return cast( - ResponseInputParam, + transformed: Final = ( [ + LiteLLMResponsesInteractionsConfig._transform_history_item(item) + for item in input + if LiteLLMResponsesInteractionsConfig._is_history_item(item) + ] + if any(LiteLLMResponsesInteractionsConfig._is_history_item(item) for item in input) + else [ { "role": "user", - "content": LiteLLMResponsesInteractionsConfig._transform_content_array(list(input), "user"), + "content": LiteLLMResponsesInteractionsConfig._transform_content_array(input, "user"), } - ], + ] ) + return cast(ResponseInputParam, transformed) if isinstance(input, dict): raw_content: Final = input.get("content") @@ -141,16 +141,17 @@ class LiteLLMResponsesInteractionsConfig: return cast(ResponseInputParam, str(input)) @staticmethod - def _is_history_item(item: Any) -> bool: + def _is_history_item(item: object) -> bool: if isinstance(item, Turn): return True return isinstance(item, dict) and ("role" in item or item.get("type") in _STEP_TYPE_ROLES) @staticmethod - def _transform_history_item(item: "Turn | dict[str, Any]") -> dict[str, Any]: + def _transform_history_item(item: object) -> Mapping[str, object]: raw: Final = item.model_dump(exclude_none=True) if isinstance(item, Turn) else item - role: Final = LiteLLMResponsesInteractionsConfig._responses_role(raw) - raw_content: Final = raw.get("content") + fields: Final = raw if isinstance(raw, Mapping) else {} + role: Final = LiteLLMResponsesInteractionsConfig._responses_role(fields) + raw_content: Final = fields.get("content") content_items: Final = ( raw_content if isinstance(raw_content, list) else [] if raw_content is None else [raw_content] ) @@ -160,7 +161,7 @@ class LiteLLMResponsesInteractionsConfig: } @staticmethod - def _responses_role(item: dict[str, Any]) -> str: + def _responses_role(item: Mapping[str, object]) -> str: step_role: Final = _STEP_TYPE_ROLES.get(str(item.get("type", ""))) if step_role is not None: return step_role @@ -168,23 +169,21 @@ class LiteLLMResponsesInteractionsConfig: return "assistant" if raw_role == "model" else raw_role @staticmethod - def _transform_content_array(content: list[Any], role: str) -> list[dict[str, Any]]: + def _transform_content_array(content: Sequence[object], role: str) -> Sequence[Mapping[str, object]]: """Transform Interactions API content parts to Responses API parts for the given role.""" return [LiteLLMResponsesInteractionsConfig._transform_content_item(item, role) for item in content] @staticmethod - def _transform_content_item(item: Any, role: str) -> dict[str, Any]: + def _transform_content_item(item: object, role: str) -> Mapping[str, object]: text_type: Final = "output_text" if role == "assistant" else "input_text" if isinstance(item, str): return {"type": text_type, "text": item} - if isinstance(item, dict): + if isinstance(item, Mapping): if item.get("type") == "text": return {"type": text_type, "text": str(item.get("text", ""))} return item - if hasattr(item, "model_dump"): - dumped: Final = item.model_dump(exclude_none=True) - if isinstance(dumped, dict): - return LiteLLMResponsesInteractionsConfig._transform_content_item(dumped, role) + if isinstance(item, BaseModel): + return LiteLLMResponsesInteractionsConfig._transform_content_item(item.model_dump(exclude_none=True), role) return {"type": text_type, "text": str(item)} @staticmethod From 73e555a3e3a761db51ca51d813b418516e1721aa Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:32:57 -0700 Subject: [PATCH 05/21] test(interactions): follow Google spec drift replacing Turn with typed steps (#36730) * test(interactions): follow Google spec drift replacing Turn with typed steps * test(interactions): send step and content-list input to the live Gemini API --- .../test_google_interactions_integration.py | 21 ++++------ .../interactions/test_openapi_compliance.py | 40 ++++++++++++++----- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/tests/test_litellm/interactions/test_google_interactions_integration.py b/tests/test_litellm/interactions/test_google_interactions_integration.py index 9c651cc94f5..41f0fa0d7fb 100644 --- a/tests/test_litellm/interactions/test_google_interactions_integration.py +++ b/tests/test_litellm/interactions/test_google_interactions_integration.py @@ -55,17 +55,10 @@ class TestGoogleInteractionsCreate: print(f"Usage: {response.usage}") def test_create_with_content_list(self, api_key): - """Test creating an interaction with a structured content list (Turn format).""" + """Test creating an interaction with a structured content list (Content[] input).""" response = interactions.create( model="gemini/gemini-2.5-flash", - input=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is the capital of France?"} - ], - } - ], + input=[{"type": "text", "text": "What is the capital of France?"}], api_key=api_key, ) @@ -169,25 +162,25 @@ class TestGoogleInteractionsStreaming: class TestGoogleInteractionsMultiTurn: - """Tests for multi-turn conversations using Turn[] input.""" + """Tests for multi-turn conversations using Step[] input.""" def test_multi_turn_conversation(self, api_key): - """Test a multi-turn conversation per OpenAPI spec (Turn[] format).""" + """Test a multi-turn conversation per OpenAPI spec (Step[] format).""" response = interactions.create( model="gemini/gemini-2.5-flash", input=[ { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "My name is Alice."}], }, { - "role": "model", + "type": "model_output", "content": [ {"type": "text", "text": "Hello Alice! Nice to meet you."} ], }, { - "role": "user", + "type": "user_input", "content": [{"type": "text", "text": "What is my name?"}], }, ], diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index 1fe343ca6ee..2665f8703a6 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -167,17 +167,39 @@ class TestRequestCompliance: assert text_schema["properties"]["type"].get("const") == "text" print("✓ TextContent schema is correct") - def test_turn_schema(self, spec_dict): - """Verify Turn schema for multi-turn conversations.""" - turn_schema = spec_dict["components"]["schemas"]["Turn"] + def test_step_schema(self, spec_dict): + """Verify step-based multi-turn input. - assert "role" in turn_schema["properties"] - assert "content" in turn_schema["properties"] + Google replaced the role-carrying `Turn` schema with typed steps + (spec update of Aug 13, 2026): conversation history is now a `Step[]` + where `UserInputStep`/`ModelOutputStep` pin `type` values that our + transformations read to recover the role. Assert exactly what our code + depends on: `InteractionsInput` accepts a Step array, both step kinds + are part of the `Step` union, each pins its `type` const, and each + carries a `Content[]` content field. + """ + input_schema = spec_dict["components"]["schemas"]["InteractionsInput"] + step_array_items = [ + option["items"]["$ref"].split("/")[-1] + for option in input_schema["oneOf"] + if option.get("type") == "array" and "$ref" in option.get("items", {}) + ] + assert "Step" in step_array_items, f"InteractionsInput should accept Step[], got arrays of {step_array_items}" - # Content can be string or Content[] - content_prop = turn_schema["properties"]["content"] - assert "oneOf" in content_prop - print("✓ Turn schema supports role + content") + step_variants = { + option["$ref"].split("/")[-1] + for option in spec_dict["components"]["schemas"]["Step"]["oneOf"] + if "$ref" in option + } + assert {"UserInputStep", "ModelOutputStep"} <= step_variants, f"Step union is missing role steps: {step_variants}" + + for step_name, type_value in [("UserInputStep", "user_input"), ("ModelOutputStep", "model_output")]: + step_schema = spec_dict["components"]["schemas"][step_name] + assert step_schema["properties"]["type"].get("const") == type_value + assert "type" in step_schema["required"] + content_items = step_schema["properties"]["content"]["items"] + assert content_items["$ref"].split("/")[-1] == "Content" + print(f"✓ {step_name} pins type '{type_value}' with Content[] content") class TestResponseCompliance: From e619106306656d703a822610ca2f1e9ed542be40 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 12 Aug 2026 23:41:28 -0700 Subject: [PATCH 06/21] refactor(ui): migrate team detail controls to shadcn (#36695) * test(ui): characterize shared migration surfaces * refactor(ui): migrate team detail controls --- ui/litellm-dashboard/eslint-suppressions.json | 6 - .../src/components/team/TeamMemberTab.tsx | 58 ++++---- .../team/TeamVirtualKeysTable.test.tsx | 5 +- .../components/team/TeamVirtualKeysTable.tsx | 136 +++++++++--------- 4 files changed, 95 insertions(+), 110 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 8a86305b4cb..c6f600d90ec 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -3525,17 +3525,11 @@ "src/components/team/TeamMemberTab.tsx": { "local/no-complex-jsx-arrow": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/components/team/TeamVirtualKeysTable.tsx": { "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 2 } }, "src/components/team/member_permissions.tsx": { diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx index 4e04063197c..ddc43ac50f1 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx @@ -1,13 +1,13 @@ import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Tooltip } from "@/components/atoms/Tooltip"; +import MemberTable from "@/components/common_components/MemberTable"; import { Member } from "@/components/networking"; import { DateCell, MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import { Space, Tooltip, Typography } from "antd"; -import type { ColumnsType } from "antd/es/table"; -import MemberTable from "@/components/common_components/MemberTable"; +import { CircleHelp } from "lucide-react"; +import type { ComponentProps } from "react"; import { TeamData } from "./TeamInfo"; interface TeamMemberTabProps { @@ -97,48 +97,48 @@ export default function TeamMemberTab({ return membership?.litellm_budget_table?.budget_reset_at ?? null; }; - const extraColumns: ColumnsType = [ + const extraColumns: NonNullable["extraColumns"]> = [ { title: ( - + Model Scope - - + + - + ), key: "model_scope", render: (_: unknown, record: Member) => { const models = getUserAllowedModels(record.user_id); if (!models) { - return (all team models); + return (all team models); } const displayed = models.slice(0, 2); const remaining = models.length - displayed.length; return ( - +
{displayed.map((m) => ( - + {m} - + ))} {remaining > 0 && ( - - +{remaining} more + + +{remaining} more )} - +
); }, }, { title: ( - + Current Cycle Spend (USD) - - + + - + ), key: "spend", render: (_: unknown, record: Member) => ( @@ -147,12 +147,12 @@ export default function TeamMemberTab({ }, { title: ( - + Total Spend (USD) - - + + - + ), key: "total_spend", render: (_: unknown, record: Member) => , @@ -171,15 +171,15 @@ export default function TeamMemberTab({ }, { title: ( - + Team Member Rate Limits - - + + - + ), key: "rate_limits", - render: (_: unknown, record: Member) => {getUserRateLimits(record.user_id)}, + render: (_: unknown, record: Member) => {getUserRateLimits(record.user_id)}, }, ]; diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx index 1c9b14d0c9d..f71b2a05f3f 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.test.tsx @@ -1,7 +1,6 @@ -import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi, MockedFunction } from "vitest"; -import { renderWithProviders } from "../../../tests/test-utils"; +import { renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; import { TeamVirtualKeysTable } from "./TeamVirtualKeysTable"; import { KeysResponse, useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; import { KeyResponse } from "../key_team_helpers/key_list"; @@ -264,7 +263,7 @@ describe("TeamVirtualKeysTable", () => { await user.click(await screen.findByTestId("datatable-filters-trigger")); const drawerBody = await screen.findByTestId("filter-drawer-body"); - const userInput = drawerBody.querySelector("input") as HTMLElement; + const userInput = within(drawerBody).getByPlaceholderText("Filter by user ID…"); await user.type(userInput, "user-42"); await user.click(screen.getByTestId("filter-drawer-apply")); diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index 66690e5478f..f4097b082f8 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -1,5 +1,7 @@ "use client"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { Tooltip } from "@/components/atoms/Tooltip"; +import CopyButton from "@/components/shared/CopyButton"; import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; import { DataTable, @@ -8,13 +10,13 @@ import { DataTableSortHeader, DataTableToolbar, } from "@/components/shared/DataTable"; +import { Badge } from "@/components/ui/badge"; +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; import { Input } from "@/components/ui/input"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; -import { ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; import { ColumnDef, ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table"; -import { Badge, Icon, Text } from "@tremor/react"; -import { Popover, Tooltip, Typography } from "antd"; +import { ChevronDown, ChevronRight } from "lucide-react"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; @@ -147,12 +149,9 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi enableSorting: true, cell: (info) => { const value = info.getValue() as string; - const width = info.cell.column.getSize(); return ( - - - {value ?? "-"} - + + {value ?? "-"} ); }, @@ -182,12 +181,9 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi cell: (info) => { const user = info.getValue() as { user_email?: string } | undefined; const value = user?.user_email; - const width = info.cell.column.getSize(); return ( - - - {value ?? "-"} - + + {value ?? "-"} ); }, @@ -201,12 +197,9 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi cell: (info) => { const userId = info.getValue() as string | null; const displayValue = userId === "default_user_id" ? "Default Proxy Admin" : userId; - const width = info.cell.column.getSize(); return ( - - - {displayValue ?? "-"} - + + {displayValue ?? "-"} ); }, @@ -234,21 +227,21 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi const userEmail = created_by_user?.user_email ?? null; const isDefaultAdmin = userId === "default_user_id"; const displayValue = userAlias || userEmail || userId; - const width = info.cell.column.getSize(); const popoverContent = ( -
+
{[ { label: "User Alias", value: userAlias }, { label: "User Email", value: userEmail }, { label: "User ID", value: userId }, ].map(({ label, value }) => (
- {label} + {label} {value ? ( - - {value} - + + {value} + + ) : ( - )} @@ -259,23 +252,24 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi if (isDefaultAdmin && !userAlias && !userEmail) { return ( - - + + }> - - + + {popoverContent} + ); } return ( - - + } > {displayValue} - - + + {popoverContent} + ); }, }, @@ -342,14 +336,14 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi const models = info.getValue() as string[]; const scope = deriveKeyModelScope(info.row.original.allowed_routes, info.row.original.key_type); const emptyModelsBadge = !scope.hasModelAccess ? ( - - - No model access + + + No model access ) : ( - - All Proxy Models + + All Proxy Models ); return ( @@ -362,57 +356,55 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi <>
{models.length > 3 && ( -
- - setExpandedAccordions((prev) => ({ - ...prev, - [info.row.id]: !prev[info.row.id], - })) - } - /> -
+ )}
{models.slice(0, 3).map((model, index) => model === "all-proxy-models" ? ( - - All Proxy Models + + All Proxy Models ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - + + {model.length > 30 + ? `${getModelDisplayName(model).slice(0, 30)}...` + : getModelDisplayName(model)} ), )} {models.length > 3 && !expandedAccordions[info.row.id] && ( - - - +{models.length - 3} {models.length - 3 === 1 ? "more model" : "more models"} - + + +{models.length - 3} {models.length - 3 === 1 ? "more model" : "more models"} )} {expandedAccordions[info.row.id] && (
{models.slice(3).map((model, index) => model === "all-proxy-models" ? ( - - All Proxy Models + + All Proxy Models ) : ( - - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - + + {model.length > 30 + ? `${getModelDisplayName(model).slice(0, 30)}...` + : getModelDisplayName(model)} ), )} From 4df421e058d94a28a887208c871c04e66089e5fd Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 12 Aug 2026 23:42:42 -0700 Subject: [PATCH 07/21] refactor(ui): migrate guardrail and duration controls to shadcn (#36693) * test(ui): characterize shared migration surfaces * refactor(ui): migrate guardrail and duration controls * fix(ui): preserve duration select callback shape * fix(ui): narrow duration selection value --- ui/litellm-dashboard/eslint-suppressions.json | 8 --- .../components/GuardrailSettingsView.test.tsx | 31 ++++++++++++ .../src/components/GuardrailSettingsView.tsx | 50 ++++++++----------- .../common_components/DurationSelect.test.tsx | 10 ++-- .../common_components/DurationSelect.tsx | 33 +++++++++--- 5 files changed, 87 insertions(+), 45 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/GuardrailSettingsView.test.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index c6f600d90ec..25778401efa 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2055,9 +2055,6 @@ "src/components/GuardrailSettingsView.tsx": { "no-nested-ternary": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/GuardrailsMonitor/LogViewer.tsx": { @@ -2649,11 +2646,6 @@ "count": 1 } }, - "src/components/common_components/DurationSelect.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/Filters/FilterInput.tsx": { "react-hooks/set-state-in-effect": { "count": 1 diff --git a/ui/litellm-dashboard/src/components/GuardrailSettingsView.test.tsx b/ui/litellm-dashboard/src/components/GuardrailSettingsView.test.tsx new file mode 100644 index 00000000000..1c206b9c3b7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailSettingsView.test.tsx @@ -0,0 +1,31 @@ +import { renderWithProviders, screen } from "../../tests/test-utils"; +import { describe, expect, it } from "vitest"; +import GuardrailSettingsView from "./GuardrailSettingsView"; + +describe("GuardrailSettingsView", () => { + it("should render", () => { + renderWithProviders(); + + expect(screen.getByText("Guardrails Settings")).toBeInTheDocument(); + }); + + it("should separate active global and team-specific guardrails", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("global-one")).toBeInTheDocument(); + expect(screen.getByText("team-one")).toBeInTheDocument(); + expect(screen.queryByText("global-two")).not.toBeInTheDocument(); + }); + + it("should show when global guardrails are bypassed", () => { + renderWithProviders(); + + expect(screen.getByText("Bypassed for this team")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx b/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx index 95957e845d9..5510eb62bdb 100644 --- a/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailSettingsView.tsx @@ -1,6 +1,8 @@ import React from "react"; -import { Tag } from "antd"; -import { GlobalOutlined } from "@ant-design/icons"; +import { Globe2 } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { cn } from "@/lib/cva.config"; interface GuardrailSettingsViewProps { globalGuardrailNames: Set; @@ -26,40 +28,36 @@ export function GuardrailSettingsView({ const isEmpty = !killSwitchOn && globalsRunning.length === 0 && nonGlobalOptIns.length === 0; const content = isEmpty ? ( - No guardrails configured + No guardrails configured ) : (
- - + + Global {killSwitchOn ? ( - Bypassed for this team + Bypassed for this team ) : globalsRunning.length > 0 ? (
{globalsRunning.map((name) => ( - - {name} - + {name} ))}
) : ( - None configured + None configured )}
- Team-specific + Team-specific {nonGlobalOptIns.length > 0 ? (
{nonGlobalOptIns.map((name) => ( - - {name} - + {name} ))}
) : ( - None configured + None configured )}
@@ -67,23 +65,19 @@ export function GuardrailSettingsView({ if (variant === "card") { return ( -
-
-
- Guardrails Settings - - Global and team-specific guardrails applied to this team - -
-
- {content} -
+ + + Guardrails Settings + Global and team-specific guardrails applied to this team + + {content} + ); } return ( -
- Guardrails Settings +
+ Guardrails Settings {content}
); diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx index 296ef1ae632..bce1093d211 100644 --- a/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DurationSelect.test.tsx @@ -1,5 +1,5 @@ import { render, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; +import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event"; import { describe, it, expect, vi } from "vitest"; import DurationSelect from "./DurationSelect"; @@ -19,6 +19,9 @@ describe("DurationSelect", () => { expect(screen.getByText("Daily")).toBeInTheDocument(); expect(screen.getByText("Weekly")).toBeInTheDocument(); expect(screen.getByText("Monthly")).toBeInTheDocument(); + const dailyLabel = screen.getByText("Daily"); + const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel; + await user.click(dailyOption); }); it("should apply className prop", () => { @@ -28,14 +31,15 @@ describe("DurationSelect", () => { }); it("should call onChange when an option is selected", async () => { - const user = userEvent.setup(); + const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); const onChange = vi.fn(); render(); const select = screen.getByRole("combobox"); await user.click(select); - const dailyOption = screen.getByText("Daily"); + const dailyLabel = screen.getByText("Daily"); + const dailyOption = dailyLabel.closest('[role="option"]') ?? dailyLabel; await user.click(dailyOption); expect(onChange).toHaveBeenCalledWith("24h", expect.any(Object)); diff --git a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx index a84e8aeb110..cd5f6f4ffdc 100644 --- a/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx +++ b/ui/litellm-dashboard/src/components/common_components/DurationSelect.tsx @@ -1,17 +1,38 @@ -import { Select } from "antd"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; interface DurationSelectProps { className?: string; value?: string; - onChange?: (value: string) => void; + onChange?: (value: string, option: { value: string; label: string }) => void; } +const DURATION_OPTIONS = [ + { value: "24h", label: "Daily" }, + { value: "7d", label: "Weekly" }, + { value: "30d", label: "Monthly" }, +]; + export default function DurationSelect({ className, value, onChange }: DurationSelectProps) { return ( - { + const selectedOption = DURATION_OPTIONS.find((option) => option.value === nextValue); + if (selectedOption) { + onChange?.(selectedOption.value, selectedOption); + } + }} + > + + + + + {DURATION_OPTIONS.map((option) => ( + + {option.label} + + ))} + ); } From fd00b98f64445d5049daa3a448c62734338d3c83 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 12 Aug 2026 23:43:12 -0700 Subject: [PATCH 08/21] refactor(ui): migrate guardrails-monitor, projects, logs to shadcn (#34606) * test(ui): pin behaviour of guardrails-monitor, projects and logs components before migration Adds role- and text-based characterisation tests for EvaluationSettingsModal, GuardrailDetail and AuditLogDrawer, which had none, and moves the remaining antd-specific assertions (.ant-spin, the icon role of an antd Spin indicator) onto library-neutral ARIA queries. Also covers the enterprise banner on the deleted keys and deleted teams pages, which no test reached. All of these pass against the current antd and Tremor components. * refactor(ui): migrate guardrails-monitor, projects and logs to shadcn Replaces antd and Tremor with installed shadcn primitives across the files these three routes exclusively own. Markup only, except where noted below. Deletes AntDLoadingSpinner, an antd-only primitive living in the shadcn ui/ folder, and moves its single call site onto ui/ui-loading-spinner. Two behaviour notes. The logs tab handler previously mapped every tab past the first to "audit logs", so the audit panel kept polling while Deleted Keys or Deleted Teams was on screen; each tab now reports its own value and panels stay mounted via keepMounted. The evaluation settings dialog is bounded to the viewport and scrolls internally, which the antd Modal got from being top-anchored on a scrolling page. The tests added in the previous commit pass unedited against these components. --- ui/litellm-dashboard/eslint-suppressions.json | 42 +-- .../EvaluationSettingsModal.test.tsx | 121 ++++++ .../_components/EvaluationSettingsModal.tsx | 154 ++++---- .../_components/GuardrailDetail.test.tsx | 145 ++++++++ .../_components/GuardrailDetail.tsx | 179 +++++---- .../_components/ProjectDetailsPage.test.tsx | 8 +- .../_components/ProjectDetailsPage.tsx | 352 ++++++++---------- .../_components/ProjectKeysSection.tsx | 63 ++-- .../projects/_components/ProjectsPage.tsx | 67 ++-- .../DeletedKeysPage/DeletedKeysPage.test.tsx | 9 + .../DeletedKeysPage/DeletedKeysPage.tsx | 17 +- .../DeletedTeamsPage.test.tsx | 9 + .../DeletedTeamsPage/DeletedTeamsPage.tsx | 17 +- .../components/ui/AntDLoadingSpinner.test.tsx | 56 --- .../src/components/ui/AntDLoadingSpinner.tsx | 12 - .../AuditLogDrawer/AuditLogDrawer.test.tsx | 134 +++++++ .../AuditLogDrawer/AuditLogDrawer.tsx | 174 ++++----- .../src/components/view_logs/index.test.tsx | 4 +- .../src/components/view_logs/index.tsx | 32 +- 19 files changed, 932 insertions(+), 663 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/ui/AntDLoadingSpinner.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/ui/AntDLoadingSpinner.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/AuditLogDrawer/AuditLogDrawer.test.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 25778401efa..18dd7c949ec 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -299,9 +299,6 @@ } }, "src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx": { - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -314,9 +311,6 @@ "src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx": { "no-nested-ternary": { "count": 3 - }, - "no-restricted-imports": { - "count": 1 } }, "src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx": { @@ -1447,16 +1441,10 @@ }, "src/app/(dashboard)/projects/_components/ProjectDetailsPage.tsx": { "no-nested-ternary": { - "count": 3 - }, - "no-restricted-imports": { - "count": 1 + "count": 2 } }, "src/app/(dashboard)/projects/_components/ProjectKeysSection.tsx": { - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1487,11 +1475,6 @@ "count": 2 } }, - "src/app/(dashboard)/projects/_components/ProjectsPage.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/prompts/_components/add_prompt_form.tsx": { "local/filename-pascal-case": { "count": 1 @@ -1994,16 +1977,6 @@ "count": 1 } }, - "src/components/DeletedKeysPage/DeletedKeysPage.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, - "src/components/DeletedTeamsPage/DeletedTeamsPage.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/DeprecationBanner.tsx": { "no-restricted-imports": { "count": 1 @@ -3586,11 +3559,6 @@ "count": 1 } }, - "src/components/ui/AntDLoadingSpinner.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/ui/alert-dialog.tsx": { "local/filename-pascal-case": { "count": 1 @@ -3799,11 +3767,6 @@ "count": 1 } }, - "src/components/view_logs/AuditLogDrawer/AuditLogDrawer.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/CostBreakdownViewer.tsx": { "no-restricted-imports": { "count": 1 @@ -3961,9 +3924,6 @@ "src/components/view_logs/index.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "no-restricted-imports": { - "count": 1 } }, "src/components/view_logs/log_filter_logic.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.test.tsx new file mode 100644 index 00000000000..41aa1087782 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.test.tsx @@ -0,0 +1,121 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { render, screen, waitFor } from "@testing-library/react"; +import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; + +const mockFetchAvailableModels = vi.fn(); +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: (...args: unknown[]) => mockFetchAvailableModels(...args), +})); + +const modelGroups = [{ model_group: "gpt-5.2" }, { model_group: "claude-sonnet-5" }]; + +const defaultProps = { + open: true, + onClose: vi.fn(), + guardrailName: "pii-detector", + accessToken: "test-token", + onRunEvaluation: vi.fn(), +}; + +async function selectModel(user: ReturnType, label: string) { + await user.click(screen.getByRole("combobox")); + const options = await screen.findAllByText(label); + await user.click(options[options.length - 1]); +} + +describe("EvaluationSettingsModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockFetchAvailableModels.mockResolvedValue(modelGroups); + }); + + it("should render nothing while closed", () => { + render(); + expect(screen.queryByText("Evaluation Settings")).not.toBeInTheDocument(); + }); + + it("should show the title and the guardrail-specific description when open", () => { + render(); + expect(screen.getByText("Evaluation Settings")).toBeInTheDocument(); + expect(screen.getByText("Configure AI evaluation for pii-detector")).toBeInTheDocument(); + }); + + it("should fall back to a generic description when no guardrail name is given", () => { + render(); + expect(screen.getByText("Configure AI evaluation for re-running on logs")).toBeInTheDocument(); + }); + + it("should prefill the prompt and the response schema with their defaults", () => { + render(); + expect(screen.getByDisplayValue(/Evaluate whether this guardrail's decision was correct/)).toBeInTheDocument(); + expect( + screen.getByDisplayValue(/"verdict": "correct" \| "false_positive" \| "false_negative"/), + ).toBeInTheDocument(); + }); + + it("should restore the default prompt when 'Reset to default' is clicked", async () => { + const user = userEvent.setup(); + render(); + + const promptBox = screen.getByDisplayValue(/Evaluate whether this guardrail's decision was correct/); + await user.clear(promptBox); + await user.type(promptBox, "custom prompt"); + expect(screen.getByDisplayValue("custom prompt")).toBeInTheDocument(); + + await user.click(screen.getByText("Reset to default")); + expect(screen.getByDisplayValue(/Evaluate whether this guardrail's decision was correct/)).toBeInTheDocument(); + }); + + it("should load the available models with the access token when opened", async () => { + render(); + await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalledWith("test-token")); + }); + + it("should not load models when there is no access token", () => { + render(); + expect(mockFetchAvailableModels).not.toHaveBeenCalled(); + }); + + it("should not run an evaluation while no model is selected", async () => { + const user = userEvent.setup(); + const onRunEvaluation = vi.fn(); + const onClose = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /run evaluation/i })); + + expect(onRunEvaluation).not.toHaveBeenCalled(); + expect(onClose).not.toHaveBeenCalled(); + }); + + it("should run the evaluation with the selected model and the current prompt and schema", async () => { + const user = userEvent.setup(); + const onRunEvaluation = vi.fn(); + const onClose = vi.fn(); + render(); + + await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalled()); + await selectModel(user, "claude-sonnet-5"); + await user.click(screen.getByRole("button", { name: /run evaluation/i })); + + expect(onRunEvaluation).toHaveBeenCalledWith({ + model: "claude-sonnet-5", + prompt: expect.stringContaining("Evaluate whether this guardrail's decision was correct"), + schema: expect.stringContaining('"verdict"'), + }); + expect(onClose).toHaveBeenCalled(); + }); + + it("should close without running when 'Cancel' is clicked", async () => { + const user = userEvent.setup(); + const onRunEvaluation = vi.fn(); + const onClose = vi.fn(); + render(); + + await user.click(screen.getByRole("button", { name: /cancel/i })); + + expect(onClose).toHaveBeenCalled(); + expect(onRunEvaluation).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx index 0edfa65dfe8..900a04e480d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx @@ -1,7 +1,17 @@ -import { CloseOutlined, PlayCircleOutlined } from "@ant-design/icons"; -import { Button, Modal, Select, Input } from "antd"; -import React, { useEffect, useState } from "react"; +import { Play } from "lucide-react"; +import React, { useEffect, useMemo, useState } from "react"; import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Textarea } from "@/components/ui/textarea"; const DEFAULT_PROMPT = `Evaluate whether this guardrail's decision was correct. Analyze the user input, the guardrail action taken, and determine if it was appropriate. @@ -73,79 +83,81 @@ export function EvaluationSettingsModal({ } }; - const modelSelectOptions = modelOptions.map((m) => ({ - value: m.model_group, - label: m.model_group, - })); + const modelSelectOptions = useMemo( + () => modelOptions.map((m) => ({ value: m.model_group, label: m.model_group })), + [modelOptions], + ); return ( - } - destroyOnClose - > -

- {guardrailName - ? `Configure AI evaluation for ${guardrailName}` - : "Configure AI evaluation for re-running on logs"} -

+ !nextOpen && onClose()}> + + + Evaluation Settings + + {guardrailName + ? `Configure AI evaluation for ${guardrailName}` + : "Configure AI evaluation for re-running on logs"} + + -
-
-
- - +
+
+
+ + +
+