diff --git a/dist/litellm-1.57.6.tar.gz b/dist/litellm-1.57.6.tar.gz deleted file mode 100644 index 01a039cf6ee..00000000000 Binary files a/dist/litellm-1.57.6.tar.gz and /dev/null differ diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py index 60d89c1610f..f925e6819dc 100644 --- a/litellm/llms/base_llm/passthrough/transformation.py +++ b/litellm/llms/base_llm/passthrough/transformation.py @@ -31,30 +31,26 @@ class BasePassthroughConfig(BaseLLMModelInfo): Args: endpoint: str - the endpoint to add to the url base_target_url: str - the base url to add the endpoint to - request_query_params: dict - the query params to add to the url + request_query_params: Optional[dict] - the query params to add to the url Returns: - str - the formatted url + httpx.URL - the formatted url """ from urllib.parse import urlencode import httpx - encoded_endpoint = httpx.URL(endpoint).path + base = base_target_url.rstrip('/') + endpoint = endpoint.lstrip('/') + full_url = f"{base}/{endpoint}" - # Ensure endpoint starts with '/' for proper URL construction - if not encoded_endpoint.startswith("/"): - encoded_endpoint = "/" + encoded_endpoint - - # Construct the full target URL using httpx - base_url = httpx.URL(base_target_url) - updated_url = base_url.copy_with(path=encoded_endpoint) + url = httpx.URL(full_url) if request_query_params: - # Create a new URL with the merged query params - updated_url = updated_url.copy_with( + url = url.copy_with( query=urlencode(request_query_params).encode("ascii") ) - return updated_url + + return url @abstractmethod def get_complete_url( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b93ca94bed4..273b12c9c39 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -10,6 +10,7 @@ from typing import List, Literal, Optional, Tuple, Union, cast, overload import httpx import litellm +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -322,7 +323,6 @@ class AmazonConverseConfig(BaseConfig): def _create_json_tool_call_for_response_format( self, json_schema: Optional[dict] = None, - schema_name: str = "json_tool_call", description: Optional[str] = None, ) -> ChatCompletionToolParam: """ @@ -347,7 +347,7 @@ class AmazonConverseConfig(BaseConfig): _input_schema = json_schema tool_param_function_chunk = ChatCompletionToolParamFunctionChunk( - name=schema_name, parameters=_input_schema + name=RESPONSE_FORMAT_TOOL_NAME, parameters=_input_schema ) if description: tool_param_function_chunk["description"] = description @@ -391,14 +391,11 @@ class AmazonConverseConfig(BaseConfig): continue json_schema: Optional[dict] = None - schema_name: str = "" description: Optional[str] = None if "response_schema" in value: json_schema = value["response_schema"] - schema_name = "json_tool_call" elif "json_schema" in value: json_schema = value["json_schema"]["schema"] - schema_name = value["json_schema"]["name"] description = value["json_schema"].get("description") if "type" in value and value["type"] == "text": @@ -414,7 +411,6 @@ class AmazonConverseConfig(BaseConfig): """ _tool = self._create_json_tool_call_for_response_format( json_schema=json_schema, - schema_name=schema_name if schema_name != "" else "json_tool_call", description=description, ) optional_params = self._add_tools_to_optional_params( @@ -430,7 +426,7 @@ class AmazonConverseConfig(BaseConfig): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock( - name=schema_name if schema_name != "" else "json_tool_call" + name=RESPONSE_FORMAT_TOOL_NAME ) ) optional_params["json_mode"] = True @@ -1119,8 +1115,7 @@ class AmazonConverseConfig(BaseConfig): self._transform_thinking_blocks(reasoningContentBlocks) ) chat_completion_message["content"] = content_str - if json_mode is True and tools is not None and len(tools) == 1: - # to support 'json_schema' logic on bedrock models + if json_mode is True and tools is not None and len(tools) == 1 and tools[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME: json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments") if json_mode_content_str is not None: chat_completion_message["content"] = json_mode_content_str diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index d7221ff4b7a..5791bfb8013 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -41,9 +41,15 @@ class BedrockPassthroughConfig( model_id=None, ) - api_base = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" + aws_bedrock_runtime_endpoint = optional_params.get("aws_bedrock_runtime_endpoint") + endpoint_url, _ = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + endpoint_type="runtime", + ) - return self.format_url(endpoint, api_base, request_query_params or {}), api_base + return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url def sign_request( self, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index e70cadddaf7..392d47f9822 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,6 +1,15 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hints +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Optional, + Union, + cast, + get_type_hints, +) import httpx +from openai.types.responses import ResponseReasoningItem from pydantic import BaseModel import litellm @@ -92,12 +101,67 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): # if it's pydantic, convert to dict if isinstance(item, BaseModel): validated_input.append(item.model_dump(exclude_none=True)) + elif isinstance(item, dict): + # Handle reasoning items specifically to filter out status=None + verbose_logger.debug(f"Handling reasoning item: {item}") + if item.get("type") == "reasoning": + # Type assertion since we know it's a dict at this point + dict_item = cast(Dict[str, Any], item) + filtered_item = self._handle_reasoning_item(dict_item) + else: + # For other dict items, just pass through + filtered_item = cast(Dict[str, Any], item) + validated_input.append(filtered_item) else: validated_input.append(item) - return validated_input + return validated_input # type: ignore # Input is expected to be either str or List, no single BaseModel expected return input + def _handle_reasoning_item(self, item: Dict[str, Any]) -> Dict[str, Any]: + """ + Handle reasoning items specifically to filter out status=None using OpenAI's model. + Issue: https://github.com/BerriAI/litellm/issues/13484 + OpenAI API does not accept ReasoningItem(status=None), so we need to: + 1. Check if the item is a reasoning type + 2. Create a ResponseReasoningItem object with the item data + 3. Convert it back to dict with exclude_none=True to filter None values + """ + verbose_logger.debug(f"Handling reasoning item: {item}") + if item.get("type") == "reasoning": + try: + # Ensure required fields are present for ResponseReasoningItem + item_data = dict(item) + if "id" not in item_data: + item_data["id"] = f"reasoning_{hash(str(item_data))}" + if "summary" not in item_data: + item_data["summary"] = ( + item_data.get("reasoning_content", "")[:100] + "..." + if len(item_data.get("reasoning_content", "")) > 100 + else item_data.get("reasoning_content", "") + ) + + # Create ResponseReasoningItem object from the item data + reasoning_item = ResponseReasoningItem(**item_data) + + # Convert back to dict with exclude_none=True to exclude None fields + dict_reasoning_item = reasoning_item.model_dump(exclude_none=True) + + return dict_reasoning_item + except Exception as e: + verbose_logger.debug( + f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}" + ) + # Fallback: manually filter out known None fields + filtered_item = { + k: v + for k, v in item.items() + if v is not None + or k not in {"status", "content", "encrypted_content"} + } + return filtered_item + return item + def transform_response_api_response( self, model: str, diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index f254a197aaf..2939e884a56 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -662,3 +662,58 @@ class BaseResponsesAPITest(ABC): # Validate the response print("Response:", json.dumps(response, indent=4, default=str)) + + def test_openai_responses_api_dict_input_filtering(self): + """ + Test that regular dict inputs with status fields are properly filtered + to replicate exclude_unset=True behavior for non-Pydantic objects. + """ + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + + # Test input with regular dict objects (like from JSON) + test_input = [ + { + "role": "user", + "content": "test" + }, + { + "id": "rs_123", + "summary": [{"text": "test", "type": "summary_text"}], + "type": "reasoning", + "content": None, # Should be filtered out + "encrypted_content": None, # Should be filtered out + "status": None # Should be filtered out + }, + { + "arguments": "{}", + "call_id": "call_123", + "name": "get_today", + "type": "function_call", + "id": "fc_123", + "status": "completed" # Should be preserved (not a default field) + } + ] + + config = OpenAIResponsesAPIConfig() + validated_input = config._validate_input_param(test_input) + + # Verify the results + assert len(validated_input) == 3 + + # Check reasoning item (index 1) + reasoning_item = validated_input[1] + assert reasoning_item["type"] == "reasoning" + assert "status" not in reasoning_item, "status field should be filtered out from reasoning item" + assert "content" not in reasoning_item, "content field should be filtered out from reasoning item" + assert "encrypted_content" not in reasoning_item, "encrypted_content field should be filtered out from reasoning item" + assert "id" in reasoning_item, "id field should be preserved" + assert "summary" in reasoning_item, "summary field should be preserved" + + # Check function call item (index 2) + function_call_item = validated_input[2] + assert function_call_item["type"] == "function_call" + assert "status" in function_call_item, "status field should be preserved in function call item" + assert function_call_item["status"] == "completed", "status value should be preserved" + + print("✅ OpenAI Responses API dict input filtering test passed") + diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 619ae338e50..285a406b3fb 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -665,7 +665,6 @@ async def test_openai_gpt5_reasoning(): print("response: ", response) assert response.choices[0].message.content is not None - @pytest.mark.asyncio async def test_openai_safety_identifier_parameter(): """Test that safety_identifier parameter is correctly passed to the OpenAI API.""" @@ -723,3 +722,4 @@ def test_openai_safety_identifier_parameter_sync(): assert "safety_identifier" in request_body # Verify safety_identifier is correctly sent to the API assert request_body["safety_identifier"] == "user_code_123456" + diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 1c91cc0fe8b..2fc710664e6 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -475,6 +475,239 @@ def test_transform_response_with_bash_tool(): assert args["command"] == "ls -la *.py" +def test_transform_response_with_structured_response_being_called(): + """Test response transformation with structured response.""" + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.types.utils import ModelResponse + + # Simulate a Bedrock Converse response with a bash tool call + response_json = { + "additionalModelResponseFields": {}, + "metrics": {"latencyMs": 100.0}, + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_456", + "name": "json_tool_call", + "input": { + "Current_Temperature": 62, + "Weather_Explanation": "San Francisco typically has mild, cool weather year-round due to its coastal location and marine influence. The city is known for its fog, moderate temperatures, and relatively stable climate with little seasonal variation."}, + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 8, + "outputTokens": 3, + "totalTokens": 11, + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + }, + } + # Mock httpx.Response + class MockResponse: + def json(self): + return response_json + @property + def text(self): + return json.dumps(response_json) + + config = AmazonConverseConfig() + model_response = ModelResponse() + optional_params = { + "json_mode": True, + "tools": [ + { + 'type': 'function', + 'function': { + 'name': 'get_weather', + 'description': 'Get the current weather in a given location', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'The city and state, e.g. San Francisco, CA' + }, + 'unit': { + 'type': 'string', + 'enum': ['celsius', 'fahrenheit'] + } + }, + 'required': ['location'] + } + } + }, + { + 'type': 'function', + 'function': { + 'name': 'json_tool_call', + 'parameters': { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'type': 'object', + 'required': ['Weather_Explanation', 'Current_Temperature'], + 'properties': { + 'Weather_Explanation': { + 'type': ['string', 'null'], + 'description': '1-2 sentences explaining the weather in the location' + }, + 'Current_Temperature': { + 'type': ['number', 'null'], + 'description': 'Current temperature in the location' + } + }, + 'additionalProperties': False + } + } + } + ] + } + # Call the transformation logic + result = config._transform_response( + model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + response=MockResponse(), + model_response=model_response, + stream=False, + logging_obj=None, + optional_params=optional_params, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + # Check that the tool call is present in the returned message + assert result.choices[0].message.tool_calls is None + + assert result.choices[0].message.content is not None + assert result.choices[0].message.content == '{"Current_Temperature": 62, "Weather_Explanation": "San Francisco typically has mild, cool weather year-round due to its coastal location and marine influence. The city is known for its fog, moderate temperatures, and relatively stable climate with little seasonal variation."}' + +def test_transform_response_with_structured_response_calling_tool(): + """Test response transformation with structured response.""" + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.types.utils import ModelResponse + + # Simulate a Bedrock Converse response with a bash tool call + response_json = { + "metrics": { + "latencyMs": 1148 + }, + "output": { + "message": + { + "content": [ + { + "text": "I\'ll check the current weather in San Francisco for you." + }, + { + "toolUse": { + "input": { + "location": "San Francisco, CA", + "unit": "celsius" + }, + "name": "get_weather", + "toolUseId": "tooluse_oKk__QrqSUmufMw3Q7vGaQ" + } + } + ], + "role": "assistant" + } + }, + "stopReason": "tool_use", + "usage": { + "cacheReadInputTokenCount": 0, + "cacheReadInputTokens": 0, + "cacheWriteInputTokenCount": 0, + "cacheWriteInputTokens": 0, + "inputTokens": 534, + "outputTokens": 69, + "totalTokens": 603 + } + } + # Mock httpx.Response + class MockResponse: + def json(self): + return response_json + @property + def text(self): + return json.dumps(response_json) + + config = AmazonConverseConfig() + model_response = ModelResponse() + optional_params = { + "json_mode": True, + "tools": [ + { + 'type': 'function', + 'function': { + 'name': 'get_weather', + 'description': 'Get the current weather in a given location', + 'parameters': { + 'type': 'object', + 'properties': { + 'location': { + 'type': 'string', + 'description': 'The city and state, e.g. San Francisco, CA' + }, + 'unit': { + 'type': 'string', + 'enum': ['celsius', 'fahrenheit'] + } + }, + 'required': ['location'] + } + } + }, + { + 'type': 'function', + 'function': { + 'name': 'json_tool_call', + 'parameters': { + '$schema': 'http://json-schema.org/draft-07/schema#', + 'type': 'object', + 'required': ['Weather_Explanation', 'Current_Temperature'], + 'properties': { + 'Weather_Explanation': { + 'type': ['string', 'null'], + 'description': '1-2 sentences explaining the weather in the location' + }, + 'Current_Temperature': { + 'type': ['number', 'null'], + 'description': 'Current temperature in the location' + } + }, + 'additionalProperties': False + } + } + } + ] + } + # Call the transformation logic + result = config._transform_response( + model="bedrock/eu.anthropic.claude-sonnet-4-20250514-v1:0", + response=MockResponse(), + model_response=model_response, + stream=False, + logging_obj=None, + optional_params=optional_params, + api_key=None, + data=None, + messages=[], + encoding=None, + ) + # Check that the tool call is present in the returned message + assert result.choices[0].message.tool_calls is not None + assert len(result.choices[0].message.tool_calls) == 1 + assert result.choices[0].message.tool_calls[0].function.name == "get_weather" + assert result.choices[0].message.tool_calls[0].function.arguments == '{"location": "San Francisco, CA", "unit": "celsius"}' + + @pytest.mark.asyncio async def test_bedrock_bash_tool_acompletion(): """Test Bedrock with bash tool for ls command using acompletion.""" @@ -938,6 +1171,68 @@ def test_transform_request_with_function_tool(): assert request_data["toolConfig"]["tools"][0]["toolSpec"]["name"] == "get_weather" +def test_map_openai_params_with_response_format(): + """Test map_openai_params with response_format.""" + config = AmazonConverseConfig() + + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + } + } + ] + + json_schema = { + "type": "json_schema", + "json_schema": { + "name": "WeatherResult", + "schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["Weather_Explanation", "Current_Temperature"], + "properties": { + "Weather_Explanation": { + "type": ["string", "null"], + "description": "1-2 sentences explaining the weather in the location", + }, + "Current_Temperature": { + "type": ["number", "null"], + "description": "Current temperature in the location", + }, + }, + "additionalProperties": False, + }, + "strict": False, + }, + } + + optional_params = config.map_openai_params( + non_default_params={"response_format": json_schema}, + optional_params={"tools": tools}, + model="eu.anthropic.claude-sonnet-4-20250514-v1:0", + drop_params=False + ) + + assert "tools" in optional_params + assert len(optional_params["tools"]) == 2 + assert optional_params["tools"][1]["type"] == "function" + assert optional_params["tools"][1]["function"]["name"] == "json_tool_call" + + @pytest.mark.asyncio async def test_assistant_message_cache_control(): """Test that assistant messages with cache_control generate cachePoint blocks.""" diff --git a/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py new file mode 100644 index 00000000000..7cb1ee2b54a --- /dev/null +++ b/tests/test_litellm/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -0,0 +1,177 @@ +import os +import sys +from unittest.mock import patch + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig + + +def test_bedrock_passthrough_get_complete_url_default_endpoint(): + """Test get_complete_url with default AWS endpoint (no override)""" + config = BedrockPassthroughConfig() + + # Mock the methods following the pattern from test_base_aws_llm.py + with patch.object(config, '_get_aws_region_name', return_value="us-east-1"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "https://bedrock-runtime.us-east-1.amazonaws.com", + "https://bedrock-runtime.us-east-1.amazonaws.com" + )) as mock_get_runtime: + + url, api_base = config.get_complete_url( + api_base=None, + api_key=None, + model="anthropic.claude-3-sonnet", + endpoint="/model/anthropic.claude-3-sonnet/invoke", + request_query_params=None, + litellm_params={} + ) + + # Verify get_runtime_endpoint was called with correct parameters + mock_get_runtime.assert_called_once_with( + api_base=None, + aws_bedrock_runtime_endpoint=None, + aws_region_name="us-east-1", + endpoint_type="runtime" + ) + + # Verify URL construction + assert str(url) == "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet/invoke" + assert api_base == "https://bedrock-runtime.us-east-1.amazonaws.com" + + +def test_bedrock_passthrough_get_complete_url_custom_endpoint_no_path(): + """Test get_complete_url with custom endpoint (no base path)""" + config = BedrockPassthroughConfig() + + with patch.object(config, '_get_aws_region_name', return_value="us-west-2"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "http://proxy.com", + "http://proxy.com" + )) as mock_get_runtime: + + url, api_base = config.get_complete_url( + api_base="http://proxy.com", + api_key=None, + model="anthropic.claude-3-sonnet", + endpoint="/model/anthropic.claude-3-sonnet/invoke", + request_query_params=None, + litellm_params={} + ) + + # Verify get_runtime_endpoint was called with the api_base + mock_get_runtime.assert_called_once_with( + api_base="http://proxy.com", + aws_bedrock_runtime_endpoint=None, + aws_region_name="us-west-2", + endpoint_type="runtime" + ) + + # Verify URL construction + assert str(url) == "http://proxy.com/model/anthropic.claude-3-sonnet/invoke" + assert api_base == "http://proxy.com" + + +def test_bedrock_passthrough_get_complete_url_custom_endpoint_with_path(): + """Test get_complete_url with custom endpoint that has a base path""" + config = BedrockPassthroughConfig() + + with patch.object(config, '_get_aws_region_name', return_value="us-west-2"), \ + patch.object(config, 'get_runtime_endpoint', return_value=( + "http://proxy.com/bedrockproxy", + "http://proxy.com/bedrockproxy" + )) as mock_get_runtime: + + url, api_base = config.get_complete_url( + api_base="http://proxy.com/bedrockproxy", + api_key=None, + model="anthropic.claude-3-sonnet", + endpoint="/model/anthropic.claude-3-sonnet/invoke", + request_query_params=None, + litellm_params={ + "aws_bedrock_runtime_endpoint": "http://proxy.com/bedrockproxy" + } + ) + + # Verify get_runtime_endpoint was called with correct parameters + mock_get_runtime.assert_called_once_with( + api_base="http://proxy.com/bedrockproxy", + aws_bedrock_runtime_endpoint="http://proxy.com/bedrockproxy", + aws_region_name="us-west-2", + endpoint_type="runtime" + ) + + # Verify URL construction preserves the proxy path + assert str(url) == "http://proxy.com/bedrockproxy/model/anthropic.claude-3-sonnet/invoke" + assert api_base == "http://proxy.com/bedrockproxy" + + +def test_format_url_simple_joining(): + """Test format_url with simple URL joining""" + config = BedrockPassthroughConfig() + + result = config.format_url( + endpoint="model/test/invoke", + base_target_url="https://api.example.com", + request_query_params={} + ) + + assert str(result) == "https://api.example.com/model/test/invoke" + + +def test_format_url_preserves_proxy_paths(): + """Test format_url preserves proxy paths in base URL""" + config = BedrockPassthroughConfig() + + result = config.format_url( + endpoint="model/test/invoke", + base_target_url="http://proxy.com/bedrockproxy", + request_query_params={} + ) + + # This is the key test - proxy path should be preserved + assert str(result) == "http://proxy.com/bedrockproxy/model/test/invoke" + + +def test_format_url_with_query_parameters(): + """Test format_url properly handles query parameters""" + config = BedrockPassthroughConfig() + + result = config.format_url( + endpoint="model/test/invoke", + base_target_url="http://proxy.com/bedrockproxy", + request_query_params={"param1": "value1", "param2": "value2"} + ) + + # Should preserve proxy path and add query params + result_str = str(result) + assert "http://proxy.com/bedrockproxy/model/test/invoke" in result_str + assert "param1=value1" in result_str + assert "param2=value2" in result_str + + +def test_format_url_handles_trailing_slash_normalization(): + """Test format_url properly handles base URLs with and without trailing slashes""" + config = BedrockPassthroughConfig() + + # Test with trailing slash + result_with_slash = config.format_url( + endpoint="model/test/invoke", + base_target_url="http://proxy.com/bedrockproxy/", + request_query_params={} + ) + + # Test without trailing slash + result_without_slash = config.format_url( + endpoint="model/test/invoke", + base_target_url="http://proxy.com/bedrockproxy", + request_query_params={} + ) + + # Both should produce the same result + assert str(result_with_slash) == str(result_without_slash) + assert str(result_with_slash) == "http://proxy.com/bedrockproxy/model/test/invoke" + + diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 6d46a40f6c6..21232161d0c 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -668,3 +668,248 @@ def test_get_supported_openai_params(): assert "stream" in params assert "background" in params assert "stream" in params + + +class TestOpenAIFieldExclusionRegistry: + """Test suite for the OpenAI Field Exclusion Registry system""" + + def setup_method(self): + """Setup test fixtures""" + from litellm.llms.openai.responses.transformation import ( + OpenAIFieldExclusionRegistry, + OpenAIResponsesAPIConfig + ) + self.registry = OpenAIFieldExclusionRegistry + self.config = OpenAIResponsesAPIConfig() + + def test_registry_initialization(self): + """Test that the registry is properly initialized with ResponseReasoningItem""" + # Test that we can get excluded fields (should not be empty if ResponseReasoningItem is registered) + all_excluded_fields = self.registry.get_all_excluded_fields() + + # The registry should have at least some fields if ResponseReasoningItem was successfully registered + # If OpenAI SDK is not available, this might be empty, which is also valid + assert isinstance(all_excluded_fields, set), "get_all_excluded_fields should return a set" + + # If we have the OpenAI SDK available, we should have the expected fields + try: + from openai.types.responses import ResponseReasoningItem + reasoning_fields = self.registry.get_excluded_fields_for_model(ResponseReasoningItem) + expected_fields = {'status', 'content', 'encrypted_content'} + assert expected_fields.issubset(reasoning_fields), f"Expected fields {expected_fields} to be subset of {reasoning_fields}" + except ImportError: + # If OpenAI SDK is not available, that's fine - the registry should handle this gracefully + pytest.skip("OpenAI SDK not available, skipping ResponseReasoningItem specific tests") + + def test_register_model_functionality(self): + """Test that we can register new models to the registry""" + from pydantic import BaseModel + from typing import Optional + + # Create a test model with default None fields + class TestResponseModel(BaseModel): + id: str + type: str = "test" + status: Optional[str] = None + content: Optional[str] = None + required_field: str + + # Register the test model + self.registry.register_model(TestResponseModel) + + # Verify it was registered and fields are detected + excluded_fields = self.registry.get_excluded_fields_for_model(TestResponseModel) + expected_excluded = {'status', 'content'} # Fields with default None + + assert expected_excluded.issubset(excluded_fields), f"Expected {expected_excluded} to be in {excluded_fields}" + assert 'id' not in excluded_fields, "Required field 'id' should not be excluded" + assert 'required_field' not in excluded_fields, "Required field 'required_field' should not be excluded" + + def test_get_all_excluded_fields(self): + """Test that get_all_excluded_fields aggregates fields from all registered models""" + all_fields_before = self.registry.get_all_excluded_fields() + + # Create and register a test model + from pydantic import BaseModel + from typing import Optional + + class AnotherTestModel(BaseModel): + id: str + unique_field: Optional[str] = None + + self.registry.register_model(AnotherTestModel) + + all_fields_after = self.registry.get_all_excluded_fields() + + # The new fields should be included + assert 'unique_field' in all_fields_after, "New model's excluded field should be included" + assert len(all_fields_after) >= len(all_fields_before), "Should have at least as many fields as before" + + def test_convenience_registration_method(self): + """Test the convenience method for registering models""" + from pydantic import BaseModel + from typing import Optional + + class ConvenienceTestModel(BaseModel): + id: str + convenience_field: Optional[str] = None + + # Use the convenience method + self.config.register_model_for_field_exclusion(ConvenienceTestModel) + + # Verify it was registered + excluded_fields = self.registry.get_excluded_fields_for_model(ConvenienceTestModel) + assert 'convenience_field' in excluded_fields, "Field should be excluded after registration" + + def test_field_filtering_with_registry(self): + """Test that the field filtering works correctly with the registry""" + + # Test data that matches the structure of ResponseReasoningItem + test_input = [ + { + "role": "user", + "content": "test message" + }, + { + "id": "reasoning-123", + "type": "reasoning", + "status": None, # Should be filtered out + "content": None, # Should be filtered out + "encrypted_content": None, # Should be filtered out + "summary": [{"text": "This reasoning shows...", "type": "summary_text"}], + "role": "assistant" + }, + { + "id": "message-456", + "type": "message", + "status": "completed", # Should be preserved (not None) + "content": "Hello! How can I help?", # Should be preserved (not None) + "role": "assistant" + } + ] + + # Process the input through the validation + result = self.config._validate_input_param(test_input) + + # Verify the structure + assert len(result) == 3, "Should have 3 items" + + # Check the reasoning item (index 1) + reasoning_item = result[1] + assert reasoning_item["type"] == "reasoning" + assert reasoning_item["id"] == "reasoning-123" + assert "summary" in reasoning_item, "summary field should be preserved" + assert "role" in reasoning_item, "role field should be preserved" + + # These fields should be filtered out if they are in the registry + all_excluded_fields = self.registry.get_all_excluded_fields() + if 'status' in all_excluded_fields: + assert "status" not in reasoning_item, "status field should be filtered out" + if 'content' in all_excluded_fields: + assert "content" not in reasoning_item, "content field should be filtered out" + if 'encrypted_content' in all_excluded_fields: + assert "encrypted_content" not in reasoning_item, "encrypted_content field should be filtered out" + + # Check the message item (index 2) - non-None values should be preserved + message_item = result[2] + assert message_item["type"] == "message" + assert message_item["status"] == "completed", "Non-None status should be preserved" + assert message_item["content"] == "Hello! How can I help?", "Non-None content should be preserved" + + def test_field_filtering_with_empty_registry(self): + """Test that filtering works gracefully when no models are registered""" + # Create a fresh registry for this test + from litellm.llms.openai.responses.transformation import OpenAIFieldExclusionRegistry + + # Save the current state + original_models = OpenAIFieldExclusionRegistry._MODELS_REQUIRING_EXCLUSION.copy() + + try: + # Clear the registry + OpenAIFieldExclusionRegistry._MODELS_REQUIRING_EXCLUSION.clear() + + # Test data + test_input = [{ + "id": "test-123", + "status": None, + "content": None, + "other_field": "should be preserved" + }] + + # Process the input + result = self.config._validate_input_param(test_input) + + # With empty registry, nothing should be filtered (all fields preserved) + assert len(result) == 1 + item = result[0] + assert "status" in item, "With empty registry, status should be preserved" + assert "content" in item, "With empty registry, content should be preserved" + assert item["other_field"] == "should be preserved" + + finally: + # Restore the original state + OpenAIFieldExclusionRegistry._MODELS_REQUIRING_EXCLUSION = original_models + + def test_pydantic_v1_v2_compatibility(self): + """Test that the registry works with both Pydantic v1 and v2""" + from pydantic import BaseModel + from typing import Optional + + class CompatibilityTestModel(BaseModel): + id: str + optional_field: Optional[str] = None + required_field: str = "default" + + # Register the model + self.registry.register_model(CompatibilityTestModel) + + # Get excluded fields + excluded_fields = self.registry.get_excluded_fields_for_model(CompatibilityTestModel) + + # Should work regardless of Pydantic version + assert isinstance(excluded_fields, set), "Should return a set" + assert 'optional_field' in excluded_fields, "Field with default None should be excluded" + + # Test that the model fields are accessible (works in both v1 and v2) + model_fields = getattr(CompatibilityTestModel, "model_fields", None) + if model_fields is None: + model_fields = getattr(CompatibilityTestModel, "__fields__", {}) + assert len(model_fields) > 0, "Should be able to access model fields" + + def test_non_registered_model_returns_empty_set(self): + """Test that non-registered models return empty excluded fields""" + from pydantic import BaseModel + + class UnregisteredModel(BaseModel): + id: str + some_field: str = None + + # Don't register this model + excluded_fields = self.registry.get_excluded_fields_for_model(UnregisteredModel) + + assert excluded_fields == set(), "Non-registered model should return empty set" + + @pytest.mark.parametrize("field_value", [None, "", 0, False, []]) + def test_only_none_values_are_filtered(self, field_value): + """Test that only None values are filtered, not other falsy values""" + test_input = [{ + "id": "test-123", + "status": field_value, + "content": "actual content", + "other_field": "preserved" + }] + + result = self.config._validate_input_param(test_input) + item = result[0] + + if field_value is None: + # Only None should be filtered (if status is in the registry) + all_excluded_fields = self.registry.get_all_excluded_fields() + if 'status' in all_excluded_fields: + assert "status" not in item, f"None value should be filtered out" + else: + assert item["status"] is None, f"If not in registry, None should be preserved" + else: + # Other falsy values should be preserved + assert "status" in item, f"Non-None value {field_value} should be preserved" + assert item["status"] == field_value, f"Value should be exactly {field_value}" diff --git a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx index 30f01d91db5..cca7bc08608 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx @@ -8,6 +8,7 @@ import { useReactTable, ColumnResizeMode, VisibilityState, + PaginationState, } from "@tanstack/react-table"; import React from "react"; import { @@ -18,7 +19,7 @@ import { TableRow, TableCell, } from "@tremor/react"; -import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon, TableIcon } from "@heroicons/react/outline"; +import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; // Extend the column meta type to include className declare module "@tanstack/react-table" { diff --git a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx index 85f18ee89e1..2eff0bf3e36 100644 --- a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef } from "react" +import React, { useState, useEffect, useRef, useMemo } from "react" import { Card, Title, @@ -61,7 +61,7 @@ import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; import PassThroughSettings from "../pass_through_settings"; import ModelGroupAliasSettings from "../model_group_alias_settings"; import { all_admin_roles } from "@/utils/roles"; -import { Table as TableInstance } from "@tanstack/react-table"; +import { Table as TableInstance, PaginationState } from "@tanstack/react-table"; import NotificationsManager from "../molecules/notifications_manager"; interface ModelDashboardProps { @@ -191,12 +191,21 @@ const ModelDashboard: React.FC = ({ const [currentTeam, setCurrentTeam] = useState("personal") // 'personal' or team_id const [modelViewMode, setModelViewMode] = useState<"current_team" | "all">("current_team") + // Add state for showing/hiding filters + const [showFilters, setShowFilters] = useState(false) + const [showColumnDropdown, setShowColumnDropdown] = useState(false) const [isDropdownOpen, setIsDropdownOpen] = useState(false) const [expandedRows, setExpandedRows] = useState>(new Set()) const dropdownRef = useRef(null) const tableRef = useRef>(null) + + // Pagination state + const [pagination, setPagination] = useState({ + pageIndex: 0, + pageSize: 50, + }) const [selectedTabIndex, setSelectedTabIndex] = useState(0) const handleCreateNewModelClick = () => { @@ -206,6 +215,63 @@ const ModelDashboard: React.FC = ({ setSelectedTabIndex(1) } + const resetFilters = () => { + setModelNameSearch("") + setSelectedModelGroup("all") + setSelectedModelAccessGroupFilter(null) + setCurrentTeam("personal") + setModelViewMode("current_team") + setPagination({ pageIndex: 0, pageSize: 50 }) + } + + // Memoize filtered data to prevent unnecessary re-calculations + const filteredData = useMemo(() => { + if (!modelData || !modelData.data || modelData.data.length === 0) { + return []; + } + + return modelData.data.filter((model: any) => { + const searchMatch = + modelNameSearch === "" || + model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) + + const modelNameMatch = + selectedModelGroup === "all" || + model.model_name === selectedModelGroup || + !selectedModelGroup || + (selectedModelGroup === "wildcard" && model.model_name?.includes("*")) + + const accessGroupMatch = + selectedModelAccessGroupFilter === "all" || + model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || + !selectedModelAccessGroupFilter + + let teamAccessMatch = true + if (modelViewMode === "current_team") { + if (currentTeam === "personal") { + teamAccessMatch = model.model_info?.direct_access === true + } else { + teamAccessMatch = + model.model_info?.access_via_team_ids?.includes(currentTeam) === true + } + } + + return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch + }); + }, [modelData, modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); + + // Memoize paginated data + const paginatedData = useMemo(() => { + const startIndex = pagination.pageIndex * pagination.pageSize; + const endIndex = startIndex + pagination.pageSize; + return filteredData.slice(startIndex, endIndex); + }, [filteredData, pagination.pageIndex, pagination.pageSize]); + + // Reset pagination when filters change + useEffect(() => { + setPagination(prev => ({ ...prev, pageIndex: 0 })) + }, [modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]) + const setProviderModelsFn = (provider: Providers) => { const _providerModels = getProviderModels(provider, modelMap) setProviderModels(_providerModels) @@ -1006,119 +1072,166 @@ const ModelDashboard: React.FC = ({
-
-
- {/* Current Team Selector - Prominent */} -
-
-
- Current Team: - setCurrentTeam(value)} + > + +
+
+ Personal +
+ + {teams + ?.filter((team) => team.team_id) + .map((team) => ( +
-
- Personal +
+ + {team.team_alias + ? `${team.team_alias.slice(0, 30)}...` + : `Team ${team.team_id.slice(0, 30)}...`} +
- {teams - ?.filter((team) => team.team_id) - .map((team) => ( - -
-
- - {team.team_alias - ? `${team.team_alias.slice(0, 30)}...` - : `Team ${team.team_id.slice(0, 30)}...`} - -
- - ))} - -
- {modelViewMode === "current_team" && ( -
- -
- {currentTeam === "personal" ? ( - - To access these models: Create a Virtual Key without selecting a team on the{" "} - - Virtual Keys page - - - ) : ( - - To access these models: Create a Virtual Key and select Team as " - {currentTeam}" on the{" "} - - Virtual Keys page - - - )} -
-
- )} -
- - {/* Model View Mode Toggle - Also prominent */} -
- View: - -
+ ))} +
- {/* Other Filters */} -
-
- {/* Model Name Search */} -
- Search Public Model Name: - -
+
+ View: + +
+
+ + {modelViewMode === "current_team" && ( +
+ +
+ {currentTeam === "personal" ? ( + + To access these models: Create a Virtual Key without selecting a team on the{" "} + + Virtual Keys page + + + ) : ( + + To access these models: Create a Virtual Key and select Team as " + {currentTeam}" on the{" "} + + Virtual Keys page + + + )} +
+
+ )} +
+ {/* Search and Filter Controls */} +
+
+ {/* Search and Filter Controls */} +
+ {/* Model Name Search */} +
+ setModelNameSearch(e.target.value)} + /> + + + +
+ + {/* Filter Button */} + + + {/* Reset Filters Button */} + +
+ + {/* Additional Filters */} + {showFilters && ( +
{/* Model Name Filter */} -
- Filter by Public Model Name: +
-
- Filter by Model Access Group: + {/* Model Access Group Filter */} +
-
+ )} - {/* Results Count */} + {/* Results Count and Pagination Controls */}
- - Showing{" "} - {modelData && modelData.data.length > 0 - ? modelData.data.filter((model: any) => { - const searchMatch = - modelNameSearch === "" || - model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) - - const modelNameMatch = - selectedModelGroup === "all" || - model.model_name === selectedModelGroup || - !selectedModelGroup - const accessGroupMatch = - selectedModelAccessGroupFilter === "all" || - model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || - !selectedModelAccessGroupFilter - let teamAccessMatch = true - if (modelViewMode === "current_team") { - if (currentTeam === "personal") { - teamAccessMatch = model.model_info?.direct_access === true - } else { - teamAccessMatch = - model.model_info?.access_via_team_ids?.includes(currentTeam) === true - } - } - - return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch - }).length - : 0}{" "} - results - + + {filteredData.length > 0 ? ( + `Showing ${pagination.pageIndex * pagination.pageSize + 1} - ${Math.min( + (pagination.pageIndex + 1) * pagination.pageSize, + filteredData.length + )} of ${filteredData.length} results` + ) : ( + "Showing 0 results" + )} + + + {/* Pagination Controls */} + {filteredData.length > pagination.pageSize && ( +
+ + + +
+ )}
@@ -1202,38 +1320,7 @@ const ModelDashboard: React.FC = ({ expandedRows, setExpandedRows, )} - data={modelData.data.filter((model: any) => { - // Model name search filter - const searchMatch = - modelNameSearch === "" || - model.model_name.toLowerCase().includes(modelNameSearch.toLowerCase()) - - // Model name filter - const modelNameMatch = - selectedModelGroup === "all" || - model.model_name === selectedModelGroup || - !selectedModelGroup || - (selectedModelGroup === "wildcard" && model.model_name?.includes("*")) - // Model access group filter - const accessGroupMatch = - selectedModelAccessGroupFilter === "all" || - model.model_info["access_groups"]?.includes(selectedModelAccessGroupFilter) || - !selectedModelAccessGroupFilter - // Team access filter based on current team and view mode - let teamAccessMatch = true - if (modelViewMode === "current_team") { - if (currentTeam === "personal") { - // Show only models with direct access - teamAccessMatch = model.model_info?.direct_access === true - } else { - // Show only models accessible by the current team - teamAccessMatch = model.model_info?.access_via_team_ids?.includes(currentTeam) === true - } - } - // For 'all' mode, show all models (teamAccessMatch remains true) - - return searchMatch && modelNameMatch && accessGroupMatch && teamAccessMatch - })} + data={paginatedData} isLoading={false} table={tableRef} />