From 1e8c2f724090079d5fc3ad12d6a2563710c985ea Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 9 Jul 2026 20:23:49 -0700 Subject: [PATCH 1/4] fix(mcp): fail closed and surface semantic filter context window errors Resolves LIT-4284 When the embedding model exceeded its context window, the MCP semantic tool filter silently passed all tools through and reported N->N success in the filter header; when the overflow happened while embedding tool descriptions at router build time, the hook was never registered at all and filtering was silently disabled Semantic filtering now fails closed on context window overflows: the request is rejected with HTTP 400 and a message that names the embedding model and advises switching to one with a larger context window or disabling the filter. Build time overflows are recorded on the filter so the hook still registers and blocks MCP tool requests with the same actionable error while leaving native-only requests untouched. The dashboard test panel renders the backend message in an error banner instead of a success state. OpenAI's embedding overflow message (maximum input length is N tokens) now maps to ContextWindowExceededError --- .../exception_mapping_utils.py | 3 + .../mcp_server/semantic_tool_filter.py | 47 ++++ .../proxy/hooks/mcp_semantic_filter/hook.py | 9 + .../test_exception_mapping_utils.py | 12 + .../mcp_server/test_semantic_tool_filter.py | 266 ++++++++++++++++++ .../MCPSemanticFilterSettings.tsx | 3 + .../MCPSemanticFilterTestPanel.test.tsx | 15 + .../MCPSemanticFilterTestPanel.tsx | 12 + .../semanticFilterTestUtils.test.ts | 32 +++ .../semanticFilterTestUtils.ts | 5 + 10 files changed, 404 insertions(+) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 9dc202c4717..fdab3d5b9d4 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -95,6 +95,9 @@ class ExceptionCheckers: if "current length is" in _error_str_lowercase and "while limit is" in _error_str_lowercase: return True + if "maximum input length is" in _error_str_lowercase and "tokens" in _error_str_lowercase: + return True + return False @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index f24d5715e83..eb86206c566 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -7,6 +7,8 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints. from typing import TYPE_CHECKING, Any, Dict, List, Optional from litellm._logging import verbose_logger +from litellm.exceptions import ContextWindowExceededError +from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR if TYPE_CHECKING: @@ -15,6 +17,33 @@ if TYPE_CHECKING: from litellm.router import Router +class SemanticToolFilterContextWindowError(Exception): + """Raised when the embedding model exceeds its context window, so semantic filtering cannot run.""" + + def __init__(self, embedding_model: str, stage: str, original_error: str): + self.embedding_model = embedding_model + self.stage = stage + self.original_error = original_error + super().__init__( + f"MCP semantic tool filtering could not run: embedding model '{embedding_model}' " + f"exceeded its context window while embedding {stage}. " + f"The request was blocked instead of silently passing all tools through. " + f"Switch to an embedding model with a larger context window, or disable " + f"semantic tool filtering. Original error: {original_error}" + ) + + +def _is_context_window_error(error: Optional[BaseException], depth: int = 5) -> bool: + """Detect a context-window overflow anywhere in an exception's cause chain.""" + if error is None or depth == 0: + return False + if isinstance(error, ContextWindowExceededError): + return True + if ExceptionCheckers.is_error_str_context_window_exceeded(str(error)): + return True + return _is_context_window_error(error.__cause__, depth - 1) + + class SemanticMCPToolFilter: """Filters MCP tools using semantic similarity to reduce context window size.""" @@ -42,6 +71,7 @@ class SemanticMCPToolFilter: self.embedding_model = embedding_model self.router_instance = litellm_router_instance self.tool_router: Optional["SemanticRouter"] = None + self.context_window_error: Optional[str] = None self._tool_map: Dict[str, Any] = {} # MCPTool objects or OpenAI function dicts async def build_router_from_mcp_registry(self) -> None: @@ -111,6 +141,7 @@ class SemanticMCPToolFilter: return try: + self.context_window_error = None # Convert tools to routes routes = [] self._tool_map = {} @@ -143,6 +174,9 @@ class SemanticMCPToolFilter: except Exception as e: verbose_logger.error(f"Failed to build semantic router: {e}") self.tool_router = None + if _is_context_window_error(e): + self.context_window_error = str(e) + return raise async def filter_tools( @@ -169,6 +203,13 @@ class SemanticMCPToolFilter: if not available_tools: return available_tools + if self.context_window_error is not None: + raise SemanticToolFilterContextWindowError( + embedding_model=self.embedding_model, + stage="the MCP tool descriptions during semantic router build", + original_error=self.context_window_error, + ) + if not query or not query.strip(): return available_tools @@ -189,6 +230,12 @@ class SemanticMCPToolFilter: return self._get_tools_by_names(matched_tool_names, available_tools) except Exception as e: + if _is_context_window_error(e): + raise SemanticToolFilterContextWindowError( + embedding_model=self.embedding_model, + stage="the user query", + original_error=str(e), + ) from e verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True) return available_tools diff --git a/litellm/proxy/hooks/mcp_semantic_filter/hook.py b/litellm/proxy/hooks/mcp_semantic_filter/hook.py index 7379096bf9b..bad6ef44ccd 100644 --- a/litellm/proxy/hooks/mcp_semantic_filter/hook.py +++ b/litellm/proxy/hooks/mcp_semantic_filter/hook.py @@ -7,6 +7,8 @@ Reduces context window size and improves tool selection accuracy. from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from fastapi import HTTPException + from litellm._logging import verbose_proxy_logger from litellm.constants import ( DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL, @@ -14,6 +16,9 @@ from litellm.constants import ( DEFAULT_MCP_SEMANTIC_FILTER_TOP_K, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, +) if TYPE_CHECKING: from litellm.caching.caching import DualCache @@ -294,6 +299,8 @@ class SemanticToolFilterHook(CustomLogger): ) return data + except SemanticToolFilterContextWindowError as e: + raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: verbose_proxy_logger.error(f"Failed to expand MCP references: {e}", exc_info=True) return None @@ -366,6 +373,8 @@ class SemanticToolFilterHook(CustomLogger): return data + except SemanticToolFilterContextWindowError as e: + raise HTTPException(status_code=400, detail={"error": str(e)}) from e except Exception as e: verbose_proxy_logger.warning(f"Semantic tool filter hook failed: {e}. Proceeding with all tools.") return None diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index f441270be7c..1fcee1b1c42 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -78,6 +78,18 @@ context_window_test_cases = [ "CerebrasException - Please reduce the length of the messages or completion. Current length is 50000 while limit is 40000", True, ), + ( + "Invalid 'input[0]': maximum input length is 8192 tokens.", + True, + ), + ( + "OpenAIException - Error code: 400 - {'error': {'message': \"Invalid 'input[0]': maximum input length is 8192 tokens.\", 'type': 'invalid_request_error'}}", + True, + ), + ( + "Invalid 'metadata': maximum input length is 512 characters.", + False, + ), # Negative cases (should return False) ("A generic API error occurred.", False), ("Invalid API Key provided.", False), diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py index 82c0aa3ccda..edf6921de1e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py @@ -1356,3 +1356,269 @@ def test_truncate_csv_at_tool_name_boundary_edges(): assert _truncate_csv_at_tool_name_boundary(tool_names_csv="ab,cd,ef", max_length=5) == "ab,cd" assert _truncate_csv_at_tool_name_boundary(tool_names_csv="ab,cd,ef", max_length=4) == "ab" assert _truncate_csv_at_tool_name_boundary(tool_names_csv="single_name_longer_than_cap", max_length=10) == "" + + +def _make_context_window_raising_router(state): + """ + Mock litellm Router whose embedding call raises ContextWindowExceededError + once state["raise_context_error"] is flipped to True. + """ + import litellm + from litellm.types.utils import Embedding, EmbeddingResponse + + def mock_embedding_sync(*args, **kwargs): + if state["raise_context_error"]: + raise litellm.ContextWindowExceededError( + message="Invalid 'input[0]': maximum input length is 8192 tokens.", + model="text-embedding-3-small", + llm_provider="openai", + ) + return EmbeddingResponse( + data=[Embedding(embedding=[0.1] * 1536, index=0, object="embedding")], + model="text-embedding-3-small", + object="list", + usage={"prompt_tokens": 10, "total_tokens": 10}, + ) + + async def mock_embedding_async(*args, **kwargs): + return mock_embedding_sync(*args, **kwargs) + + mock_router = Mock() + mock_router.embedding = mock_embedding_sync + mock_router.aembedding = mock_embedding_async + return mock_router + + +def _make_context_window_filter(state, top_k: int = 3): + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticMCPToolFilter, + ) + + return SemanticMCPToolFilter( + embedding_model="text-embedding-3-small", + litellm_router_instance=_make_context_window_raising_router(state), + top_k=top_k, + similarity_threshold=0.3, + enabled=True, + ) + + +@pytest.mark.asyncio +async def test_semantic_filter_fails_closed_on_query_time_context_window_error(): + """ + Regression test (LIT-4284): a context-window overflow while embedding the + user query must fail closed with a typed error instead of silently + returning all tools (previously reported as N->N "success"). + """ + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, + ) + + state = {"raise_context_error": False} + filter_instance = _make_context_window_filter(state) + + tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(tools) + assert filter_instance.tool_router is not None + + state["raise_context_error"] = True + with pytest.raises(SemanticToolFilterContextWindowError) as exc_info: + await filter_instance.filter_tools(query="send an email", available_tools=tools) + + message = str(exc_info.value) + assert "context window" in message + assert "text-embedding-3-small" in message + print("✅ Query-time context window overflow fails closed") + + +@pytest.mark.asyncio +async def test_semantic_filter_records_build_time_context_window_error(): + """ + Regression test (LIT-4284): a context-window overflow while embedding the + tool descriptions at router-build time must be recorded (not raised out of + the build, which previously left the hook unregistered and filtering + silently disabled) and must fail subsequent filtering closed. + """ + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + SemanticToolFilterContextWindowError, + ) + + state = {"raise_context_error": True} + filter_instance = _make_context_window_filter(state) + + tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(tools) + + assert filter_instance.tool_router is None + assert filter_instance.context_window_error is not None + + with pytest.raises(SemanticToolFilterContextWindowError) as exc_info: + await filter_instance.filter_tools(query="send an email", available_tools=tools) + + message = str(exc_info.value) + assert "context window" in message + assert "tool descriptions" in message + print("✅ Build-time context window overflow is recorded and fails closed") + + +@pytest.mark.asyncio +async def test_semantic_filter_hook_fails_closed_on_context_window_error(): + """ + Regression test (LIT-4284): the pre-call hook must reject the request with + an actionable HTTP 400 when the embedding model overflows its context + window, instead of forwarding all tools and emitting an N->N success + header. + """ + from fastapi import HTTPException + + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + + state = {"raise_context_error": False} + filter_instance = _make_context_window_filter(state) + + tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(tools) + hook = SemanticToolFilterHook(filter_instance) + + state["raise_context_error"] = True + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Send an email"}], + "tools": tools, + "metadata": {}, + } + + with pytest.raises(HTTPException) as exc_info: + await hook.async_pre_call_hook( + user_api_key_dict=Mock(), + cache=Mock(), + data=data, + call_type="completion", + ) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert "context window" in error_message + assert "text-embedding-3-small" in error_message + assert "larger context window" in error_message + print("✅ Hook fails closed with actionable 400 on context window overflow") + + +@pytest.mark.asyncio +async def test_semantic_filter_hook_fails_closed_on_expanded_tools_context_window_error(): + """ + Regression test (LIT-4284): the litellm_proxy MCP expansion path (driven + by the dashboard test panel via /v1/responses) must also fail closed with + an actionable HTTP 400 instead of being swallowed by the expansion + catch-all. + """ + from fastapi import HTTPException + + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + + state = {"raise_context_error": False} + filter_instance = _make_context_window_filter(state) + + registry_tools = [ + MCPTool(name=f"srv-tool_{i}", description=f"Registry tool {i}", inputSchema={"type": "object"}) + for i in range(5) + ] + filter_instance._build_router(registry_tools) + + expanded_tools = [ + { + "type": "function", + "name": f"srv-tool_{i}", + "description": f"Registry tool {i}", + "parameters": {"type": "object", "properties": {}}, + } + for i in range(5) + ] + + hook = SemanticToolFilterHook(filter_instance) + hook._expand_mcp_tools = AsyncMock( # type: ignore[method-assign] + return_value=expanded_tools + ) + + state["raise_context_error"] = True + data = { + "model": "gpt-4", + "input": [{"role": "user", "content": "Send an email", "type": "message"}], + "tools": [ + { + "type": "mcp", + "server_url": "litellm_proxy", + "require_approval": "never", + } + ], + "metadata": {}, + } + + with pytest.raises(HTTPException) as exc_info: + await hook.async_pre_call_hook( + user_api_key_dict=Mock(), + cache=Mock(), + data=data, + call_type="aresponses", + ) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert "context window" in error_message + assert "larger context window" in error_message + print("✅ Expansion path fails closed with actionable 400 on context window overflow") + + +@pytest.mark.asyncio +async def test_semantic_filter_hook_ignores_build_error_for_native_only_tools(): + """ + A recorded build-time context-window error must only block requests that + rely on MCP tool filtering; requests carrying only native tools pass + through untouched. + """ + from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook + + state = {"raise_context_error": True} + filter_instance = _make_context_window_filter(state) + + mcp_tools = [ + MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"}) + for i in range(3) + ] + filter_instance._build_router(mcp_tools) + assert filter_instance.context_window_error is not None + + hook = SemanticToolFilterHook(filter_instance) + + native_tools = [ + { + "type": "function", + "function": {"name": "local_fn", "description": "A local function", "parameters": {}}, + } + ] + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Send an email"}], + "tools": native_tools, + "metadata": {}, + } + + result = await hook.async_pre_call_hook( + user_api_key_dict=Mock(), + cache=Mock(), + data=data, + call_type="completion", + ) + + assert result is not None + assert result["tools"] == native_tools + print("✅ Native-only requests pass through despite recorded build error") diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx index 38b1420f97e..888ffa43261 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx @@ -46,6 +46,7 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi const [testQuery, setTestQuery] = useState(""); const [testModel, setTestModel] = useState("gpt-4o"); const [testResult, setTestResult] = useState(null); + const [testError, setTestError] = useState(null); const [isTesting, setIsTesting] = useState(false); const schema = data?.field_schema; @@ -113,6 +114,7 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi testQuery, setIsTesting, setTestResult, + setTestError, }); }; @@ -285,6 +287,7 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi onTest={handleTest} filterEnabled={!!values.enabled} testResult={testResult} + testError={testError} curlCommand={getCurlCommand(testModel, testQuery)} /> diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx index 280a28e4765..c35caffc4be 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.test.tsx @@ -27,6 +27,7 @@ const buildProps = (overrides: Partial { expect(screen.queryByText("Results")).not.toBeInTheDocument(); }); + it("should render an error banner with the backend message when testError is set", () => { + const testError = + "MCP semantic tool filtering could not run: embedding model 'text-embedding-3-small' exceeded its context window while embedding the user query. Switch to an embedding model with a larger context window, or disable semantic tool filtering."; + render(); + + expect(screen.getByText("Semantic filtering did not run")).toBeInTheDocument(); + expect(screen.getByText(testError)).toBeInTheDocument(); + }); + + it("should not render the error banner when testError is null", () => { + render(); + expect(screen.queryByText("Semantic filtering did not run")).not.toBeInTheDocument(); + }); + it("should show the curl command in the API Usage tab", async () => { const user = userEvent.setup(); const curlCommand = "curl --location 'http://localhost:4000/v1/responses' --header 'Authorization: Bearer sk-1234'"; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.tsx index 74850020aa8..21aded8733c 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterTestPanel.tsx @@ -13,6 +13,7 @@ interface MCPSemanticFilterTestPanelProps { onTest: () => void; filterEnabled: boolean; testResult: TestResult | null; + testError: string | null; curlCommand: string; } @@ -26,6 +27,7 @@ export default function MCPSemanticFilterTestPanel({ onTest, filterEnabled, testResult, + testError, curlCommand, }: MCPSemanticFilterTestPanelProps) { return ( @@ -82,6 +84,16 @@ export default function MCPSemanticFilterTestPanel({ /> )} + {testError && ( + + )} + {testResult && (
Results diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts index abb18edb053..5880cca3f8f 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.test.ts @@ -27,12 +27,14 @@ describe("getCurlCommand", () => { describe("runSemanticFilterTest", () => { const mockSetIsTesting = vi.fn(); const mockSetTestResult = vi.fn(); + const mockSetTestError = vi.fn(); const baseArgs = { accessToken: "test-token", testModel: "gpt-4o", testQuery: "find relevant files", setIsTesting: mockSetIsTesting, setTestResult: mockSetTestResult, + setTestError: mockSetTestError, }; beforeEach(() => { @@ -112,4 +114,34 @@ describe("runSemanticFilterTest", () => { expect(NotificationManager.error).toHaveBeenCalledWith("Failed to test semantic filter"); expect(mockSetIsTesting).toHaveBeenLastCalledWith(false); }); + + it("should surface the backend error message via setTestError when the API call fails", async () => { + const backendMessage = + "MCP semantic tool filtering could not run: embedding model 'text-embedding-3-small' exceeded its context window while embedding the user query."; + vi.mocked(testMCPSemanticFilter).mockRejectedValueOnce(new Error(backendMessage)); + + await runSemanticFilterTest(baseArgs); + + expect(mockSetTestError).toHaveBeenLastCalledWith(backendMessage); + }); + + it("should clear the previous test error before making a new request", async () => { + vi.mocked(testMCPSemanticFilter).mockResolvedValueOnce({ + data: {}, + headers: { filter: "5->2", tools: "tool-a,tool-b" }, + }); + + await runSemanticFilterTest(baseArgs); + + expect(mockSetTestError).toHaveBeenCalledTimes(1); + expect(mockSetTestError).toHaveBeenCalledWith(null); + }); + + it("should fall back to a generic message when the thrown error has no message", async () => { + vi.mocked(testMCPSemanticFilter).mockRejectedValueOnce(new Error("")); + + await runSemanticFilterTest(baseArgs); + + expect(mockSetTestError).toHaveBeenLastCalledWith("Failed to test semantic filter"); + }); }); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.ts index 97fc1465d56..065014e2869 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.ts +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/semanticFilterTestUtils.ts @@ -29,12 +29,14 @@ export const runSemanticFilterTest = async ({ testQuery, setIsTesting, setTestResult, + setTestError, }: { accessToken: string; testModel: string; testQuery: string; setIsTesting: (value: boolean) => void; setTestResult: (result: TestResult | null) => void; + setTestError: (error: string | null) => void; }) => { if (!testQuery || !testModel || !accessToken) { NotificationManager.error("Please enter a query and select a model"); @@ -43,6 +45,7 @@ export const runSemanticFilterTest = async ({ setIsTesting(true); setTestResult(null); + setTestError(null); try { const { headers } = await testMCPSemanticFilter(accessToken, testModel, testQuery); @@ -57,6 +60,8 @@ export const runSemanticFilterTest = async ({ NotificationManager.success("Semantic filter test completed successfully"); } catch (error) { console.error("Test failed:", error); + const message = error instanceof Error && error.message ? error.message : "Failed to test semantic filter"; + setTestError(message); NotificationManager.error("Failed to test semantic filter"); } finally { setIsTesting(false); From 898182b0e6e34111489f414cbdffb0343de25b38 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 9 Jul 2026 20:35:57 -0700 Subject: [PATCH 2/4] fix(mcp): redact provider error from client-facing semantic filter message Keep the full provider exception in server-side logs only; the client receives a fixed actionable message. Also follow implicit exception context when detecting context window overflows and pin the detection variants plus the redaction in tests --- .../mcp_server/semantic_tool_filter.py | 8 +++- .../mcp_server/test_semantic_tool_filter.py | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index eb86206c566..0a73420e9d2 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -29,7 +29,7 @@ class SemanticToolFilterContextWindowError(Exception): f"exceeded its context window while embedding {stage}. " f"The request was blocked instead of silently passing all tools through. " f"Switch to an embedding model with a larger context window, or disable " - f"semantic tool filtering. Original error: {original_error}" + f"semantic tool filtering." ) @@ -41,7 +41,7 @@ def _is_context_window_error(error: Optional[BaseException], depth: int = 5) -> return True if ExceptionCheckers.is_error_str_context_window_exceeded(str(error)): return True - return _is_context_window_error(error.__cause__, depth - 1) + return _is_context_window_error(error.__cause__ or error.__context__, depth - 1) class SemanticMCPToolFilter: @@ -231,6 +231,10 @@ class SemanticMCPToolFilter: except Exception as e: if _is_context_window_error(e): + verbose_logger.error( + f"Semantic tool filter embedding exceeded its context window: {e}", + exc_info=True, + ) raise SemanticToolFilterContextWindowError( embedding_model=self.embedding_model, stage="the user query", diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py index edf6921de1e..9bc0a525326 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_semantic_tool_filter.py @@ -1510,6 +1510,7 @@ async def test_semantic_filter_hook_fails_closed_on_context_window_error(): assert "context window" in error_message assert "text-embedding-3-small" in error_message assert "larger context window" in error_message + assert "maximum input length" not in error_message print("✅ Hook fails closed with actionable 400 on context window overflow") @@ -1575,6 +1576,7 @@ async def test_semantic_filter_hook_fails_closed_on_expanded_tools_context_windo error_message = exc_info.value.detail["error"] assert "context window" in error_message assert "larger context window" in error_message + assert "maximum input length" not in error_message print("✅ Expansion path fails closed with actionable 400 on context window overflow") @@ -1622,3 +1624,43 @@ async def test_semantic_filter_hook_ignores_build_error_for_native_only_tools(): assert result is not None assert result["tools"] == native_tools print("✅ Native-only requests pass through despite recorded build error") + + +def test_is_context_window_error_detection_variants(): + """ + _is_context_window_error must detect the overflow in every shape it + reaches filter_tools in: the raw typed exception, the encoder's + explicitly chained ValueError wrapper, an implicitly chained wrapper, + and a bare error whose message carries a known overflow phrase; a + generic error must not match. + """ + import litellm + from litellm.proxy._experimental.mcp_server.semantic_tool_filter import ( + _is_context_window_error, + ) + + cwe = litellm.ContextWindowExceededError( + message="Invalid 'input[0]': maximum input length is 8192 tokens.", + model="text-embedding-3-small", + llm_provider="openai", + ) + assert _is_context_window_error(cwe) + + try: + raise ValueError("Internal_litellm_router API call failed") from cwe + except ValueError as explicitly_chained: + assert _is_context_window_error(explicitly_chained) + + try: + try: + raise litellm.ContextWindowExceededError( + message="overflow", model="m", llm_provider="openai" + ) + except litellm.ContextWindowExceededError: + raise ValueError("wrapper without explicit chaining") + except ValueError as implicitly_chained: + assert _is_context_window_error(implicitly_chained) + + assert _is_context_window_error(ValueError("Invalid 'input[0]': maximum input length is 8192 tokens.")) + assert not _is_context_window_error(ValueError("A generic API error occurred.")) + assert not _is_context_window_error(None) From 38efe9872010ab96add30d64fcca7891a5d89384 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 9 Jul 2026 23:06:47 -0700 Subject: [PATCH 3/4] refactor(mcp): make context window detection iterative for the recursion gate The code-quality recursive_detector CI step bans recursive functions under litellm/; walk the exception cause chain with a bounded loop instead --- .../mcp_server/semantic_tool_filter.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py index 0a73420e9d2..e12c6cdbd56 100644 --- a/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py +++ b/litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py @@ -33,15 +33,18 @@ class SemanticToolFilterContextWindowError(Exception): ) -def _is_context_window_error(error: Optional[BaseException], depth: int = 5) -> bool: +def _is_context_window_error(error: Optional[BaseException], max_depth: int = 5) -> bool: """Detect a context-window overflow anywhere in an exception's cause chain.""" - if error is None or depth == 0: - return False - if isinstance(error, ContextWindowExceededError): - return True - if ExceptionCheckers.is_error_str_context_window_exceeded(str(error)): - return True - return _is_context_window_error(error.__cause__ or error.__context__, depth - 1) + current = error + for _ in range(max_depth): + if current is None: + return False + if isinstance(current, ContextWindowExceededError): + return True + if ExceptionCheckers.is_error_str_context_window_exceeded(str(current)): + return True + current = current.__cause__ or current.__context__ + return False class SemanticMCPToolFilter: From d2b7996170c16767b1f08d9327fd2f96c9422d78 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 10 Jul 2026 00:57:31 -0700 Subject: [PATCH 4/4] test(ui): pin deriveErrorMessage against the ProxyException wire shape Both automated reviewers assumed the semantic filter 400 reaches the browser as FastAPI's flat detail dict and would render as raw JSON in the test panel banner. The proxy converts a pre-call hook HTTPException into a ProxyException that serializes as {"error": {"message": ...}}, which deriveErrorMessage unpacks first; pin that contract with direct tests --- .../src/lib/http/client.test.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/lib/http/client.test.ts b/ui/litellm-dashboard/src/lib/http/client.test.ts index 6d50f99feca..e0b5a73d11a 100644 --- a/ui/litellm-dashboard/src/lib/http/client.test.ts +++ b/ui/litellm-dashboard/src/lib/http/client.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi } from "vitest"; -import { createApiClient, ApiError } from "./client"; +import { createApiClient, ApiError, deriveErrorMessage } from "./client"; const okResponse = (data: unknown): Response => ({ ok: true, status: 200, text: async () => JSON.stringify(data) }) as unknown as Response; @@ -101,3 +101,22 @@ describe("createApiClient", () => { } }); }); + +describe("deriveErrorMessage", () => { + it("extracts error.message from a ProxyException body, the shape the proxy emits for a pre-call hook HTTPException", () => { + const actionable = + "MCP semantic tool filtering could not run: embedding model 'text-embedding-3-small' exceeded its context window while embedding the user query. The request was blocked instead of silently passing all tools through. Switch to an embedding model with a larger context window, or disable semantic tool filtering."; + const wireBody = { + error: { message: actionable, type: "None", param: "None", code: "400" }, + }; + expect(deriveErrorMessage(wireBody)).toBe(actionable); + }); + + it("returns error directly when it is a plain string", () => { + expect(deriveErrorMessage({ error: "flat error text" })).toBe("flat error text"); + }); + + it("falls back to a string detail field", () => { + expect(deriveErrorMessage({ detail: "detail text" })).toBe("detail text"); + }); +});