mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
fix(mcp): fail closed and surface semantic filter context window errors
Resolves LIT-4284 When the embedding model exceeded its context window, the MCP semantic tool filter silently passed all tools through and reported N->N success in the filter header; when the overflow happened while embedding tool descriptions at router build time, the hook was never registered at all and filtering was silently disabled Semantic filtering now fails closed on context window overflows: the request is rejected with HTTP 400 and a message that names the embedding model and advises switching to one with a larger context window or disabling the filter. Build time overflows are recorded on the filter so the hook still registers and blocks MCP tool requests with the same actionable error while leaving native-only requests untouched. The dashboard test panel renders the backend message in an error banner instead of a success state. OpenAI's embedding overflow message (maximum input length is N tokens) now maps to ContextWindowExceededError
This commit is contained in:
Tin Chi Lo
parent
bf02a4a47f
commit
1e8c2f7240
10 changed files with 404 additions and 0 deletions
|
|
@ -95,6 +95,9 @@ class ExceptionCheckers:
|
|||
if "current length is" in _error_str_lowercase and "while limit is" in _error_str_lowercase:
|
||||
return True
|
||||
|
||||
if "maximum input length is" in _error_str_lowercase and "tokens" in _error_str_lowercase:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ Filters MCP tools semantically for /chat/completions and /responses endpoints.
|
|||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.exceptions import ContextWindowExceededError
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
|
||||
from litellm.proxy._experimental.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -15,6 +17,33 @@ if TYPE_CHECKING:
|
|||
from litellm.router import Router
|
||||
|
||||
|
||||
class SemanticToolFilterContextWindowError(Exception):
|
||||
"""Raised when the embedding model exceeds its context window, so semantic filtering cannot run."""
|
||||
|
||||
def __init__(self, embedding_model: str, stage: str, original_error: str):
|
||||
self.embedding_model = embedding_model
|
||||
self.stage = stage
|
||||
self.original_error = original_error
|
||||
super().__init__(
|
||||
f"MCP semantic tool filtering could not run: embedding model '{embedding_model}' "
|
||||
f"exceeded its context window while embedding {stage}. "
|
||||
f"The request was blocked instead of silently passing all tools through. "
|
||||
f"Switch to an embedding model with a larger context window, or disable "
|
||||
f"semantic tool filtering. Original error: {original_error}"
|
||||
)
|
||||
|
||||
|
||||
def _is_context_window_error(error: Optional[BaseException], depth: int = 5) -> bool:
|
||||
"""Detect a context-window overflow anywhere in an exception's cause chain."""
|
||||
if error is None or depth == 0:
|
||||
return False
|
||||
if isinstance(error, ContextWindowExceededError):
|
||||
return True
|
||||
if ExceptionCheckers.is_error_str_context_window_exceeded(str(error)):
|
||||
return True
|
||||
return _is_context_window_error(error.__cause__, depth - 1)
|
||||
|
||||
|
||||
class SemanticMCPToolFilter:
|
||||
"""Filters MCP tools using semantic similarity to reduce context window size."""
|
||||
|
||||
|
|
@ -42,6 +71,7 @@ class SemanticMCPToolFilter:
|
|||
self.embedding_model = embedding_model
|
||||
self.router_instance = litellm_router_instance
|
||||
self.tool_router: Optional["SemanticRouter"] = None
|
||||
self.context_window_error: Optional[str] = None
|
||||
self._tool_map: Dict[str, Any] = {} # MCPTool objects or OpenAI function dicts
|
||||
|
||||
async def build_router_from_mcp_registry(self) -> None:
|
||||
|
|
@ -111,6 +141,7 @@ class SemanticMCPToolFilter:
|
|||
return
|
||||
|
||||
try:
|
||||
self.context_window_error = None
|
||||
# Convert tools to routes
|
||||
routes = []
|
||||
self._tool_map = {}
|
||||
|
|
@ -143,6 +174,9 @@ class SemanticMCPToolFilter:
|
|||
except Exception as e:
|
||||
verbose_logger.error(f"Failed to build semantic router: {e}")
|
||||
self.tool_router = None
|
||||
if _is_context_window_error(e):
|
||||
self.context_window_error = str(e)
|
||||
return
|
||||
raise
|
||||
|
||||
async def filter_tools(
|
||||
|
|
@ -169,6 +203,13 @@ class SemanticMCPToolFilter:
|
|||
if not available_tools:
|
||||
return available_tools
|
||||
|
||||
if self.context_window_error is not None:
|
||||
raise SemanticToolFilterContextWindowError(
|
||||
embedding_model=self.embedding_model,
|
||||
stage="the MCP tool descriptions during semantic router build",
|
||||
original_error=self.context_window_error,
|
||||
)
|
||||
|
||||
if not query or not query.strip():
|
||||
return available_tools
|
||||
|
||||
|
|
@ -189,6 +230,12 @@ class SemanticMCPToolFilter:
|
|||
return self._get_tools_by_names(matched_tool_names, available_tools)
|
||||
|
||||
except Exception as e:
|
||||
if _is_context_window_error(e):
|
||||
raise SemanticToolFilterContextWindowError(
|
||||
embedding_model=self.embedding_model,
|
||||
stage="the user query",
|
||||
original_error=str(e),
|
||||
) from e
|
||||
verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True)
|
||||
return available_tools
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ Reduces context window size and improves tool selection accuracy.
|
|||
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import (
|
||||
DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL,
|
||||
|
|
@ -14,6 +16,9 @@ from litellm.constants import (
|
|||
DEFAULT_MCP_SEMANTIC_FILTER_TOP_K,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._experimental.mcp_server.semantic_tool_filter import (
|
||||
SemanticToolFilterContextWindowError,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.caching.caching import DualCache
|
||||
|
|
@ -294,6 +299,8 @@ class SemanticToolFilterHook(CustomLogger):
|
|||
)
|
||||
return data
|
||||
|
||||
except SemanticToolFilterContextWindowError as e:
|
||||
raise HTTPException(status_code=400, detail={"error": str(e)}) from e
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"Failed to expand MCP references: {e}", exc_info=True)
|
||||
return None
|
||||
|
|
@ -366,6 +373,8 @@ class SemanticToolFilterHook(CustomLogger):
|
|||
|
||||
return data
|
||||
|
||||
except SemanticToolFilterContextWindowError as e:
|
||||
raise HTTPException(status_code=400, detail={"error": str(e)}) from e
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(f"Semantic tool filter hook failed: {e}. Proceeding with all tools.")
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -78,6 +78,18 @@ context_window_test_cases = [
|
|||
"CerebrasException - Please reduce the length of the messages or completion. Current length is 50000 while limit is 40000",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"Invalid 'input[0]': maximum input length is 8192 tokens.",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"OpenAIException - Error code: 400 - {'error': {'message': \"Invalid 'input[0]': maximum input length is 8192 tokens.\", 'type': 'invalid_request_error'}}",
|
||||
True,
|
||||
),
|
||||
(
|
||||
"Invalid 'metadata': maximum input length is 512 characters.",
|
||||
False,
|
||||
),
|
||||
# Negative cases (should return False)
|
||||
("A generic API error occurred.", False),
|
||||
("Invalid API Key provided.", False),
|
||||
|
|
|
|||
|
|
@ -1356,3 +1356,269 @@ def test_truncate_csv_at_tool_name_boundary_edges():
|
|||
assert _truncate_csv_at_tool_name_boundary(tool_names_csv="ab,cd,ef", max_length=5) == "ab,cd"
|
||||
assert _truncate_csv_at_tool_name_boundary(tool_names_csv="ab,cd,ef", max_length=4) == "ab"
|
||||
assert _truncate_csv_at_tool_name_boundary(tool_names_csv="single_name_longer_than_cap", max_length=10) == ""
|
||||
|
||||
|
||||
def _make_context_window_raising_router(state):
|
||||
"""
|
||||
Mock litellm Router whose embedding call raises ContextWindowExceededError
|
||||
once state["raise_context_error"] is flipped to True.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.types.utils import Embedding, EmbeddingResponse
|
||||
|
||||
def mock_embedding_sync(*args, **kwargs):
|
||||
if state["raise_context_error"]:
|
||||
raise litellm.ContextWindowExceededError(
|
||||
message="Invalid 'input[0]': maximum input length is 8192 tokens.",
|
||||
model="text-embedding-3-small",
|
||||
llm_provider="openai",
|
||||
)
|
||||
return EmbeddingResponse(
|
||||
data=[Embedding(embedding=[0.1] * 1536, index=0, object="embedding")],
|
||||
model="text-embedding-3-small",
|
||||
object="list",
|
||||
usage={"prompt_tokens": 10, "total_tokens": 10},
|
||||
)
|
||||
|
||||
async def mock_embedding_async(*args, **kwargs):
|
||||
return mock_embedding_sync(*args, **kwargs)
|
||||
|
||||
mock_router = Mock()
|
||||
mock_router.embedding = mock_embedding_sync
|
||||
mock_router.aembedding = mock_embedding_async
|
||||
return mock_router
|
||||
|
||||
|
||||
def _make_context_window_filter(state, top_k: int = 3):
|
||||
from litellm.proxy._experimental.mcp_server.semantic_tool_filter import (
|
||||
SemanticMCPToolFilter,
|
||||
)
|
||||
|
||||
return SemanticMCPToolFilter(
|
||||
embedding_model="text-embedding-3-small",
|
||||
litellm_router_instance=_make_context_window_raising_router(state),
|
||||
top_k=top_k,
|
||||
similarity_threshold=0.3,
|
||||
enabled=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_fails_closed_on_query_time_context_window_error():
|
||||
"""
|
||||
Regression test (LIT-4284): a context-window overflow while embedding the
|
||||
user query must fail closed with a typed error instead of silently
|
||||
returning all tools (previously reported as N->N "success").
|
||||
"""
|
||||
from litellm.proxy._experimental.mcp_server.semantic_tool_filter import (
|
||||
SemanticToolFilterContextWindowError,
|
||||
)
|
||||
|
||||
state = {"raise_context_error": False}
|
||||
filter_instance = _make_context_window_filter(state)
|
||||
|
||||
tools = [
|
||||
MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"})
|
||||
for i in range(5)
|
||||
]
|
||||
filter_instance._build_router(tools)
|
||||
assert filter_instance.tool_router is not None
|
||||
|
||||
state["raise_context_error"] = True
|
||||
with pytest.raises(SemanticToolFilterContextWindowError) as exc_info:
|
||||
await filter_instance.filter_tools(query="send an email", available_tools=tools)
|
||||
|
||||
message = str(exc_info.value)
|
||||
assert "context window" in message
|
||||
assert "text-embedding-3-small" in message
|
||||
print("✅ Query-time context window overflow fails closed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_records_build_time_context_window_error():
|
||||
"""
|
||||
Regression test (LIT-4284): a context-window overflow while embedding the
|
||||
tool descriptions at router-build time must be recorded (not raised out of
|
||||
the build, which previously left the hook unregistered and filtering
|
||||
silently disabled) and must fail subsequent filtering closed.
|
||||
"""
|
||||
from litellm.proxy._experimental.mcp_server.semantic_tool_filter import (
|
||||
SemanticToolFilterContextWindowError,
|
||||
)
|
||||
|
||||
state = {"raise_context_error": True}
|
||||
filter_instance = _make_context_window_filter(state)
|
||||
|
||||
tools = [
|
||||
MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"})
|
||||
for i in range(5)
|
||||
]
|
||||
filter_instance._build_router(tools)
|
||||
|
||||
assert filter_instance.tool_router is None
|
||||
assert filter_instance.context_window_error is not None
|
||||
|
||||
with pytest.raises(SemanticToolFilterContextWindowError) as exc_info:
|
||||
await filter_instance.filter_tools(query="send an email", available_tools=tools)
|
||||
|
||||
message = str(exc_info.value)
|
||||
assert "context window" in message
|
||||
assert "tool descriptions" in message
|
||||
print("✅ Build-time context window overflow is recorded and fails closed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_fails_closed_on_context_window_error():
|
||||
"""
|
||||
Regression test (LIT-4284): the pre-call hook must reject the request with
|
||||
an actionable HTTP 400 when the embedding model overflows its context
|
||||
window, instead of forwarding all tools and emitting an N->N success
|
||||
header.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
|
||||
|
||||
state = {"raise_context_error": False}
|
||||
filter_instance = _make_context_window_filter(state)
|
||||
|
||||
tools = [
|
||||
MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"})
|
||||
for i in range(5)
|
||||
]
|
||||
filter_instance._build_router(tools)
|
||||
hook = SemanticToolFilterHook(filter_instance)
|
||||
|
||||
state["raise_context_error"] = True
|
||||
data = {
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Send an email"}],
|
||||
"tools": tools,
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await hook.async_pre_call_hook(
|
||||
user_api_key_dict=Mock(),
|
||||
cache=Mock(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
error_message = exc_info.value.detail["error"]
|
||||
assert "context window" in error_message
|
||||
assert "text-embedding-3-small" in error_message
|
||||
assert "larger context window" in error_message
|
||||
print("✅ Hook fails closed with actionable 400 on context window overflow")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_fails_closed_on_expanded_tools_context_window_error():
|
||||
"""
|
||||
Regression test (LIT-4284): the litellm_proxy MCP expansion path (driven
|
||||
by the dashboard test panel via /v1/responses) must also fail closed with
|
||||
an actionable HTTP 400 instead of being swallowed by the expansion
|
||||
catch-all.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
|
||||
|
||||
state = {"raise_context_error": False}
|
||||
filter_instance = _make_context_window_filter(state)
|
||||
|
||||
registry_tools = [
|
||||
MCPTool(name=f"srv-tool_{i}", description=f"Registry tool {i}", inputSchema={"type": "object"})
|
||||
for i in range(5)
|
||||
]
|
||||
filter_instance._build_router(registry_tools)
|
||||
|
||||
expanded_tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": f"srv-tool_{i}",
|
||||
"description": f"Registry tool {i}",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
hook = SemanticToolFilterHook(filter_instance)
|
||||
hook._expand_mcp_tools = AsyncMock( # type: ignore[method-assign]
|
||||
return_value=expanded_tools
|
||||
)
|
||||
|
||||
state["raise_context_error"] = True
|
||||
data = {
|
||||
"model": "gpt-4",
|
||||
"input": [{"role": "user", "content": "Send an email", "type": "message"}],
|
||||
"tools": [
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never",
|
||||
}
|
||||
],
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await hook.async_pre_call_hook(
|
||||
user_api_key_dict=Mock(),
|
||||
cache=Mock(),
|
||||
data=data,
|
||||
call_type="aresponses",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
error_message = exc_info.value.detail["error"]
|
||||
assert "context window" in error_message
|
||||
assert "larger context window" in error_message
|
||||
print("✅ Expansion path fails closed with actionable 400 on context window overflow")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_semantic_filter_hook_ignores_build_error_for_native_only_tools():
|
||||
"""
|
||||
A recorded build-time context-window error must only block requests that
|
||||
rely on MCP tool filtering; requests carrying only native tools pass
|
||||
through untouched.
|
||||
"""
|
||||
from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
|
||||
|
||||
state = {"raise_context_error": True}
|
||||
filter_instance = _make_context_window_filter(state)
|
||||
|
||||
mcp_tools = [
|
||||
MCPTool(name=f"tool_{i}", description=f"Tool {i}", inputSchema={"type": "object"})
|
||||
for i in range(3)
|
||||
]
|
||||
filter_instance._build_router(mcp_tools)
|
||||
assert filter_instance.context_window_error is not None
|
||||
|
||||
hook = SemanticToolFilterHook(filter_instance)
|
||||
|
||||
native_tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "local_fn", "description": "A local function", "parameters": {}},
|
||||
}
|
||||
]
|
||||
data = {
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Send an email"}],
|
||||
"tools": native_tools,
|
||||
"metadata": {},
|
||||
}
|
||||
|
||||
result = await hook.async_pre_call_hook(
|
||||
user_api_key_dict=Mock(),
|
||||
cache=Mock(),
|
||||
data=data,
|
||||
call_type="completion",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result["tools"] == native_tools
|
||||
print("✅ Native-only requests pass through despite recorded build error")
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi
|
|||
const [testQuery, setTestQuery] = useState("");
|
||||
const [testModel, setTestModel] = useState<string>("gpt-4o");
|
||||
const [testResult, setTestResult] = useState<TestResult | null>(null);
|
||||
const [testError, setTestError] = useState<string | null>(null);
|
||||
const [isTesting, setIsTesting] = useState(false);
|
||||
|
||||
const schema = data?.field_schema;
|
||||
|
|
@ -113,6 +114,7 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi
|
|||
testQuery,
|
||||
setIsTesting,
|
||||
setTestResult,
|
||||
setTestError,
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -285,6 +287,7 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi
|
|||
onTest={handleTest}
|
||||
filterEnabled={!!values.enabled}
|
||||
testResult={testResult}
|
||||
testError={testError}
|
||||
curlCommand={getCurlCommand(testModel, testQuery)}
|
||||
/>
|
||||
</Col>
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ const buildProps = (overrides: Partial<React.ComponentProps<typeof MCPSemanticFi
|
|||
onTest: vi.fn(),
|
||||
filterEnabled: true,
|
||||
testResult: null as TestResult | null,
|
||||
testError: null as string | null,
|
||||
curlCommand: "curl --location 'http://localhost:4000/v1/responses'",
|
||||
...overrides,
|
||||
});
|
||||
|
|
@ -134,6 +135,20 @@ describe("MCPSemanticFilterTestPanel", () => {
|
|||
expect(screen.queryByText("Results")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should render an error banner with the backend message when testError is set", () => {
|
||||
const testError =
|
||||
"MCP semantic tool filtering could not run: embedding model 'text-embedding-3-small' exceeded its context window while embedding the user query. Switch to an embedding model with a larger context window, or disable semantic tool filtering.";
|
||||
render(<MCPSemanticFilterTestPanel {...buildProps({ testError })} />);
|
||||
|
||||
expect(screen.getByText("Semantic filtering did not run")).toBeInTheDocument();
|
||||
expect(screen.getByText(testError)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should not render the error banner when testError is null", () => {
|
||||
render(<MCPSemanticFilterTestPanel {...buildProps({ testError: null })} />);
|
||||
expect(screen.queryByText("Semantic filtering did not run")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show the curl command in the API Usage tab", async () => {
|
||||
const user = userEvent.setup();
|
||||
const curlCommand = "curl --location 'http://localhost:4000/v1/responses' --header 'Authorization: Bearer sk-1234'";
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ interface MCPSemanticFilterTestPanelProps {
|
|||
onTest: () => void;
|
||||
filterEnabled: boolean;
|
||||
testResult: TestResult | null;
|
||||
testError: string | null;
|
||||
curlCommand: string;
|
||||
}
|
||||
|
||||
|
|
@ -26,6 +27,7 @@ export default function MCPSemanticFilterTestPanel({
|
|||
onTest,
|
||||
filterEnabled,
|
||||
testResult,
|
||||
testError,
|
||||
curlCommand,
|
||||
}: MCPSemanticFilterTestPanelProps) {
|
||||
return (
|
||||
|
|
@ -82,6 +84,16 @@ export default function MCPSemanticFilterTestPanel({
|
|||
/>
|
||||
)}
|
||||
|
||||
{testError && (
|
||||
<Alert
|
||||
type="error"
|
||||
message="Semantic filtering did not run"
|
||||
description={testError}
|
||||
showIcon
|
||||
style={{ marginBottom: 16 }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{testResult && (
|
||||
<div>
|
||||
<Typography.Title level={5}>Results</Typography.Title>
|
||||
|
|
|
|||
|
|
@ -27,12 +27,14 @@ describe("getCurlCommand", () => {
|
|||
describe("runSemanticFilterTest", () => {
|
||||
const mockSetIsTesting = vi.fn();
|
||||
const mockSetTestResult = vi.fn();
|
||||
const mockSetTestError = vi.fn();
|
||||
const baseArgs = {
|
||||
accessToken: "test-token",
|
||||
testModel: "gpt-4o",
|
||||
testQuery: "find relevant files",
|
||||
setIsTesting: mockSetIsTesting,
|
||||
setTestResult: mockSetTestResult,
|
||||
setTestError: mockSetTestError,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
|
|
@ -112,4 +114,34 @@ describe("runSemanticFilterTest", () => {
|
|||
expect(NotificationManager.error).toHaveBeenCalledWith("Failed to test semantic filter");
|
||||
expect(mockSetIsTesting).toHaveBeenLastCalledWith(false);
|
||||
});
|
||||
|
||||
it("should surface the backend error message via setTestError when the API call fails", async () => {
|
||||
const backendMessage =
|
||||
"MCP semantic tool filtering could not run: embedding model 'text-embedding-3-small' exceeded its context window while embedding the user query.";
|
||||
vi.mocked(testMCPSemanticFilter).mockRejectedValueOnce(new Error(backendMessage));
|
||||
|
||||
await runSemanticFilterTest(baseArgs);
|
||||
|
||||
expect(mockSetTestError).toHaveBeenLastCalledWith(backendMessage);
|
||||
});
|
||||
|
||||
it("should clear the previous test error before making a new request", async () => {
|
||||
vi.mocked(testMCPSemanticFilter).mockResolvedValueOnce({
|
||||
data: {},
|
||||
headers: { filter: "5->2", tools: "tool-a,tool-b" },
|
||||
});
|
||||
|
||||
await runSemanticFilterTest(baseArgs);
|
||||
|
||||
expect(mockSetTestError).toHaveBeenCalledTimes(1);
|
||||
expect(mockSetTestError).toHaveBeenCalledWith(null);
|
||||
});
|
||||
|
||||
it("should fall back to a generic message when the thrown error has no message", async () => {
|
||||
vi.mocked(testMCPSemanticFilter).mockRejectedValueOnce(new Error(""));
|
||||
|
||||
await runSemanticFilterTest(baseArgs);
|
||||
|
||||
expect(mockSetTestError).toHaveBeenLastCalledWith("Failed to test semantic filter");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -29,12 +29,14 @@ export const runSemanticFilterTest = async ({
|
|||
testQuery,
|
||||
setIsTesting,
|
||||
setTestResult,
|
||||
setTestError,
|
||||
}: {
|
||||
accessToken: string;
|
||||
testModel: string;
|
||||
testQuery: string;
|
||||
setIsTesting: (value: boolean) => void;
|
||||
setTestResult: (result: TestResult | null) => void;
|
||||
setTestError: (error: string | null) => void;
|
||||
}) => {
|
||||
if (!testQuery || !testModel || !accessToken) {
|
||||
NotificationManager.error("Please enter a query and select a model");
|
||||
|
|
@ -43,6 +45,7 @@ export const runSemanticFilterTest = async ({
|
|||
|
||||
setIsTesting(true);
|
||||
setTestResult(null);
|
||||
setTestError(null);
|
||||
|
||||
try {
|
||||
const { headers } = await testMCPSemanticFilter(accessToken, testModel, testQuery);
|
||||
|
|
@ -57,6 +60,8 @@ export const runSemanticFilterTest = async ({
|
|||
NotificationManager.success("Semantic filter test completed successfully");
|
||||
} catch (error) {
|
||||
console.error("Test failed:", error);
|
||||
const message = error instanceof Error && error.message ? error.message : "Failed to test semantic filter";
|
||||
setTestError(message);
|
||||
NotificationManager.error("Failed to test semantic filter");
|
||||
} finally {
|
||||
setIsTesting(false);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue