mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
[Feat] UI - Add support for MCP Semantic Filtering on UI (#20454)
* feat: remove_callbacks_by_type * feat: _init_semantic_filter_settings_in_db * add MCP semantic filtering settings endpoints * v0 - semantic filter UI * fix endpint _init_semantic_filter_settings_in_db * refactor ui * add LITELLM_UI_ALLOW_HEADERS * ui fix * ui fix
This commit is contained in:
Ishaan Jaff
GitHub
parent
2a3843aa57
commit
15b717c300
13 changed files with 970 additions and 3 deletions
|
|
@ -81,6 +81,11 @@ MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH = int(
|
|||
os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)
|
||||
)
|
||||
|
||||
LITELLM_UI_ALLOW_HEADERS = [
|
||||
"x-litellm-semantic-filter",
|
||||
"x-litellm-semantic-filter-tools",
|
||||
]
|
||||
|
||||
# Gemini model-specific minimal thinking budget constants
|
||||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH = int(
|
||||
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1)
|
||||
|
|
|
|||
|
|
@ -114,6 +114,27 @@ class LoggingCallbackManager:
|
|||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
|
||||
def remove_callbacks_by_type(self, callback_list, callback_type):
|
||||
"""
|
||||
Remove all callbacks of a specific type from a callback list.
|
||||
|
||||
Args:
|
||||
callback_list: The list to remove callbacks from (e.g., litellm.callbacks)
|
||||
callback_type: The class type to match (e.g., SemanticToolFilterHook)
|
||||
|
||||
Example:
|
||||
litellm.logging_callback_manager.remove_callbacks_by_type(
|
||||
litellm.callbacks, SemanticToolFilterHook
|
||||
)
|
||||
"""
|
||||
if not isinstance(callback_list, list):
|
||||
return
|
||||
|
||||
remove_list = [c for c in callback_list if isinstance(c, callback_type)]
|
||||
|
||||
for c in remove_list:
|
||||
callback_list.remove(c)
|
||||
|
||||
def _add_string_callback_to_list(
|
||||
self, callback: str, parent_list: List[Union[CustomLogger, Callable, str]]
|
||||
):
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
|
||||
async def route_a2a_agent_request(data: dict, route_type: str) -> Optional[Any]:
|
||||
def route_a2a_agent_request(data: dict, route_type: str) -> Optional[Any]:
|
||||
"""
|
||||
Route A2A agent requests directly to litellm with injected API base.
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ from litellm.constants import (
|
|||
DEFAULT_SLACK_ALERTING_THRESHOLD,
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS,
|
||||
LITELLM_SETTINGS_SAFE_DB_OVERRIDES,
|
||||
LITELLM_UI_ALLOW_HEADERS,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_init_custom_logger_compatible_class,
|
||||
|
|
@ -1214,6 +1215,7 @@ app.add_middleware(
|
|||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
expose_headers=LITELLM_UI_ALLOW_HEADERS,
|
||||
)
|
||||
|
||||
app.add_middleware(PrometheusAuthMiddleware)
|
||||
|
|
@ -3920,6 +3922,93 @@ class ProxyConfig:
|
|||
prisma_client=prisma_client, proxy_config=self
|
||||
)
|
||||
|
||||
if self._should_load_db_object(object_type="semantic_filter_settings"):
|
||||
await self._init_semantic_filter_settings_in_db(
|
||||
prisma_client=prisma_client
|
||||
)
|
||||
|
||||
async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
Initialize MCP semantic filter settings from database.
|
||||
Called periodically (approximately every 10 seconds) by background task to hot-reload settings across all pods.
|
||||
"""
|
||||
import json
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
|
||||
|
||||
try:
|
||||
# Load litellm_settings from DB
|
||||
config_record = await prisma_client.db.litellm_config.find_unique(
|
||||
where={"param_name": "litellm_settings"}
|
||||
)
|
||||
|
||||
if config_record is None or config_record.param_value is None:
|
||||
return
|
||||
|
||||
litellm_settings = config_record.param_value
|
||||
if isinstance(litellm_settings, str):
|
||||
litellm_settings = json.loads(litellm_settings)
|
||||
|
||||
mcp_semantic_filter_config = litellm_settings.get(
|
||||
"mcp_semantic_tool_filter", None
|
||||
)
|
||||
|
||||
if mcp_semantic_filter_config is None:
|
||||
return
|
||||
|
||||
# Check if settings have changed (compare with in-memory state)
|
||||
if hasattr(self, "_last_semantic_filter_config"):
|
||||
if self._last_semantic_filter_config == mcp_semantic_filter_config:
|
||||
# If hook is missing or router isn't built yet, reinitialize anyway
|
||||
active_hooks = (
|
||||
litellm.logging_callback_manager.get_custom_loggers_for_type(
|
||||
SemanticToolFilterHook
|
||||
)
|
||||
)
|
||||
if active_hooks:
|
||||
for active_hook in active_hooks:
|
||||
if isinstance(active_hook, SemanticToolFilterHook):
|
||||
if (
|
||||
active_hook.filter is not None
|
||||
and active_hook.filter.tool_router is not None
|
||||
):
|
||||
verbose_proxy_logger.debug(
|
||||
"Semantic filter settings unchanged, skipping reinitialization"
|
||||
)
|
||||
return
|
||||
verbose_proxy_logger.info(
|
||||
"Semantic filter settings unchanged, but hook is missing or uninitialized. Reinitializing."
|
||||
)
|
||||
|
||||
# Remove old hooks using logging callback manager
|
||||
litellm.logging_callback_manager.remove_callbacks_by_type(
|
||||
litellm.callbacks, SemanticToolFilterHook
|
||||
)
|
||||
|
||||
# Initialize new hook if enabled
|
||||
if mcp_semantic_filter_config.get("enabled", False):
|
||||
global llm_router
|
||||
hook = await SemanticToolFilterHook.initialize_from_config(
|
||||
config=mcp_semantic_filter_config,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if hook:
|
||||
litellm.logging_callback_manager.add_litellm_callback(hook)
|
||||
verbose_proxy_logger.info(
|
||||
"MCP Semantic Filter reinitialized from DB"
|
||||
)
|
||||
else:
|
||||
verbose_proxy_logger.info("MCP Semantic Filter disabled")
|
||||
|
||||
# Store current config for comparison next time
|
||||
self._last_semantic_filter_config = mcp_semantic_filter_config.copy()
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
f"Error initializing semantic filter settings from DB: {e}"
|
||||
)
|
||||
|
||||
async def _init_sso_settings_in_db(self, prisma_client: PrismaClient):
|
||||
"""
|
||||
Initialize SSO settings from database into the router on startup.
|
||||
|
|
|
|||
|
|
@ -332,7 +332,10 @@ async def route_request(
|
|||
route_a2a_agent_request,
|
||||
)
|
||||
|
||||
return await route_a2a_agent_request(data, route_type)
|
||||
result = route_a2a_agent_request(data, route_type)
|
||||
if result is not None:
|
||||
return result
|
||||
# Fall through to raise exception below if result is None
|
||||
|
||||
elif user_model is not None:
|
||||
return getattr(litellm, f"{route_type}")(**data)
|
||||
|
|
|
|||
|
|
@ -98,6 +98,40 @@ ALLOWED_UI_SETTINGS_FIELDS = {
|
|||
}
|
||||
|
||||
|
||||
class MCPSemanticFilterSettings(BaseModel):
|
||||
"""Configuration for MCP Semantic Tool Filter"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable semantic filtering of MCP tools based on query relevance",
|
||||
)
|
||||
|
||||
embedding_model: str = Field(
|
||||
default="text-embedding-3-small",
|
||||
description="Embedding model to use for semantic similarity (e.g., 'text-embedding-3-small', 'text-embedding-ada-002')",
|
||||
)
|
||||
|
||||
top_k: int = Field(
|
||||
default=10,
|
||||
description="Number of most relevant tools to return",
|
||||
ge=1,
|
||||
le=100,
|
||||
)
|
||||
|
||||
similarity_threshold: float = Field(
|
||||
default=0.3,
|
||||
description="Minimum similarity score for tool inclusion (0.0 to 1.0, where 1.0 = exact match)",
|
||||
ge=0.0,
|
||||
le=1.0,
|
||||
)
|
||||
|
||||
|
||||
class MCPSemanticFilterSettingsResponse(SettingsResponse):
|
||||
"""Response model for MCP semantic filter settings"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/allowed_ips",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
|
|
@ -325,7 +359,7 @@ async def update_default_team_member_budget(
|
|||
|
||||
|
||||
async def _update_litellm_setting(
|
||||
settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams],
|
||||
settings: Union[DefaultInternalUserParams, DefaultTeamSSOParams, MCPSemanticFilterSettings],
|
||||
settings_key: str,
|
||||
in_memory_var: Any,
|
||||
success_message: str,
|
||||
|
|
@ -769,6 +803,70 @@ async def update_ui_theme_settings(theme_config: UIThemeConfig):
|
|||
}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/get/mcp_semantic_filter_settings",
|
||||
tags=["Settings"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=MCPSemanticFilterSettingsResponse,
|
||||
)
|
||||
async def get_mcp_semantic_filter_settings(
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Get MCP semantic filter configuration.
|
||||
Returns current settings for semantic tool filtering.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_config
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "Database not connected. Please connect a database."},
|
||||
)
|
||||
|
||||
config = await proxy_config.get_config()
|
||||
|
||||
return await _get_settings_with_schema(
|
||||
settings_key="mcp_semantic_tool_filter",
|
||||
settings_class=MCPSemanticFilterSettings,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/update/mcp_semantic_filter_settings",
|
||||
tags=["Settings"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def update_mcp_semantic_filter_settings(
|
||||
settings: MCPSemanticFilterSettings,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
Update MCP semantic filter settings in database.
|
||||
Settings will be picked up by all pods within approximately 10 seconds via background polling.
|
||||
"""
|
||||
result = await _update_litellm_setting(
|
||||
settings=settings,
|
||||
settings_key="mcp_semantic_tool_filter",
|
||||
in_memory_var=None,
|
||||
success_message="MCP Semantic Filter settings updated successfully. Changes will be applied across all pods within 10 seconds.",
|
||||
)
|
||||
try:
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_config
|
||||
|
||||
if prisma_client is not None:
|
||||
await proxy_config._init_semantic_filter_settings_in_db(
|
||||
prisma_client=prisma_client
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.warning(
|
||||
f"Failed to reinitialize MCP semantic filter settings immediately: {e}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.get(
|
||||
"/in_product_nudges",
|
||||
tags=["UI Settings"],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
import { getMCPSemanticFilterSettings } from "@/components/networking";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { createQueryKeys } from "../common/queryKeysFactory";
|
||||
import useAuthorized from "../useAuthorized";
|
||||
|
||||
const mcpSemanticFilterSettingsKeys = createQueryKeys(
|
||||
"mcpSemanticFilterSettings"
|
||||
);
|
||||
|
||||
export const useMCPSemanticFilterSettings = () => {
|
||||
const { accessToken } = useAuthorized();
|
||||
return useQuery<Record<string, any>>({
|
||||
queryKey: mcpSemanticFilterSettingsKeys.list({}),
|
||||
queryFn: async () => await getMCPSemanticFilterSettings(accessToken),
|
||||
enabled: !!accessToken,
|
||||
staleTime: 60 * 60 * 1000, // 1 hour
|
||||
gcTime: 60 * 60 * 1000, // 1 hour
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import { updateMCPSemanticFilterSettings } from "@/components/networking";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { createQueryKeys } from "../common/queryKeysFactory";
|
||||
|
||||
const mcpSemanticFilterSettingsKeys = createQueryKeys(
|
||||
"mcpSemanticFilterSettings"
|
||||
);
|
||||
|
||||
export const useUpdateMCPSemanticFilterSettings = (accessToken: string) => {
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
return useMutation({
|
||||
mutationFn: async (settings: Record<string, any>) => {
|
||||
if (!accessToken) {
|
||||
throw new Error("Access token is required");
|
||||
}
|
||||
return updateMCPSemanticFilterSettings(accessToken, settings);
|
||||
},
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: mcpSemanticFilterSettingsKeys.all,
|
||||
});
|
||||
},
|
||||
});
|
||||
};
|
||||
|
|
@ -0,0 +1,312 @@
|
|||
"use client";
|
||||
|
||||
import { useMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useMCPSemanticFilterSettings";
|
||||
import { useUpdateMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useUpdateMCPSemanticFilterSettings";
|
||||
import NotificationManager from "@/components/molecules/notifications_manager";
|
||||
import {
|
||||
Alert,
|
||||
Button,
|
||||
Card,
|
||||
Col,
|
||||
Form,
|
||||
InputNumber,
|
||||
Row,
|
||||
Select,
|
||||
Skeleton,
|
||||
Slider,
|
||||
Space,
|
||||
Switch,
|
||||
Typography,
|
||||
Tooltip,
|
||||
} from "antd";
|
||||
import { QuestionCircleOutlined, CheckCircleOutlined, SaveOutlined } from "@ant-design/icons";
|
||||
import { useEffect, useState } from "react";
|
||||
import { fetchAvailableModels, ModelGroup } from "@/components/playground/llm_calls/fetch_models";
|
||||
import MCPSemanticFilterTestPanel from "./MCPSemanticFilterTestPanel";
|
||||
import { getCurlCommand, runSemanticFilterTest, TestResult } from "./semanticFilterTestUtils";
|
||||
|
||||
interface MCPSemanticFilterSettingsProps {
|
||||
accessToken: string | null;
|
||||
}
|
||||
|
||||
export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFilterSettingsProps) {
|
||||
const { data, isLoading, isError, error } = useMCPSemanticFilterSettings();
|
||||
const {
|
||||
mutate: updateSettings,
|
||||
isPending: isUpdating,
|
||||
error: updateError,
|
||||
} = useUpdateMCPSemanticFilterSettings(accessToken || "");
|
||||
const [form] = Form.useForm();
|
||||
const [saveSuccess, setSaveSuccess] = useState(false);
|
||||
const [isDirty, setIsDirty] = useState(false);
|
||||
const [embeddingModels, setEmbeddingModels] = useState<ModelGroup[]>([]);
|
||||
const [loadingModels, setLoadingModels] = useState(true);
|
||||
|
||||
// Test section state
|
||||
const [testQuery, setTestQuery] = useState("");
|
||||
const [testModel, setTestModel] = useState<string>("gpt-4o");
|
||||
const [testResult, setTestResult] = useState<TestResult | null>(null);
|
||||
const [isTesting, setIsTesting] = useState(false);
|
||||
const [showCurl, setShowCurl] = useState(false);
|
||||
|
||||
const schema = data?.field_schema;
|
||||
const values = data?.values ?? {};
|
||||
|
||||
useEffect(() => {
|
||||
const loadEmbeddingModels = async () => {
|
||||
if (!accessToken) return;
|
||||
try {
|
||||
setLoadingModels(true);
|
||||
const models = await fetchAvailableModels(accessToken);
|
||||
const embeddingOnly = models.filter((model) => model.mode === "embedding");
|
||||
setEmbeddingModels(embeddingOnly);
|
||||
} catch (error) {
|
||||
console.error("Error fetching embedding models:", error);
|
||||
} finally {
|
||||
setLoadingModels(false);
|
||||
}
|
||||
};
|
||||
|
||||
loadEmbeddingModels();
|
||||
}, [accessToken]);
|
||||
|
||||
useEffect(() => {
|
||||
if (values) {
|
||||
form.setFieldsValue({
|
||||
enabled: values.enabled ?? false,
|
||||
embedding_model: values.embedding_model ?? "text-embedding-3-small",
|
||||
top_k: values.top_k ?? 10,
|
||||
similarity_threshold: values.similarity_threshold ?? 0.3,
|
||||
});
|
||||
setIsDirty(false);
|
||||
}
|
||||
}, [values, form]);
|
||||
|
||||
const handleSave = async () => {
|
||||
try {
|
||||
const formValues = await form.validateFields();
|
||||
updateSettings(formValues, {
|
||||
onSuccess: () => {
|
||||
setIsDirty(false);
|
||||
setSaveSuccess(true);
|
||||
setTimeout(() => setSaveSuccess(false), 3000);
|
||||
NotificationManager.success(
|
||||
"Settings updated successfully. Changes will be applied across all pods within 10 seconds."
|
||||
);
|
||||
},
|
||||
onError: (error) => {
|
||||
NotificationManager.fromBackend(error);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Form validation failed:", error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleTest = async () => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
await runSemanticFilterTest({
|
||||
accessToken,
|
||||
testModel,
|
||||
testQuery,
|
||||
setIsTesting,
|
||||
setTestResult,
|
||||
});
|
||||
};
|
||||
|
||||
if (!accessToken) {
|
||||
return (
|
||||
<div className="p-6 text-center text-gray-500">
|
||||
Please log in to configure semantic filter settings.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div style={{ width: "100%" }}>
|
||||
{isLoading ? (
|
||||
<Skeleton active />
|
||||
) : isError ? (
|
||||
<Alert
|
||||
type="error"
|
||||
message="Could not load MCP Semantic Filter settings"
|
||||
description={error instanceof Error ? error.message : undefined}
|
||||
style={{ marginBottom: 24 }}
|
||||
/>
|
||||
) : (
|
||||
<>
|
||||
<Alert
|
||||
type="info"
|
||||
message="Semantic Tool Filtering"
|
||||
description="Filter MCP tools semantically based on query relevance. This reduces context window size and improves tool selection accuracy. Click 'Save Settings' to apply changes across all pods (takes effect within 10 seconds)."
|
||||
showIcon
|
||||
style={{ marginBottom: 24 }}
|
||||
/>
|
||||
|
||||
{saveSuccess && (
|
||||
<Alert
|
||||
type="success"
|
||||
message="Settings saved successfully"
|
||||
icon={<CheckCircleOutlined />}
|
||||
showIcon
|
||||
closable
|
||||
style={{ marginBottom: 16 }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{updateError && (
|
||||
<Alert
|
||||
type="error"
|
||||
message="Could not update settings"
|
||||
description={
|
||||
updateError instanceof Error ? updateError.message : undefined
|
||||
}
|
||||
style={{ marginBottom: 16 }}
|
||||
/>
|
||||
)}
|
||||
|
||||
<Row gutter={24}>
|
||||
{/* Left Column - Settings */}
|
||||
<Col xs={24} lg={12}>
|
||||
<Form
|
||||
form={form}
|
||||
layout="vertical"
|
||||
disabled={isUpdating}
|
||||
onValuesChange={() => {
|
||||
setIsDirty(true);
|
||||
}}
|
||||
>
|
||||
<Card style={{ marginBottom: 16 }}>
|
||||
<Form.Item
|
||||
name="enabled"
|
||||
label={
|
||||
<Space>
|
||||
<Typography.Text strong>Enable Semantic Filtering</Typography.Text>
|
||||
<Tooltip title="When enabled, only the most relevant MCP tools will be included in requests based on semantic similarity">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</Space>
|
||||
}
|
||||
valuePropName="checked"
|
||||
>
|
||||
<Switch disabled={isUpdating} />
|
||||
</Form.Item>
|
||||
|
||||
<Typography.Text type="secondary" style={{ display: "block", marginTop: -16, marginBottom: 16 }}>
|
||||
{schema?.properties?.enabled?.description}
|
||||
</Typography.Text>
|
||||
</Card>
|
||||
|
||||
<Card title="Configuration" style={{ marginBottom: 16 }}>
|
||||
<Form.Item
|
||||
name="embedding_model"
|
||||
label={
|
||||
<Space>
|
||||
<Typography.Text strong>Embedding Model</Typography.Text>
|
||||
<Tooltip title="The model used to generate embeddings for semantic matching">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</Space>
|
||||
}
|
||||
>
|
||||
<Select
|
||||
options={embeddingModels.map((model) => ({
|
||||
label: model.model_group,
|
||||
value: model.model_group,
|
||||
}))}
|
||||
placeholder={loadingModels ? "Loading models..." : "Select embedding model"}
|
||||
showSearch
|
||||
disabled={isUpdating || loadingModels}
|
||||
loading={loadingModels}
|
||||
notFoundContent={
|
||||
loadingModels ? "Loading..." : "No embedding models available"
|
||||
}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="top_k"
|
||||
label={
|
||||
<Space>
|
||||
<Typography.Text strong>Top K Results</Typography.Text>
|
||||
<Tooltip title="Maximum number of tools to return after filtering">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</Space>
|
||||
}
|
||||
>
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={100}
|
||||
style={{ width: "100%" }}
|
||||
disabled={isUpdating}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="similarity_threshold"
|
||||
label={
|
||||
<Space>
|
||||
<Typography.Text strong>Similarity Threshold</Typography.Text>
|
||||
<Tooltip title="Minimum similarity score (0-1) for a tool to be included">
|
||||
<QuestionCircleOutlined style={{ color: "#8c8c8c" }} />
|
||||
</Tooltip>
|
||||
</Space>
|
||||
}
|
||||
>
|
||||
<Slider
|
||||
min={0}
|
||||
max={1}
|
||||
step={0.05}
|
||||
marks={{
|
||||
0: "0.0",
|
||||
0.3: "0.3",
|
||||
0.5: "0.5",
|
||||
0.7: "0.7",
|
||||
1: "1.0",
|
||||
}}
|
||||
disabled={isUpdating}
|
||||
/>
|
||||
</Form.Item>
|
||||
</Card>
|
||||
|
||||
<div style={{ display: "flex", justifyContent: "flex-end", gap: 8 }}>
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<SaveOutlined />}
|
||||
onClick={handleSave}
|
||||
loading={isUpdating}
|
||||
disabled={!isDirty}
|
||||
>
|
||||
Save Settings
|
||||
</Button>
|
||||
</div>
|
||||
</Form>
|
||||
</Col>
|
||||
|
||||
{/* Right Column - Test Configuration */}
|
||||
<Col xs={24} lg={12}>
|
||||
<MCPSemanticFilterTestPanel
|
||||
accessToken={accessToken}
|
||||
testQuery={testQuery}
|
||||
setTestQuery={setTestQuery}
|
||||
testModel={testModel}
|
||||
setTestModel={setTestModel}
|
||||
isTesting={isTesting}
|
||||
onTest={handleTest}
|
||||
filterEnabled={!!values.enabled}
|
||||
testResult={testResult}
|
||||
showCurl={showCurl}
|
||||
setShowCurl={setShowCurl}
|
||||
curlCommand={getCurlCommand(testModel, testQuery)}
|
||||
/>
|
||||
</Col>
|
||||
</Row>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
import { CodeOutlined, PlayCircleOutlined } from "@ant-design/icons";
|
||||
import { Alert, Button, Card, Input, Space, Tabs, Typography } from "antd";
|
||||
import ModelSelector from "@/components/common_components/ModelSelector";
|
||||
import { TestResult } from "./semanticFilterTestUtils";
|
||||
|
||||
interface MCPSemanticFilterTestPanelProps {
|
||||
accessToken: string | null;
|
||||
testQuery: string;
|
||||
setTestQuery: (value: string) => void;
|
||||
testModel: string;
|
||||
setTestModel: (value: string) => void;
|
||||
isTesting: boolean;
|
||||
onTest: () => void;
|
||||
filterEnabled: boolean;
|
||||
testResult: TestResult | null;
|
||||
curlCommand: string;
|
||||
}
|
||||
|
||||
export default function MCPSemanticFilterTestPanel({
|
||||
accessToken,
|
||||
testQuery,
|
||||
setTestQuery,
|
||||
testModel,
|
||||
setTestModel,
|
||||
isTesting,
|
||||
onTest,
|
||||
filterEnabled,
|
||||
testResult,
|
||||
curlCommand,
|
||||
}: MCPSemanticFilterTestPanelProps) {
|
||||
return (
|
||||
<Card title="Test Configuration" style={{ marginBottom: 16 }}>
|
||||
<Tabs
|
||||
defaultActiveKey="test"
|
||||
items={[
|
||||
{
|
||||
key: "test",
|
||||
label: "Test",
|
||||
children: (
|
||||
<Space direction="vertical" style={{ width: "100%" }} size="large">
|
||||
<div>
|
||||
<Typography.Text strong style={{ display: "block", marginBottom: 8 }}>
|
||||
<PlayCircleOutlined /> Test Query
|
||||
</Typography.Text>
|
||||
<Input.TextArea
|
||||
placeholder="Enter a test query to see which tools would be selected..."
|
||||
value={testQuery}
|
||||
onChange={(e) => setTestQuery(e.target.value)}
|
||||
rows={4}
|
||||
disabled={isTesting}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<ModelSelector
|
||||
accessToken={accessToken || ""}
|
||||
value={testModel}
|
||||
onChange={setTestModel}
|
||||
disabled={isTesting}
|
||||
showLabel={true}
|
||||
labelText="Select Model"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<PlayCircleOutlined />}
|
||||
onClick={onTest}
|
||||
loading={isTesting}
|
||||
disabled={!testQuery || !testModel || !filterEnabled}
|
||||
block
|
||||
>
|
||||
Test Filter
|
||||
</Button>
|
||||
|
||||
{!filterEnabled && (
|
||||
<Alert
|
||||
type="warning"
|
||||
message="Semantic filtering is disabled"
|
||||
description="Enable semantic filtering and save settings to test the filter."
|
||||
showIcon
|
||||
/>
|
||||
)}
|
||||
|
||||
{testResult && (
|
||||
<div>
|
||||
<Typography.Title level={5}>Results</Typography.Title>
|
||||
<Alert
|
||||
type="success"
|
||||
message={`${testResult.selectedTools} tools selected`}
|
||||
description={`Filtered from ${testResult.totalTools} available tools`}
|
||||
showIcon
|
||||
style={{ marginBottom: 16 }}
|
||||
/>
|
||||
<div>
|
||||
<Typography.Text strong style={{ display: "block", marginBottom: 8 }}>
|
||||
Selected Tools:
|
||||
</Typography.Text>
|
||||
<ul style={{ paddingLeft: 20, margin: 0 }}>
|
||||
{testResult.tools.map((tool, index) => (
|
||||
<li key={index} style={{ marginBottom: 4 }}>
|
||||
<Typography.Text>{tool}</Typography.Text>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Space>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "api",
|
||||
label: "API Usage",
|
||||
children: (
|
||||
<div>
|
||||
<Space style={{ marginBottom: 8 }}>
|
||||
<CodeOutlined />
|
||||
<Typography.Text strong>API Usage</Typography.Text>
|
||||
</Space>
|
||||
<Typography.Text type="secondary" style={{ display: "block", marginBottom: 8 }}>
|
||||
Use this curl command to test the semantic filter with your current configuration.
|
||||
</Typography.Text>
|
||||
<Typography.Text strong style={{ display: "block", marginBottom: 8 }}>
|
||||
Response headers to check:
|
||||
</Typography.Text>
|
||||
<ul style={{ paddingLeft: 20, margin: "0 0 12px 0" }}>
|
||||
<li>
|
||||
<Typography.Text>
|
||||
x-litellm-semantic-filter: shows total tools → selected tools
|
||||
</Typography.Text>
|
||||
<Typography.Text type="secondary" style={{ display: "block" }}>
|
||||
Example: 10→3
|
||||
</Typography.Text>
|
||||
</li>
|
||||
<li>
|
||||
<Typography.Text>
|
||||
x-litellm-semantic-filter-tools: CSV of selected tool names
|
||||
</Typography.Text>
|
||||
<Typography.Text type="secondary" style={{ display: "block" }}>
|
||||
Example: wikipedia-fetch,github-search,slack-post
|
||||
</Typography.Text>
|
||||
</li>
|
||||
</ul>
|
||||
<pre
|
||||
style={{
|
||||
background: "#f5f5f5",
|
||||
padding: 12,
|
||||
borderRadius: 4,
|
||||
overflow: "auto",
|
||||
fontSize: 12,
|
||||
margin: 0,
|
||||
}}
|
||||
>
|
||||
{curlCommand}
|
||||
</pre>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
import NotificationManager from "@/components/molecules/notifications_manager";
|
||||
import { testMCPSemanticFilter } from "@/components/networking";
|
||||
|
||||
export interface TestResult {
|
||||
totalTools: number;
|
||||
selectedTools: number;
|
||||
tools: string[];
|
||||
}
|
||||
|
||||
interface FilterHeaders {
|
||||
filter: string | null;
|
||||
tools: string | null;
|
||||
}
|
||||
|
||||
const parseFilterHeaders = (headers: FilterHeaders): TestResult | null => {
|
||||
if (!headers.filter) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [total, selected] = headers.filter.split("->").map(Number);
|
||||
const tools = headers.tools
|
||||
? headers.tools.split(",").map((name) => name.trim())
|
||||
: [];
|
||||
|
||||
return { totalTools: total, selectedTools: selected, tools };
|
||||
};
|
||||
|
||||
export const runSemanticFilterTest = async ({
|
||||
accessToken,
|
||||
testModel,
|
||||
testQuery,
|
||||
setIsTesting,
|
||||
setTestResult,
|
||||
}: {
|
||||
accessToken: string;
|
||||
testModel: string;
|
||||
testQuery: string;
|
||||
setIsTesting: (value: boolean) => void;
|
||||
setTestResult: (result: TestResult | null) => void;
|
||||
}) => {
|
||||
if (!testQuery || !testModel || !accessToken) {
|
||||
NotificationManager.error("Please enter a query and select a model");
|
||||
return;
|
||||
}
|
||||
|
||||
setIsTesting(true);
|
||||
setTestResult(null);
|
||||
|
||||
try {
|
||||
const { headers } = await testMCPSemanticFilter(
|
||||
accessToken,
|
||||
testModel,
|
||||
testQuery
|
||||
);
|
||||
const parsedResult = parseFilterHeaders(headers);
|
||||
|
||||
if (!parsedResult) {
|
||||
NotificationManager.warning(
|
||||
"Semantic filter is not enabled or no tools were filtered"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
setTestResult(parsedResult);
|
||||
NotificationManager.success("Semantic filter test completed successfully");
|
||||
} catch (error) {
|
||||
console.error("Test failed:", error);
|
||||
NotificationManager.error("Failed to test semantic filter");
|
||||
} finally {
|
||||
setIsTesting(false);
|
||||
}
|
||||
};
|
||||
|
||||
export const getCurlCommand = (testModel: string, testQuery: string) =>
|
||||
`curl --location 'http://localhost:4000/v1/responses' \\
|
||||
--header 'Content-Type: application/json' \\
|
||||
--header 'Authorization: Bearer sk-1234' \\
|
||||
--data '{
|
||||
"model": "${testModel}",
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "${testQuery || "Your query here"}",
|
||||
"type": "message"
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "mcp",
|
||||
"server_url": "litellm_proxy",
|
||||
"require_approval": "never"
|
||||
}
|
||||
],
|
||||
"tool_choice": "required"
|
||||
}'`;
|
||||
|
|
@ -13,6 +13,7 @@ import MCPConnect from "./mcp_connect";
|
|||
import { mcpServerColumns } from "./mcp_server_columns";
|
||||
import { MCPServerView } from "./mcp_server_view";
|
||||
import { MCPServer, MCPServerProps, Team } from "./types";
|
||||
import MCPSemanticFilterSettings from "../Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings";
|
||||
|
||||
const { Text: AntdText, Title: AntdTitle } = Typography;
|
||||
const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state";
|
||||
|
|
@ -302,6 +303,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID })
|
|||
<div className="flex">
|
||||
<Tab>All Servers</Tab>
|
||||
<Tab>Connect</Tab>
|
||||
<Tab>Semantic Filter</Tab>
|
||||
</div>
|
||||
</TabList>
|
||||
<TabPanels>
|
||||
|
|
@ -390,6 +392,9 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID })
|
|||
<TabPanel>
|
||||
<MCPConnect />
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<MCPSemanticFilterSettings accessToken={accessToken} />
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -5398,6 +5398,137 @@ export const updateUISettings = async (accessToken: string, settings: any) => {
|
|||
}
|
||||
};
|
||||
|
||||
export const getMCPSemanticFilterSettings = async (accessToken: string) => {
|
||||
/**
|
||||
* Get MCP semantic filter configuration
|
||||
*/
|
||||
try {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/get/mcp_semantic_filter_settings`
|
||||
: `/get/mcp_semantic_filter_settings`;
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
const errorMessage = deriveErrorMessage(errorData);
|
||||
handleError(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to get MCP semantic filter settings:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const updateMCPSemanticFilterSettings = async (
|
||||
accessToken: string,
|
||||
settings: Record<string, any>
|
||||
) => {
|
||||
/**
|
||||
* Update MCP semantic filter settings
|
||||
* Settings will be applied across all pods within 10 seconds
|
||||
*/
|
||||
try {
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/update/mcp_semantic_filter_settings`
|
||||
: `/update/mcp_semantic_filter_settings`;
|
||||
const response = await fetch(url, {
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(settings),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
const errorMessage = deriveErrorMessage(errorData);
|
||||
handleError(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to update MCP semantic filter settings:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const testMCPSemanticFilter = async (
|
||||
accessToken: string,
|
||||
model: string,
|
||||
query: string
|
||||
) => {
|
||||
/**
|
||||
* Test MCP semantic filter by making a responses API call
|
||||
* Returns both the response data and headers containing filter information
|
||||
*/
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/v1/responses` : `/v1/responses`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
content: query,
|
||||
type: "message",
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
type: "mcp",
|
||||
server_url: "litellm_proxy",
|
||||
require_approval: "never",
|
||||
},
|
||||
],
|
||||
tool_choice: "required",
|
||||
}),
|
||||
});
|
||||
|
||||
// Extract headers before checking response status
|
||||
const filterHeader = response.headers.get("x-litellm-semantic-filter");
|
||||
const toolsHeader = response.headers.get("x-litellm-semantic-filter-tools");
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
const errorMessage = deriveErrorMessage(errorData);
|
||||
handleError(errorMessage);
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
// Return both data and headers
|
||||
return {
|
||||
data,
|
||||
headers: {
|
||||
filter: filterHeader,
|
||||
tools: toolsHeader,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Failed to test MCP semantic filter:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const getGuardrailsList = async (accessToken: string) => {
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/v2/guardrails/list` : `/v2/guardrails/list`;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue