initialize_from_config

litellm/constants.py

@@ -67,6 +67,17 @@ DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0)
 )
 
+# MCP Semantic Tool Filter Defaults
+DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL = str(
+    os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL", "text-embedding-3-small")
+)
+DEFAULT_MCP_SEMANTIC_FILTER_TOP_K = int(
+    os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10)
+)
+DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD = float(
+    os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3)
+)
+
 # Gemini model-specific minimal thinking budget constants
 DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH = int(
     os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1)

litellm/proxy/hooks/mcp_semantic_filter/hook.py

@@ -7,6 +7,11 @@ Reduces context window size and improves tool selection accuracy.
 from typing import TYPE_CHECKING, Any, Dict, Optional, Union
 
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import (
+    DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL,
+    DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD,
+    DEFAULT_MCP_SEMANTIC_FILTER_TOP_K,
+)
 from litellm.integrations.custom_logger import CustomLogger
 
 if TYPE_CHECKING:
@@ -15,6 +20,7 @@ if TYPE_CHECKING:
         SemanticMCPToolFilter,
     )
     from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.router import Router
 
 
 class SemanticToolFilterHook(CustomLogger):
@@ -77,6 +83,16 @@ class SemanticToolFilterHook(CustomLogger):
             verbose_proxy_logger.debug("No tools in request, skipping semantic filter")
             return None
         
+        # Skip filtering if tools are MCP references (type: "mcp")
+        # These will be expanded by the responses API handler
+        if isinstance(tools, list) and len(tools) > 0:
+            first_tool = tools[0]
+            if isinstance(first_tool, dict) and first_tool.get("type") == "mcp":
+                verbose_proxy_logger.debug(
+                    "Skipping semantic filter for MCP tool references (type: mcp)"
+                )
+                return None
+        
         # Check if messages are present
         messages = data.get("messages", [])
         if not messages:
@@ -127,3 +143,71 @@ class SemanticToolFilterHook(CustomLogger):
                 f"Semantic tool filter hook failed: {e}. Proceeding with all tools."
             )
             return None
+    
+    @staticmethod
+    def initialize_from_config(
+        config: Optional[Dict[str, Any]],
+        llm_router: Optional["Router"],
+    ) -> Optional["SemanticToolFilterHook"]:
+        """
+        Initialize semantic tool filter from proxy config.
+        
+        Args:
+            config: Proxy configuration dict (litellm_settings.mcp_semantic_tool_filter)
+            llm_router: LiteLLM router instance for embeddings
+            
+        Returns:
+            SemanticToolFilterHook instance if enabled, None otherwise
+        """
+        from litellm.proxy._experimental.mcp_server.semantic_tool_filter import (
+            SemanticMCPToolFilter,
+        )
+        if not config or not config.get("enabled", False):
+            verbose_proxy_logger.debug("Semantic tool filter not enabled in config")
+            return None
+        
+        if llm_router is None:
+            verbose_proxy_logger.warning(
+                "Cannot initialize semantic filter: llm_router is None"
+            )
+            return None
+        
+        try:
+            
+            embedding_model = config.get(
+                "embedding_model", DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL
+            )
+            top_k = config.get("top_k", DEFAULT_MCP_SEMANTIC_FILTER_TOP_K)
+            similarity_threshold = config.get(
+                "similarity_threshold", DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD
+            )
+            
+            semantic_filter = SemanticMCPToolFilter(
+                embedding_model=embedding_model,
+                litellm_router_instance=llm_router,
+                top_k=top_k,
+                similarity_threshold=similarity_threshold,
+                enabled=True,
+            )
+            
+            hook = SemanticToolFilterHook(semantic_filter)
+            
+            verbose_proxy_logger.info(
+                f"✅ MCP Semantic Tool Filter enabled: "
+                f"embedding_model={embedding_model}, top_k={top_k}, "
+                f"similarity_threshold={similarity_threshold}"
+            )
+            
+            return hook
+            
+        except ImportError as e:
+            verbose_proxy_logger.warning(
+                f"semantic-router not installed. Install with: "
+                f"pip install 'litellm[semantic-router]'. Error: {e}"
+            )
+            return None
+        except Exception as e:
+            verbose_proxy_logger.exception(
+                f"Failed to initialize MCP semantic tool filter: {e}"
+            )
+            return None

litellm/proxy/proxy_server.py

@@ -793,6 +793,15 @@ async def proxy_startup_event(app: FastAPI):  # noqa: PLR0915
         redis_usage_cache=redis_usage_cache,
     )
 
+    ## SEMANTIC TOOL FILTER ##
+    # Read litellm_settings from config for semantic filter initialization
+    _config = proxy_config.get_config_state()
+    _litellm_settings = _config.get("litellm_settings", {})
+    ProxyStartupEvent._initialize_semantic_tool_filter(
+        llm_router=llm_router,
+        litellm_settings=_litellm_settings,
+    )
+
     ## JWT AUTH ##
     ProxyStartupEvent._initialize_jwt_auth(
         general_settings=general_settings,
@@ -4741,6 +4750,24 @@ class ProxyStartupEvent:
             llm_router=llm_router, redis_usage_cache=redis_usage_cache
         )
 
+    @classmethod
+    def _initialize_semantic_tool_filter(
+        cls,
+        llm_router: Optional[Router],
+        litellm_settings: Dict[str, Any],
+    ):
+        """Initialize MCP semantic tool filter if configured"""
+        from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
+        
+        mcp_semantic_filter_config = litellm_settings.get("mcp_semantic_tool_filter", None)
+        hook = SemanticToolFilterHook.initialize_from_config(
+            config=mcp_semantic_filter_config,
+            llm_router=llm_router,
+        )
+        
+        if hook:
+            litellm.logging_callback_manager.add_litellm_callback(hook)
+
     @classmethod
     def _initialize_jwt_auth(
         cls,