From 9c841075883354b8917a8196c3907bb283d0e86e Mon Sep 17 00:00:00 2001 From: tye-lightshifthealth Date: Wed, 25 Feb 2026 16:25:57 -0500 Subject: [PATCH] Fix: support file_search + vector_store with sync completion() and Anthropic - Pass tools into should_run_prompt_management_hooks and get_chat_completion_prompt in the sync completion path (main.py) so the vector store pre-call hook runs. - Add tools parameter to get_chat_completion_prompt in Logging and CustomLogger. - Implement sync get_chat_completion_prompt in VectorStorePreCallHook using pop_vector_stores_to_run and vector_stores.search() so file_search tools are stripped and context is injected before the request reaches Anthropic. - After the hook, set tools to None when the list is empty (match async behavior). Fixes: Unsupported tool type: file_search when using completion() with vector_store_ids and Anthropic models (e.g. https://github.com/BerriAI/litellm/issues/22124) Made-with: Cursor --- litellm/integrations/custom_logger.py | 1 + .../vector_store_pre_call_hook.py | 66 +++++++++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 3 + litellm/main.py | 14 +++- 4 files changed, 83 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index c244363e389..c213f28f0bb 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -216,6 +216,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_version: Optional[int] = None, ignore_prompt_manager_model: Optional[bool] = False, ignore_prompt_manager_optional_params: Optional[bool] = False, + tools: Optional[List[Dict]] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index c94b925ea21..0d345383b1b 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -41,6 +41,72 @@ class VectorStorePreCallHook(CustomLogger): def __init__(self): super().__init__() + def get_chat_completion_prompt( + self, + model: str, + messages: List[AllMessageValues], + non_default_params: dict, + prompt_id: Optional[str], + prompt_variables: Optional[dict], + dynamic_callback_params: StandardCallbackDynamicParams, + prompt_spec: Optional[PromptSpec] = None, + prompt_label: Optional[str] = None, + prompt_version: Optional[int] = None, + ignore_prompt_manager_model: Optional[bool] = False, + ignore_prompt_manager_optional_params: Optional[bool] = False, + tools: Optional[List[Dict]] = None, + ) -> Tuple[str, List[AllMessageValues], dict]: + """ + Perform vector store search and append results as context to messages (sync). + Used when completion() is called so file_search tools are handled before the request. + """ + try: + if litellm.vector_store_registry is None: + return model, messages, non_default_params + + vector_stores_to_run: List[LiteLLM_ManagedVectorStore] = ( + litellm.vector_store_registry.pop_vector_stores_to_run( + non_default_params=non_default_params, + tools=tools, + ) + ) + + if not vector_stores_to_run: + return model, messages, non_default_params + + query = self._extract_query_from_messages(messages) + if not query: + verbose_logger.debug( + "No query found in messages for vector store search" + ) + return model, messages, non_default_params + + modified_messages: List[AllMessageValues] = messages.copy() + + for vector_store_to_run in vector_stores_to_run: + vector_store_id = vector_store_to_run.get("vector_store_id", "") + custom_llm_provider = vector_store_to_run.get("custom_llm_provider") + litellm_params_for_vector_store = ( + vector_store_to_run.get("litellm_params", {}) or {} + ) + search_response = litellm.vector_stores.search( + **{ + "vector_store_id": vector_store_id, + "query": query, + "custom_llm_provider": custom_llm_provider, + **litellm_params_for_vector_store, + }, + ) + modified_messages = self._append_search_results_to_messages( + messages=modified_messages, search_response=search_response + ) + + return model, modified_messages, non_default_params + + except Exception as e: + verbose_logger.exception(f"Error in VectorStorePreCallHook: {str(e)}") + return model, messages, non_default_params + async def async_get_chat_completion_prompt( self, model: str, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0601e7e8455..fb8c3df5352 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -636,6 +636,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_management_logger: Optional[CustomLogger] = None, prompt_label: Optional[str] = None, prompt_version: Optional[int] = None, + tools: Optional[List[Dict]] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -645,6 +646,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id=prompt_id, prompt_spec=prompt_spec, dynamic_callback_params=self.standard_callback_dynamic_params, + tools=tools, ) ) @@ -663,6 +665,7 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_callback_params=self.standard_callback_dynamic_params, prompt_label=prompt_label, prompt_version=prompt_version, + tools=tools, ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/main.py b/litellm/main.py index 8b239c454f4..9b439a7da17 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1262,7 +1262,9 @@ def completion( # type: ignore # noqa: PLR0915 if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( litellm_logging_obj.should_run_prompt_management_hooks( - prompt_id=prompt_id, non_default_params=non_default_params + prompt_id=prompt_id, + non_default_params=non_default_params, + tools=tools, ) ): ( @@ -1277,7 +1279,17 @@ def completion( # type: ignore # noqa: PLR0915 prompt_variables=prompt_variables, prompt_label=kwargs.get("prompt_label", None), prompt_version=kwargs.get("prompt_version", None), + tools=tools, ) + ######################################################### + # if the chat completion logging hook removed all tools, + # set tools to None + # eg. in certain cases when users send vector stores as tools + # we don't want the tools to go to the upstream llm + # relevant issue: https://github.com/BerriAI/litellm/issues/11404 + ######################################################### + if tools is not None and len(tools) == 0: + tools = None ### LITELLM SYSTEM PROMPT ### if litellm_system_prompt: