From a1514efa210c60c00809b21d2906503b0c452cc8 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Mon, 27 Jul 2026 17:07:58 +0200 Subject: [PATCH 1/6] fix(vector_stores): S3 Vectors search router bypass + rag query config drop + UI error swallow --- .../azure_ai/vector_stores/transformation.py | 2 + .../base_llm/vector_store/transformation.py | 4 + .../bedrock/vector_stores/transformation.py | 2 + litellm/llms/custom_httpx/llm_http_handler.py | 7 + .../gemini/vector_stores/transformation.py | 2 + .../milvus/vector_stores/transformation.py | 2 + .../openai/vector_stores/transformation.py | 2 + .../pg_vector/vector_stores/transformation.py | 2 + .../ragflow/vector_stores/transformation.py | 2 + .../vector_stores/transformation.py | 32 ++- .../vector_stores/rag_api/transformation.py | 2 + .../search_api/transformation.py | 2 + litellm/proxy/rag_endpoints/endpoints.py | 14 ++ litellm/rag/main.py | 14 +- litellm/router.py | 8 + litellm/vector_stores/main.py | 9 +- .../test_s3_vectors_transformation.py | 189 +++++++++++++++++- .../proxy/rag_endpoints/test_rag_endpoints.py | 103 ++++++++++ tests/test_litellm/rag/test_main.py | 90 +++++++++ tests/test_litellm/test_router.py | 55 +++++ tests/test_litellm/vector_stores/test_main.py | 77 +++++++ .../_components/VectorStoreTester.test.tsx | 25 ++- .../_components/VectorStoreTester.tsx | 8 +- .../src/components/networking.tsx | 2 +- 24 files changed, 628 insertions(+), 27 deletions(-) create mode 100644 tests/test_litellm/vector_stores/test_main.py diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index da6a4a93cd8..bd3eaeee989 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -92,6 +93,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index b222e3dd160..9a0e401b527 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -16,6 +16,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -56,6 +57,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: pass @@ -68,6 +70,7 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """ Optional async version of transform_search_vector_store_request. @@ -83,6 +86,7 @@ class BaseVectorStoreConfig: litellm_logging_obj=litellm_logging_obj, litellm_params=litellm_params, extra_body=extra_body, + router=router, ) @abstractmethod diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index c1b124caec1..7a6a0eb6d84 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -27,6 +27,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -196,6 +197,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: if isinstance(query, list): query = " ".join(query) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..ec701fbe87e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -167,6 +167,7 @@ if TYPE_CHECKING: AnthropicMessagesStreamingResponse, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.router import Router from litellm.types.llms.openai_evals import ( CancelEvalResponse, CancelRunResponse, @@ -9409,6 +9410,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + router: Optional["Router"] = None, ) -> VectorStoreSearchResponse: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( @@ -9443,6 +9445,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) else: ( @@ -9456,6 +9459,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Dict[str, Any] = dict(litellm_params) all_optional_params.update(vector_store_search_optional_params or {}) @@ -9507,6 +9511,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + router: Optional["Router"] = None, ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: if _is_async: return self.async_vector_store_search_handler( @@ -9521,6 +9526,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + router=router, ) if client is None or not isinstance(client, HTTPHandler): @@ -9551,6 +9557,7 @@ class BaseLLMHTTPHandler: litellm_logging_obj=logging_obj, litellm_params=dict(litellm_params), extra_body=extra_body, + router=router, ) all_optional_params: Dict[str, Any] = dict(litellm_params) diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index f98cb0e5b0c..5aba5752a44 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -31,6 +31,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -111,6 +112,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """ Transform search request to Gemini's generateContent format. diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index a53075ba1d6..589063cd188 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -123,6 +124,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Azure AI Search API diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 6ccf8e271e5..9ab1568a375 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -21,6 +21,7 @@ from litellm.utils import add_openai_metadata if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -99,6 +100,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: encoded_vector_store_id = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index b58b6e7f498..116f79c834f 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -8,6 +8,7 @@ from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -80,6 +81,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: encoded_vector_store_id = encode_url_path_segment(vector_store_id, field_name="vector_store_id") url = f"{api_base}/{encoded_vector_store_id}/search" diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index d8bdd981425..332ed7f0c6b 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -17,6 +17,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -92,6 +93,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """RAGFlow vector stores are management-only, search is not supported.""" raise NotImplementedError("RAGFlow vector stores support dataset management only, not search/retrieval") diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index b31e6f4511a..a999db21dbe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -1,8 +1,8 @@ -import re from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx +from litellm.caching._embedding_router import resolve_embedding_router from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.types.router import GenericLiteLLMParams @@ -18,6 +18,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.router import Router else: LiteLLMLoggingObj = Any @@ -58,13 +59,18 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): return headers def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str: - aws_region_name = litellm_params.get("aws_region_name") - if not aws_region_name: - raise ValueError("aws_region_name is required for S3 Vectors") - if not re.match(r"^[a-z][a-z0-9-]*$", aws_region_name): - raise ValueError("Invalid aws_region_name format") + # Resolve region the same way the ingestion path does: + # dynamic param -> AWS_REGION_NAME -> AWS_REGION -> default (us-west-2) + aws_region_name = self.get_aws_region_name_for_non_llm_api_calls(litellm_params.get("aws_region_name")) return f"https://s3vectors.{aws_region_name}.api.aws" + def _resolve_query_embedding_router(self, embedding_model: str, router: Optional["Router"]) -> Optional["Router"]: + """Return the router iff it serves ``embedding_model`` as a deployment.""" + if router is None: + return None + model_list = [dict(m) for m in (router.get_model_list() or [])] + return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) + def transform_search_vector_store_request( self, vector_store_id: str, @@ -74,6 +80,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """Sync version - generates embedding synchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -99,10 +106,14 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response = litellm_module.embedding(model=embedding_model, input=[query]) + if embedding_router is not None: + embedding_response = embedding_router.embedding(model=embedding_model, input=[query]) + else: + embedding_response = litellm_module.embedding(model=embedding_model, input=[query]) query_embedding = embedding_response.data[0]["embedding"] url = f"{api_base}/QueryVectors" @@ -128,6 +139,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict]: """Async version - generates embedding asynchronously.""" # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name @@ -153,10 +165,14 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Generate embedding for the query asynchronously embedding_model = litellm_params.get("embedding_model", "text-embedding-3-small") + embedding_router = self._resolve_query_embedding_router(embedding_model=embedding_model, router=router) import litellm as litellm_module - embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query]) + if embedding_router is not None: + embedding_response = await embedding_router.aembedding(model=embedding_model, input=[query]) + else: + embedding_response = await litellm_module.aembedding(model=embedding_model, input=[query]) query_embedding = embedding_response.data[0]["embedding"] url = f"{api_base}/QueryVectors" diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 47a81fc07bf..93ad40616b5 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -19,6 +19,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -97,6 +98,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform search request for Vertex AI RAG API diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 958839d4a48..f6f9e34dc75 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -23,6 +23,7 @@ from litellm.types.vector_stores import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.router import Router LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -197,6 +198,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, extra_body: Optional[Dict[str, Any]] = None, + router: Optional["Router"] = None, ) -> Tuple[str, Dict[str, Any]]: """ Transform a search request for the Vertex AI Search (Discovery Engine) API. diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 27ffc49901b..0d93f20373c 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -26,6 +26,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_headers, get_form_data, ) +from litellm.proxy.vector_store_endpoints.endpoints import ( + _update_request_data_with_litellm_managed_vector_store_registry, +) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, ) @@ -652,6 +655,17 @@ async def rag_query( user_api_key_dict=user_api_key_dict, ) + # Merge litellm-managed vector store params (provider, region, embedding + # model, credentials, ...) from the registry — same source the direct + # /vector_stores/{id}/search endpoint uses. User-supplied + # retrieval_config keys win on conflict. + store_data = await _update_request_data_with_litellm_managed_vector_store_registry( + data={}, + vector_store_id=retrieval_config["vector_store_id"], + user_api_key_dict=user_api_key_dict, + ) + retrieval_config = {**store_data, **retrieval_config} + # Add litellm data request_data: Dict[str, Any] = {} request_data = await add_litellm_data_to_request( diff --git a/litellm/rag/main.py b/litellm/rag/main.py index 29891ccfd24..2329a820f1f 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -59,6 +59,14 @@ INGESTION_REGISTRY: Dict[str, Type[BaseRAGIngestion]] = { "vertex_ai": VertexAIRAGIngestion, } +# retrieval_config keys consumed by the query pipeline itself; everything else is +# forwarded to vector_stores.asearch as provider-specific params (e.g. +# aws_region_name, embedding_model, vector_bucket_name for S3 Vectors). +# `filters`/`retrieval_filter` are reserved for the explicit filter param. +_CONSUMED_RETRIEVAL_CONFIG_KEYS = frozenset( + {"vector_store_id", "custom_llm_provider", "top_k", "filters", "retrieval_filter"} +) + def get_ingestion_class(provider: str) -> Type[BaseRAGIngestion]: """ @@ -233,13 +241,17 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store + # Forward provider-specific retrieval_config extras (region, embedding model, + # bucket, credentials refs, ...) to the search call; kwargs win on conflict. + provider_search_params = {k: v for k, v in retrieval_config.items() if k not in _CONSUMED_RETRIEVAL_CONFIG_KEYS} with _suppressed_sub_call_billing(): search_response = await litellm.vector_stores.asearch( vector_store_id=retrieval_config["vector_store_id"], query=query_text, max_num_results=retrieval_config.get("top_k", 10), custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"), - **kwargs, + router=router, + **{**provider_search_params, **kwargs}, ) search_provider = retrieval_config.get("custom_llm_provider", "openai") diff --git a/litellm/router.py b/litellm/router.py index 78fe3ff025e..bffd1df3814 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5820,6 +5820,7 @@ class Router: return await self._init_vector_store_api_endpoints( original_function=original_function, custom_llm_provider=custom_llm_provider, + call_type=call_type, **kwargs, ) elif call_type in ("afile_delete", "afile_content"): @@ -5860,6 +5861,7 @@ class Router: self, original_function: Callable, custom_llm_provider: Optional[str] = None, + call_type: Optional[str] = None, **kwargs, ): """ @@ -5878,6 +5880,12 @@ class Router: **kwargs, ) + # For search, pass the router so provider transforms can resolve + # router-managed embedding models (e.g. S3 Vectors query embeddings). + # Assigning into kwargs also overrides any client-supplied `router` key. + if call_type == "avector_store_search": + kwargs["router"] = self + # Otherwise, call the original function directly return await original_function(**kwargs) diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index f768ee75545..4035125120e 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -6,7 +6,7 @@ import asyncio import builtins import contextvars from functools import partial -from typing import Any, Coroutine, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Union import httpx @@ -28,6 +28,9 @@ from litellm.types.vector_stores import ( from litellm.utils import ProviderConfigManager, client from litellm.vector_stores.utils import VectorStoreRequestUtils +if TYPE_CHECKING: + from litellm.router import Router + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -279,6 +282,7 @@ async def asearch( timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, custom_llm_provider: Optional[str] = None, + router: Optional["Router"] = None, **kwargs, ) -> VectorStoreSearchResponse: """ @@ -307,6 +311,7 @@ async def asearch( extra_body=extra_body, timeout=timeout, custom_llm_provider=custom_llm_provider, + router=router, **kwargs, ) @@ -346,6 +351,7 @@ def search( timeout: Optional[Union[float, httpx.Timeout]] = None, # LiteLLM specific params, custom_llm_provider: Optional[str] = None, + router: Optional["Router"] = None, **kwargs, ) -> Union[VectorStoreSearchResponse, Coroutine[Any, Any, VectorStoreSearchResponse]]: """ @@ -449,6 +455,7 @@ def search( timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), + router=router, ) return response diff --git a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py index 7085e45cdc3..9389476bef4 100644 --- a/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py +++ b/tests/test_litellm/llms/s3_vectors/vector_stores/test_s3_vectors_transformation.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, Mock +from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest @@ -9,6 +9,18 @@ from litellm.llms.s3_vectors.vector_stores.transformation import ( from litellm.types.vector_stores import VectorStoreSearchResponse +def _mock_router(model_names, sync=False): + """Router mock serving the given embedding model names.""" + router = MagicMock() + router.get_model_list.return_value = [{"model_name": name} for name in model_names] + embedding_response = Mock(data=[{"embedding": [0.1, 0.2, 0.3]}]) + if sync: + router.embedding = MagicMock(return_value=embedding_response) + else: + router.aembedding = AsyncMock(return_value=embedding_response) + return router + + class TestS3VectorsVectorStoreConfig: def test_init(self): """Test that S3VectorsVectorStoreConfig initializes correctly""" @@ -28,19 +40,174 @@ class TestS3VectorsVectorStoreConfig: url = config.get_complete_url(None, litellm_params) assert url == "https://s3vectors.us-west-2.api.aws" - def test_get_complete_url_missing_region(self): - """Test that missing region raises error""" + def test_get_complete_url_missing_region(self, monkeypatch): + """Missing region falls back to the default region (parity with ingestion)""" + monkeypatch.delenv("AWS_REGION_NAME", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) config = S3VectorsVectorStoreConfig() - litellm_params = {} - with pytest.raises(ValueError, match="aws_region_name is required"): - config.get_complete_url(None, litellm_params) + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.us-west-2.api.aws" + + def test_get_complete_url_uses_env_region(self, monkeypatch): + """Missing region param resolves from AWS_REGION_NAME env var""" + monkeypatch.setenv("AWS_REGION_NAME", "eu-west-1") + monkeypatch.delenv("AWS_REGION", raising=False) + config = S3VectorsVectorStoreConfig() + url = config.get_complete_url(None, {}) + assert url == "https://s3vectors.eu-west-1.api.aws" + + def test_get_complete_url_invalid_region_format(self): + """Invalid region format raises""" + config = S3VectorsVectorStoreConfig() + with pytest.raises(ValueError, match="Invalid AWS region format"): + config.get_complete_url(None, {"aws_region_name": "Bad_Region!"}) - @pytest.mark.skip(reason="Requires embedding API call, tested in integration tests") def test_transform_search_request(self): - """Test search request transformation""" - # This test requires making an actual embedding API call - # It's better tested in integration tests - pass + """Full request-body transformation with a router-injected embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["text-embedding-3-small"], sync=True) + + url, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={"max_num_results": 7}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + router=router, + ) + + assert url == "https://s3vectors.us-west-2.api.aws/QueryVectors" + assert request_body == { + "vectorBucketName": "test-bucket", + "indexName": "test-index", + "queryVector": {"float32": [0.1, 0.2, 0.3]}, + "topK": 7, + "returnDistance": True, + "returnMetadata": True, + } + assert mock_logging_obj.model_call_details["query"] == "test query" + + @pytest.mark.asyncio + async def test_atransform_search_uses_router_for_virtual_model(self): + """Regression: router-served embedding models must resolve via the router, + not a bare litellm.aembedding call (which has no deployment credentials).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"]) + + with patch("litellm.aembedding", new=AsyncMock()) as mock_bare_aembedding: + url, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.aembedding.assert_awaited_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + assert request_body["topK"] == 5 # default + + @pytest.mark.asyncio + async def test_atransform_search_falls_back_when_router_does_not_serve_model(self): + """Router present but embedding_model is not a router deployment -> + bare litellm.aembedding keeps working (provider-prefixed + env creds stores).""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["some-other-model"]) + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.4, 0.5]}])) + with patch("litellm.aembedding", new=mock_bare): + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "azure/text-embedding-3-small"}, + extra_body=None, + router=router, + ) + + mock_bare.assert_awaited_once_with(model="azure/text-embedding-3-small", input=["test query"]) + router.aembedding.assert_not_awaited() + assert request_body["queryVector"]["float32"] == [0.4, 0.5] + + @pytest.mark.asyncio + async def test_atransform_search_without_router_uses_bare_embedding(self): + """Backward compat: no router -> bare litellm.aembedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = AsyncMock(return_value=Mock(data=[{"embedding": [0.6, 0.7]}])) + with patch("litellm.aembedding", new=mock_bare): + _, request_body = await config.atransform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_awaited_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.6, 0.7] + + def test_transform_search_uses_router_for_virtual_model_sync(self): + """Sync twin: router-served embedding model resolves via router.embedding""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + router = _mock_router(["my-embedding-model"], sync=True) + + with patch("litellm.embedding", new=MagicMock()) as mock_bare_embedding: + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={"embedding_model": "my-embedding-model"}, + extra_body=None, + router=router, + ) + + router.embedding.assert_called_once_with(model="my-embedding-model", input=["test query"]) + mock_bare_embedding.assert_not_called() + assert request_body["queryVector"]["float32"] == [0.1, 0.2, 0.3] + + def test_transform_search_without_router_uses_bare_embedding_sync(self): + """Sync twin: no router -> bare litellm.embedding as before""" + config = S3VectorsVectorStoreConfig() + mock_logging_obj = Mock() + mock_logging_obj.model_call_details = {} + + mock_bare = MagicMock(return_value=Mock(data=[{"embedding": [0.8, 0.9]}])) + with patch("litellm.embedding", new=mock_bare): + _, request_body = config.transform_search_vector_store_request( + vector_store_id="test-bucket:test-index", + query="test query", + vector_store_search_optional_params={}, + api_base="https://s3vectors.us-west-2.api.aws", + litellm_logging_obj=mock_logging_obj, + litellm_params={}, + extra_body=None, + ) + + mock_bare.assert_called_once_with(model="text-embedding-3-small", input=["test query"]) + assert request_body["queryVector"]["float32"] == [0.8, 0.9] def test_transform_search_request_invalid_vector_store_id(self): """Test that invalid vector_store_id format raises error""" diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 15a117bd6fc..8bd67754952 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -327,3 +327,106 @@ def test_rag_query_stream_returns_event_stream(client_internal_user): assert response.headers.get("content-type", "").startswith("text/event-stream") assert '"object":"chat.completion.chunk"' in response.text assert "data: [DONE]" in response.text + + +def test_rag_query_merges_managed_store_params(client_internal_user): + """ + Regression: /v1/rag/query must consult the managed vector store registry + (like the direct /v1/vector_stores/{id}/search endpoint does) so that + provider, region, embedding model, etc. don't have to be repeated in + retrieval_config. Pre-fix the registry was never read, so managed S3 + Vectors stores failed with "aws_region_name is required". + """ + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": { + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( + "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", + new=AsyncMock(), + ), patch( + "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", + new=AsyncMock(), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store"}, + }, + ) + + assert response.status_code == 200, response.json() + mock_aquery.assert_awaited_once() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["vector_store_id"] == "s3-store" + assert forwarded_config["custom_llm_provider"] == "s3_vectors" + assert forwarded_config["aws_region_name"] == "eu-west-1" + assert forwarded_config["embedding_model"] == "my-embed" + assert forwarded_config["vector_bucket_name"] == "bkt" + + +def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): + """User-supplied retrieval_config keys must win over registry values.""" + import litellm + from litellm.types.utils import ModelResponse + + mock_vector_store = { + "vector_store_id": "s3-store", + "custom_llm_provider": "s3_vectors", + "litellm_params": {"aws_region_name": "eu-west-1"}, + } + mock_registry = MagicMock() + mock_registry.get_litellm_managed_vector_store_from_registry.return_value = mock_vector_store + + mock_response = ModelResponse( + id="chatcmpl-test", + choices=[{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + model="gpt-4o-mini", + ) + + with patch( + "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( + "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", + new=AsyncMock(), + ), patch( + "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", + new=AsyncMock(), + ): + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", "aws_region_name": "us-east-1"}, + }, + ) + + assert response.status_code == 200, response.json() + forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] + assert forwarded_config["aws_region_name"] == "us-east-1" diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index 584124ba06a..d8ffae667b1 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -254,6 +254,96 @@ async def test_aquery_streaming_bills_sub_call_costs_into_final_event(): assert standard_logging_object["response_cost"] >= 0.003 +@pytest.mark.asyncio +async def test_aquery_forwards_provider_retrieval_config_and_router_to_search(): + """ + Regression: provider-specific retrieval_config keys (aws_region_name, + embedding_model, vector_bucket_name, ...) and the router must be forwarded + to the vector store search call. Pre-fix they were silently dropped, so + /v1/rag/query failed with provider config errors (e.g. S3 Vectors + "aws_region_name is required") even when the caller supplied them. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + } + ] + ) + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): + response = await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "top_k": 5, + "aws_region_name": "eu-west-1", + "embedding_model": "my-embed", + "vector_bucket_name": "bkt", + }, + router=router, + mock_response="hi", + ) + + assert isinstance(response, ModelResponse) + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "bkt:idx" + assert search_kwargs["custom_llm_provider"] == "s3_vectors" + assert search_kwargs["max_num_results"] == 5 + assert search_kwargs["router"] is router + # provider-specific extras forwarded + assert search_kwargs["aws_region_name"] == "eu-west-1" + assert search_kwargs["embedding_model"] == "my-embed" + assert search_kwargs["vector_bucket_name"] == "bkt" + # consumed keys are not duplicated into the spread + assert "top_k" not in search_kwargs + + +@pytest.mark.asyncio +async def test_aquery_minimal_retrieval_config_forwards_no_extras(): + """ + A minimal retrieval_config must not leak consumed keys (or invent extras) + into the vector store search call. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={"vector_store_id": "vs_test_123", "custom_llm_provider": "openai"}, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["vector_store_id"] == "vs_test_123" + assert search_kwargs["custom_llm_provider"] == "openai" + assert search_kwargs["router"] is None + leaked = {"top_k", "filters", "retrieval_filter", "aws_region_name", "embedding_model", "vector_bucket_name"} + assert not (leaked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index a9e5b3316e0..91d2973af76 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5936,3 +5936,58 @@ async def test_acreate_batch_request_bedrock_tags_override_deployment_tags(): bedrock_tags=request_tags, ) assert mock_sign.call_args.kwargs["data"]["tags"] == request_tags + + +@pytest.mark.asyncio +async def test_avector_store_search_injects_router(): + """ + Regression: router.avector_store_search must pass the router down to the + SDK search call so provider transforms can resolve router-managed + embedding models (e.g. S3 Vectors query embeddings). + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + mock_asearch = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + # Router.__init__ binds asearch via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.asearch", new=mock_asearch): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + await router.avector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + mock_asearch.assert_awaited_once() + assert mock_asearch.await_args.kwargs["router"] is router + + +@pytest.mark.asyncio +async def test_avector_store_create_does_not_inject_router(): + """The router injection is gated on the search call type: the create path + must keep calling the SDK without a router kwarg.""" + mock_acreate = AsyncMock(return_value={"id": "vs_1", "object": "vector_store"}) + # avector_store_create(model=None) resolves acreate via a local import at + # call time, so patching after Router construction works here. + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + with patch("litellm.vector_stores.main.acreate", new=mock_acreate): + await router.avector_store_create(model=None, custom_llm_provider="openai") + + mock_acreate.assert_awaited_once() + assert "router" not in mock_acreate.await_args.kwargs diff --git a/tests/test_litellm/vector_stores/test_main.py b/tests/test_litellm/vector_stores/test_main.py new file mode 100644 index 00000000000..3fdf4d9daa5 --- /dev/null +++ b/tests/test_litellm/vector_stores/test_main.py @@ -0,0 +1,77 @@ +""" +Tests for litellm/vector_stores/main.py. + +Pins the router threading contract for vector store search: the router is an +explicit named parameter that reaches the HTTP handler, and it must never leak +into litellm_params/kwargs where logging would model_dump() it (the #19550 +serialization trap). +""" + +from unittest.mock import MagicMock, patch + +import litellm.vector_stores.main as vector_stores_main +from litellm.vector_stores.main import search + +MOCK_SEARCH_RESPONSE = { + "object": "vector_store.search_results.page", + "search_query": "q", + "data": [], +} + + +def test_search_threads_router_to_handler(): + """search() must pass its router param through to the HTTP handler""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + mock_handler.assert_called_once() + assert mock_handler.call_args.kwargs["router"] is mock_router + + +def test_search_router_not_in_litellm_params(): + """Regression (#19550 class): the router must stay out of GenericLiteLLMParams, + otherwise pre-call logging model_dump()s it and breaks serialization.""" + mock_router = MagicMock() + logger = MagicMock() + + with ( + patch( + "litellm.vector_stores.main.ProviderConfigManager.get_provider_vector_stores_config", + return_value=MagicMock(), + ), + patch.object( + vector_stores_main.base_llm_http_handler, + "vector_store_search_handler", + return_value=MOCK_SEARCH_RESPONSE, + ) as mock_handler, + ): + search( + vector_store_id="bkt:idx", + query="q", + custom_llm_provider="s3_vectors", + router=mock_router, + litellm_logging_obj=logger, + ) + + litellm_params = mock_handler.call_args.kwargs["litellm_params"] + assert "router" not in litellm_params.model_dump(exclude_none=True) + assert getattr(litellm_params, "router", None) is None diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx index cbabcc6dca5..f375bfdd351 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.test.tsx @@ -128,16 +128,33 @@ describe("VectorStoreTester", () => { await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(1)); }); - it("reports a failed search and keeps the history empty", async () => { + it("shows the backend error in the history when a search fails", async () => { const user = userEvent.setup(); - mockSearch.mockRejectedValue(new Error("boom")); + const errorBody = '{"error":{"message":"OpenAIException - api_key is required"}}'; + mockSearch.mockRejectedValue(new Error(errorBody)); renderTester(); await user.type(queryInput(), "hello"); await user.click(searchButton()); - await waitFor(() => expect(mockFromBackend).toHaveBeenCalledWith("Failed to search vector store")); - expect(screen.getByText(EMPTY_STATE)).toBeInTheDocument(); + await waitFor(() => expect(mockFromBackend).toHaveBeenCalledWith(errorBody)); + expect(screen.getByText(`Search failed: ${errorBody}`)).toBeInTheDocument(); + expect(screen.queryByText("No results found")).not.toBeInTheDocument(); + expect(screen.queryByText(EMPTY_STATE)).not.toBeInTheDocument(); + // the failed query stays in the input for retry + expect(queryInput()).toHaveValue("hello"); + }); + + it('renders "No results found" for an empty result set, not an error', async () => { + const user = userEvent.setup(); + mockSearch.mockResolvedValue({ object: "vector_store.search_results.page", search_query: "hello", data: [] }); + renderTester(); + + await user.type(queryInput(), "hello"); + await user.click(searchButton()); + + expect(await screen.findByText("No results found")).toBeInTheDocument(); + expect(screen.queryByText(/search failed/i)).not.toBeInTheDocument(); }); it("clears the search history", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx index 015d58e8649..65b31bb911a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreTester.tsx @@ -41,6 +41,7 @@ export const VectorStoreTester: React.FC = ({ vectorStor { query: string; response: VectorStoreSearchResponse | null; + error: string | null; timestamp: number; }[] >([]); @@ -60,6 +61,7 @@ export const VectorStoreTester: React.FC = ({ vectorStor const historyEntry = { query, response, + error: null, timestamp: Date.now(), }; @@ -67,7 +69,9 @@ export const VectorStoreTester: React.FC = ({ vectorStor setQuery(""); } catch (error) { console.error("Error searching vector store:", error); - NotificationsManager.fromBackend("Failed to search vector store"); + const errorMessage = error instanceof Error ? error.message : String(error); + NotificationsManager.fromBackend(errorMessage); + setSearchHistory((prev) => [{ query, response: null, error: errorMessage, timestamp: Date.now() }, ...prev]); } finally { setIsLoading(false); } @@ -228,6 +232,8 @@ export const VectorStoreTester: React.FC = ({ vectorStor ); })} + ) : entry.error ? ( +
Search failed: {entry.error}
) : (
No results found
)} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 576e16cbb37..a8408fecaad 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -6851,7 +6851,7 @@ export const vectorStoreSearchCall = async ( if (!response.ok) { const errorData = await response.text(); await handleError(errorData); - return null; + throw new Error(errorData); } const data = await response.json(); From cf4738c3b736edb62f111ab26d899da6a7f9f284 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:32:04 -0700 Subject: [PATCH 2/6] style: format s3 vectors transformation and rag endpoints --- litellm/llms/s3_vectors/vector_stores/transformation.py | 4 +++- litellm/proxy/rag_endpoints/endpoints.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 3c1b0025d08..733358381fe 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -68,7 +68,9 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): """Return the router iff it serves ``embedding_model`` as a deployment.""" if router is None: return None - model_list: Final = [dict(m) for m in (router.get_model_list() or ())] # mutable-ok: resolve_embedding_router requires list[dict] + model_list: Final = [ + dict(m) for m in (router.get_model_list() or ()) + ] # mutable-ok: resolve_embedding_router requires list[dict] return resolve_embedding_router(embedding_model=embedding_model, llm_router=router, llm_model_list=model_list) def transform_search_vector_store_request( diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 31998d2040e..5c392c30018 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -717,7 +717,10 @@ async def rag_query( vector_store_id=retrieval_config["vector_store_id"], user_api_key_dict=user_api_key_dict, ) - merged_retrieval_config: Final = {**store_data, **retrieval_config} # mutable-ok: litellm.aquery requires a plain dict payload + merged_retrieval_config: Final = { + **store_data, + **retrieval_config, + } # mutable-ok: litellm.aquery requires a plain dict payload # Add litellm data request_data: dict[str, object] = {} From 3914de24eff6c0f3deda46ec2f2217cb0305d916 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:52:45 -0700 Subject: [PATCH 3/6] fix(router): route model-less sync vector store calls to the SDK _generic_api_call_with_fallbacks requires a model, so sync vector_store_search and vector_store_create raised a TypeError whenever the call carried no model. Model-less calls now go directly to the SDK function, with the router injected for search, matching the async wrapper's behavior --- litellm/router.py | 19 +++++++---- tests/test_litellm/test_router.py | 55 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d8769525815..96d5b1e1488 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6299,8 +6299,6 @@ class Router: "responses", "generate_content", "generate_content_stream", - "vector_store_search", - "vector_store_create", "ocr", "search", "video_generation", @@ -6324,6 +6322,8 @@ class Router: return sync_wrapper if call_type in ( + "vector_store_search", + "vector_store_create", "vector_store_retrieve", "vector_store_list", "vector_store_update", @@ -6335,11 +6335,16 @@ class Router: client: object | None = None, **kwargs, ): - if custom_llm_provider and "custom_llm_provider" not in kwargs: - kwargs["custom_llm_provider"] = custom_llm_provider - if kwargs.get("model"): - return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) - return original_function(**kwargs) + provider_kwargs: Final = ( + MappingProxyType({**kwargs, "custom_llm_provider": custom_llm_provider}) + if custom_llm_provider and "custom_llm_provider" not in kwargs + else MappingProxyType(kwargs) + ) + if provider_kwargs.get("model"): + return self._generic_api_call_with_fallbacks(original_function=original_function, **provider_kwargs) + if call_type == "vector_store_search": + return original_function(**MappingProxyType({**provider_kwargs, "router": self})) + return original_function(**provider_kwargs) return vector_store_sync_wrapper diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c222fab79f0..3abfe8ce522 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7567,6 +7567,61 @@ async def test_avector_store_create_does_not_inject_router(): assert "router" not in mock_acreate.await_args.kwargs +def test_vector_store_search_injects_router(): + """ + Sync parity for the router injection: router.vector_store_search must pass + the router down to the SDK search call so provider transforms can resolve + router-managed embedding models, same as avector_store_search. + """ + from litellm.types.vector_stores import VectorStoreSearchResponse + + expected_response = VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + mock_search = MagicMock(return_value=expected_response) + # Router.__init__ binds search via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.search", new=mock_search): # test-quality-ok: the SDK call is the only place the injected router kwarg is observable + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + search_response = router.vector_store_search( + vector_store_id="v", query="q", custom_llm_provider="s3_vectors" + ) + + assert search_response is expected_response + mock_search.assert_called_once() + assert mock_search.call_args.kwargs["router"] is router + assert mock_search.call_args.kwargs["custom_llm_provider"] == "s3_vectors" + + +def test_vector_store_create_does_not_inject_router(): + """The sync create path must keep calling the SDK without a router kwarg.""" + expected_response = {"id": "vs_1", "object": "vector_store"} + mock_create = MagicMock(return_value=expected_response) + # Router.__init__ binds create via a local import, so patch the module + # attribute before constructing the Router. + with patch("litellm.vector_stores.main.create", new=mock_create): # test-quality-ok: the SDK call is the only place a leaked router kwarg would surface + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "test-key"}, + } + ] + ) + create_response = router.vector_store_create(custom_llm_provider="openai") + + assert create_response is expected_response + mock_create.assert_called_once() + assert "router" not in mock_create.call_args.kwargs + + class TestPreRoutingStrategyRegistryLifecycle: """ Regression tests: a deployment leaving the model_list must release the From babe7816ada8d622be993b542fc2512037d2466f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 12:52:45 -0700 Subject: [PATCH 4/6] fix(rag): store-wins merge, single lookup, allowlisted search params rag_query reuses the store resolved during authorization instead of a second registry lookup, merges registry data store-wins so callers cannot override a managed store's provider or credentials, and logs ids instead of the merged config, which can carry resolved credentials. aquery forwards only allowlisted retrieval_config keys to vector store search, keeping caller-supplied connection overrides like api_base and api_key away from the search call --- litellm/proxy/rag_endpoints/endpoints.py | 53 ++++++++---- .../proxy/vector_store_endpoints/endpoints.py | 86 ++++++++++--------- .../management_endpoints.py | 4 +- litellm/rag/main.py | 25 ++++-- .../proxy/rag_endpoints/test_rag_endpoints.py | 26 +++--- tests/test_litellm/rag/test_main.py | 39 +++++++++ 6 files changed, 149 insertions(+), 84 deletions(-) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index 5c392c30018..d2c7d6f93ee 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -9,6 +9,7 @@ Provides: import base64 import json from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import orjson @@ -19,6 +20,9 @@ from starlette.datastructures import UploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import ( + LiteLLM_ManagedVectorStore, +) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import * from litellm.proxy.auth.auth_utils import is_request_body_safe @@ -37,7 +41,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( validate_upload, ) from litellm.proxy.vector_store_endpoints.endpoints import ( - _update_request_data_with_litellm_managed_vector_store_registry, # pyright: ignore[reportPrivateUsage] # shared registry-merge helper used by the direct search endpoint + build_request_data_from_managed_vector_store, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -123,12 +127,21 @@ def _collect_vector_store_ids_from_payload(payload: object) -> set[str]: async def _authorize_nested_vector_store_ids( payload: object, user_api_key_dict: UserAPIKeyAuth, -) -> None: - for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)): - await assert_user_can_access_vector_store_id( - vector_store_id=vector_store_id, - user_api_key_dict=user_api_key_dict, - ) +) -> Mapping[str, LiteLLM_ManagedVectorStore]: + """Authorize every nested vector store id and return the managed stores it resolved.""" + return MappingProxyType( + { + vector_store_id: store + for vector_store_id in sorted(_collect_vector_store_ids_from_payload(payload)) + if ( + store := await assert_user_can_access_vector_store_id( + vector_store_id=vector_store_id, + user_api_key_dict=user_api_key_dict, + ) + ) + is not None + } + ) def _build_file_metadata_entry( @@ -703,23 +716,24 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) - await _authorize_nested_vector_store_ids( + resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, ) # Merge litellm-managed vector store params (provider, region, embedding - # model, credentials, ...) from the registry — same source the direct - # /vector_stores/{id}/search endpoint uses. User-supplied - # retrieval_config keys win on conflict. - store_data: Final = await _update_request_data_with_litellm_managed_vector_store_registry( - data={}, # mutable-ok: the helper mutates and returns the seed dict - vector_store_id=retrieval_config["vector_store_id"], - user_api_key_dict=user_api_key_dict, + # model, credentials, ...) from the registry: the same source the direct + # /vector_stores/{id}/search endpoint uses. Store-managed keys win on + # conflict so callers cannot override the store's provider or credentials. + managed_store: Final = resolved_stores.get(retrieval_config["vector_store_id"]) + store_data: Final = ( + await build_request_data_from_managed_vector_store(managed_store) + if managed_store is not None + else MappingProxyType({}) ) merged_retrieval_config: Final = { - **store_data, **retrieval_config, + **store_data, } # mutable-ok: litellm.aquery requires a plain dict payload # Add litellm data @@ -733,7 +747,12 @@ async def rag_query( proxy_config=proxy_config, ) - verbose_proxy_logger.debug("RAG Query - model: %s, retrieval_config: %s", model, merged_retrieval_config) + verbose_proxy_logger.debug( + "RAG Query - model: %s, vector_store_id: %s, custom_llm_provider: %s", + model, + retrieval_config["vector_store_id"], + merged_retrieval_config.get("custom_llm_provider"), + ) # Call query response: Final = await litellm.aquery( diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index a59d7a277cc..3fc6749f18a 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from types import MappingProxyType from typing import ( Annotated, Any, # noqa: TID251 # jsonify_object in proxy/utils.py is annotated with a bare dict @@ -32,6 +34,41 @@ router: Final = APIRouter() ######################################################## +async def build_request_data_from_managed_vector_store( + vector_store: LiteLLM_ManagedVectorStore, +) -> Mapping[str, object]: + """ + Build request params (provider, credential ref, litellm_params) from an + already-resolved managed vector store. + + ``litellm_embedding_config`` is resolved here, at request-handling time, + instead of at row-creation time: the resolved api_key/api_base/api_version + lives only in the returned per-request mapping and is never persisted back + to the registry cache. Legacy rows that already carry a resolved + (cleartext) config skip the lookup and pass through unchanged. + """ + top_level: Final = MappingProxyType( + { + key: vector_store.get(key) + for key in ("custom_llm_provider", "litellm_credential_name") + if key in vector_store + } + ) + litellm_params: Final = vector_store.get("litellm_params") or MappingProxyType({}) + embedding_model: Final = litellm_params.get("litellm_embedding_model") + if not embedding_model or litellm_params.get("litellm_embedding_config"): + return MappingProxyType({**top_level, **litellm_params}) + + from litellm.proxy.proxy_server import prisma_client + + resolved_config: Final = await _resolve_embedding_config( + embedding_model=embedding_model, prisma_client=prisma_client + ) + if not resolved_config: + return MappingProxyType({**top_level, **litellm_params}) + return MappingProxyType({**top_level, **litellm_params, "litellm_embedding_config": resolved_config}) + + async def _update_request_data_with_litellm_managed_vector_store_registry( data: dict, vector_store_id: str, @@ -51,47 +88,14 @@ async def _update_request_data_with_litellm_managed_vector_store_registry( vector_store_to_run: Final[LiteLLM_ManagedVectorStore | None] = await get_litellm_managed_vector_store( vector_store_id=vector_store_id ) - if vector_store_to_run is not None: - if user_api_key_dict is not None: - await assert_user_can_access_vector_store( - vector_store=vector_store_to_run, - user_api_key_dict=user_api_key_dict, - ) - - if "custom_llm_provider" in vector_store_to_run: - data["custom_llm_provider"] = vector_store_to_run.get("custom_llm_provider") - - if "litellm_credential_name" in vector_store_to_run: - data["litellm_credential_name"] = vector_store_to_run.get("litellm_credential_name") - - if "litellm_params" in vector_store_to_run: - litellm_params = vector_store_to_run.get("litellm_params", {}) or {} - # Resolve ``litellm_embedding_config`` here, at request-handling - # time, instead of at row-creation time. The resolved - # ``api_key`` / ``api_base`` / ``api_version`` lives only in - # this per-request ``data`` dict and is never persisted. - # Legacy rows that already carry a resolved (cleartext) - # ``litellm_embedding_config`` skip the lookup and pass through - # unchanged so the embed call keeps working. - embedding_model: Final = litellm_params.get("litellm_embedding_model") - if embedding_model and not litellm_params.get("litellm_embedding_config"): - from litellm.proxy.proxy_server import prisma_client - - resolved_config: Final = await _resolve_embedding_config( - embedding_model=embedding_model, prisma_client=prisma_client - ) - if resolved_config: - # Build a fresh dict via spread instead of mutating - # ``litellm_params`` in place — the registry hands back - # a reference to its cached object, so an in-place - # update would persist the resolved cleartext into the - # in-memory cache for the lifetime of the process. - litellm_params = { - **litellm_params, - "litellm_embedding_config": resolved_config, - } - data.update(litellm_params) - return data + if vector_store_to_run is None: + return data + if user_api_key_dict is not None: + await assert_user_can_access_vector_store( + vector_store=vector_store_to_run, + user_api_key_dict=user_api_key_dict, + ) + return {**data, **(await build_request_data_from_managed_vector_store(vector_store_to_run))} @router.post( diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py index 183a03cc13c..244798ba05e 100644 --- a/litellm/proxy/vector_store_endpoints/management_endpoints.py +++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py @@ -470,7 +470,7 @@ async def create_vector_store_in_db( # exposed every env-stored embedding-model credential on the # ``/vector_store/{new,info,update,list}`` responses. Keep the user's # raw ``litellm_embedding_model`` reference; resolution now happens in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # ``build_request_data_from_managed_vector_store`` # at request-handling time so the cleartext config exists only in # per-request memory and never reaches the database. if litellm_params: @@ -864,7 +864,7 @@ async def update_vector_store( # embedding-config auto-resolve previously persisted cleartext # credentials into the row; resolution now happens at request- # handling time in - # ``_update_request_data_with_litellm_managed_vector_store_registry`` + # ``build_request_data_from_managed_vector_store`` # so this row only ever stores the user-supplied # ``litellm_embedding_model`` reference. if "litellm_params" in update_data: diff --git a/litellm/rag/main.py b/litellm/rag/main.py index bd6788b3a1b..94bfc305a6a 100644 --- a/litellm/rag/main.py +++ b/litellm/rag/main.py @@ -51,12 +51,19 @@ INGESTION_REGISTRY: Final[dict[str, type[BaseRAGIngestion]]] = { "vertex_ai": VertexAIRAGIngestion, } -# retrieval_config keys consumed by the query pipeline itself; everything else is -# forwarded to vector_stores.asearch as provider-specific params (e.g. -# aws_region_name, embedding_model, vector_bucket_name for S3 Vectors). -# `filters`/`retrieval_filter` are reserved for the explicit filter param. -_CONSUMED_RETRIEVAL_CONFIG_KEYS: Final = frozenset( - {"vector_store_id", "custom_llm_provider", "top_k", "filters", "retrieval_filter"} +# Only these retrieval_config keys are forwarded to vector_stores.asearch as +# provider-specific params. The explicit allowlist keeps caller-controlled +# connection overrides (api_base, api_key, ...) away from the search call, +# where they could redirect store credentials to an attacker-chosen host. +_FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset( + { + "aws_region_name", + "vector_bucket_name", + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } ) @@ -233,10 +240,10 @@ async def _execute_query_pipeline( raise ValueError("No query found in messages for RAG query") # 2. Search vector store - # Forward provider-specific retrieval_config extras (region, embedding model, - # bucket, credentials refs, ...) to the search call; kwargs win on conflict. + # Forward allowlisted provider retrieval_config extras (region, embedding + # model, bucket, credential refs) to the search call; kwargs win on conflict. provider_search_params: Final = MappingProxyType( - {k: v for k, v in retrieval_config.items() if k not in _CONSUMED_RETRIEVAL_CONFIG_KEYS} + {k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS} ) forwarded_search_params: Final = MappingProxyType({**provider_search_params, **kwargs}) with _suppressed_sub_call_billing(): diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 5561ee1e6ae..342a4535b21 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -357,12 +357,9 @@ def test_rag_query_merges_managed_store_params(client_internal_user): "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", new_callable=AsyncMock, return_value=mock_response, - ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and stubs the access assert covered by auth tests - "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", - new=AsyncMock(), - ), patch( # test-quality-ok: stubs the direct-endpoint access assert covered by auth tests - "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", - new=AsyncMock(), + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), ): response = client_internal_user.post( "/v1/rag/query", @@ -383,8 +380,8 @@ def test_rag_query_merges_managed_store_params(client_internal_user): assert forwarded_config["vector_bucket_name"] == "bkt" -def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): - """User-supplied retrieval_config keys must win over registry values.""" +def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_user): + """Registry values must win over user-supplied retrieval_config keys so callers cannot override store credentials.""" import litellm from litellm.types.utils import ModelResponse @@ -406,12 +403,9 @@ def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): "litellm.proxy.rag_endpoints.endpoints.litellm.aquery", new_callable=AsyncMock, return_value=mock_response, - ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and stubs the access assert covered by auth tests - "litellm.proxy.rag_endpoints.endpoints.assert_user_can_access_vector_store_id", - new=AsyncMock(), - ), patch( # test-quality-ok: stubs the direct-endpoint access assert covered by auth tests - "litellm.proxy.vector_store_endpoints.endpoints.assert_user_can_access_vector_store", - new=AsyncMock(), + ) as mock_aquery, patch.object(litellm, "vector_store_registry", mock_registry), patch( # test-quality-ok: seeds the managed-store registry the merge under test reads and grants access so real store resolution runs + "litellm.proxy.vector_store_endpoints.utils.can_user_access_vector_store", + new=AsyncMock(return_value=True), ): response = client_internal_user.post( "/v1/rag/query", @@ -424,7 +418,9 @@ def test_rag_query_user_retrieval_config_wins_over_store(client_internal_user): assert response.status_code == 200, response.json() forwarded_config = mock_aquery.await_args.kwargs["retrieval_config"] - assert forwarded_config["aws_region_name"] == "us-east-1" + assert forwarded_config["aws_region_name"] == "eu-west-1" + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/rag/test_main.py b/tests/test_litellm/rag/test_main.py index fdcdf342eea..51d03544910 100644 --- a/tests/test_litellm/rag/test_main.py +++ b/tests/test_litellm/rag/test_main.py @@ -349,6 +349,45 @@ async def test_aquery_minimal_retrieval_config_forwards_no_extras(): assert not (leaked & set(search_kwargs.keys())) +@pytest.mark.asyncio +async def test_aquery_does_not_forward_connection_override_keys_to_search(): + """ + Only allowlisted retrieval_config keys may reach the vector store search + call. Caller-controlled connection overrides (api_base, api_key, arbitrary + extras) must be dropped, otherwise a caller could redirect store + credentials to an attacker-chosen host. + """ + from unittest.mock import AsyncMock + + from litellm.types.vector_stores import VectorStoreSearchResponse + + fake_search = AsyncMock( + return_value=VectorStoreSearchResponse( + object="vector_store.search_results.page", search_query="q", data=[] + ) + ) + with patch("litellm.vector_stores.asearch", new=fake_search): # test-quality-ok: asearch is the boundary the forwarding contract under test targets + await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + retrieval_config={ + "vector_store_id": "bkt:idx", + "custom_llm_provider": "s3_vectors", + "aws_region_name": "eu-west-1", + "api_base": "https://attacker.example.com", + "api_key": "attacker-key", + "arbitrary_extra": "nope", + }, + mock_response="hi", + ) + + fake_search.assert_awaited_once() + search_kwargs = fake_search.await_args.kwargs + assert search_kwargs["aws_region_name"] == "eu-west-1" + blocked = {"api_base", "api_key", "arbitrary_extra"} + assert not (blocked & set(search_kwargs.keys())) + + def test_rag_call_types_are_registered(): """ query/aquery/ingest/aingest are @client-decorated entry points, so their From 8b5ae3da9d49e08c4f6c8ca22d6f59c64c6bfae5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:15:51 -0700 Subject: [PATCH 5/6] test(vector_stores): package the suite dir to avoid test_main basename collision --- tests/test_litellm/vector_stores/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_litellm/vector_stores/__init__.py diff --git a/tests/test_litellm/vector_stores/__init__.py b/tests/test_litellm/vector_stores/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 8b0441a628c01f0cd6caa10176ae887c06d75fa7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:29:25 -0700 Subject: [PATCH 6/6] fix(vector_stores): block caller-supplied embedding selection params on query surfaces --- litellm/proxy/rag_endpoints/endpoints.py | 2 ++ .../proxy/vector_store_endpoints/endpoints.py | 24 ++++++++++++++ .../proxy/rag_endpoints/test_rag_endpoints.py | 24 ++++++++++++++ .../test_vector_store_endpoints.py | 32 +++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index d2c7d6f93ee..0ab7d99e4e4 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -42,6 +42,7 @@ from litellm.proxy.rag_endpoints.upload_security import ( ) from litellm.proxy.vector_store_endpoints.endpoints import ( build_request_data_from_managed_vector_store, + reject_caller_embedding_selection_params, ) from litellm.proxy.vector_store_endpoints.utils import ( assert_user_can_access_vector_store_id, @@ -716,6 +717,7 @@ async def rag_query( status_code=400, detail={"error": "retrieval_config must contain 'vector_store_id'"}, ) + reject_caller_embedding_selection_params(payload=retrieval_config, source="retrieval_config") resolved_stores: Final = await _authorize_nested_vector_store_ids( payload=retrieval_config, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 3fc6749f18a..7d64e648e08 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -29,6 +29,29 @@ from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry router: Final = APIRouter() + +BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS: Final = frozenset( + { + "embedding_model", + "litellm_embedding_model", + "litellm_embedding_config", + "litellm_credential_name", + } +) + + +def reject_caller_embedding_selection_params(payload: Mapping[str, object], source: str) -> None: + blocked: Final = sorted(BLOCKED_QUERY_EMBEDDING_SELECTION_PARAMS & payload.keys()) + if blocked: + raise HTTPException( + status_code=400, + detail={ + "error": f"'{blocked[0]}' cannot be set in {source}. " + "Embedding configuration comes from the vector store's server-side registration." + }, + ) + + ######################################################## # OpenAI Compatible Endpoints ######################################################## @@ -134,6 +157,7 @@ async def vector_store_search( ) data = await _read_request_body(request=request) + reject_caller_embedding_selection_params(payload=data, source="the search request body") data["vector_store_id"] = vector_store_id # Check for legacy vector store registry (non-managed vector stores) diff --git a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py index 342a4535b21..0085b6ebd36 100644 --- a/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py +++ b/tests/test_litellm/proxy/rag_endpoints/test_rag_endpoints.py @@ -421,6 +421,30 @@ def test_rag_query_store_params_win_over_user_retrieval_config(client_internal_u assert forwarded_config["aws_region_name"] == "eu-west-1" +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_rag_query_rejects_caller_embedding_selection_params(client_internal_user, blocked_key): + """ + Regression: a caller must not pick the embedding model or credential used at + search time. Those resolve through the Router with the proxy's credentials, + bypassing the key's model permissions, so they may only come from the + managed store's server-side registration. + """ + response = client_internal_user.post( + "/v1/rag/query", + json={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "retrieval_config": {"vector_store_id": "s3-store", blocked_key: "attacker-choice"}, + }, + ) + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json()) + + EICAR = r"X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*" INGEST_REQUEST = '{"ingest_options":{"vector_store":{"custom_llm_provider":"openai"}}}' diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py index eae6f90863a..45a0221c8a6 100644 --- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -3158,3 +3158,35 @@ class TestAzureAIAnalyzeNamedIndexClassification: user_api_key_dict=self._team_member("analyze", ["read"]), ) assert result is True + + +@pytest.mark.parametrize( + "blocked_key", + ["embedding_model", "litellm_embedding_model", "litellm_embedding_config", "litellm_credential_name"], +) +def test_vector_store_search_rejects_caller_embedding_selection_params(blocked_key): + """ + Regression: the search request body must not pick the embedding model or + credential used to embed the query. Those resolve through the Router with + the proxy's credentials, bypassing the key's model permissions, so they may + only come from the managed store's server-side registration. + """ + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.proxy_server import app + + mock_auth = UserAPIKeyAuth(user_id="test_internal_user", user_role=LitellmUserRoles.INTERNAL_USER.value) + original_overrides = app.dependency_overrides.copy() + app.dependency_overrides[user_api_key_auth] = lambda: mock_auth + try: + client = TestClient(app) + response = client.post( + "/v1/vector_stores/s3-store/search", + json={"query": "hello", blocked_key: "attacker-choice"}, + ) + finally: + app.dependency_overrides = original_overrides + + assert response.status_code == 400, response.json() + assert blocked_key in str(response.json())