From dddc86ccb450a83025640017a6f0086ff99b8266 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:43:53 -0800 Subject: [PATCH] fix(vector_stores): keep a caller's image query through store and router params A deployment's retrievalQuery no longer reaches a routed search, a caller's query fields survive a store's own extra_body, and an extra_body image stops before the provider. file_search forwards rewrite_query, both retrieval query spellings are redacted, and the cost error stops echoing the response when message logging is off (cherry picked from commit c65634d4ade59d5c06dabacb946b77161aa79365) --- litellm/cost_calculator.py | 15 ++++++-- litellm/litellm_core_utils/redact_messages.py | 3 +- .../proxy/vector_store_endpoints/endpoints.py | 15 ++++++-- litellm/router.py | 14 ++++++- litellm/types/vector_stores.py | 2 + litellm/vector_stores/main.py | 8 +++- litellm/vector_stores/search_query.py | 10 +++++ .../test_redact_messages.py | 12 ++++++ .../test_mongodb_transformation.py | 37 +++++++++++++++++++ .../test_vector_store_endpoints.py | 12 ++++++ tests/unit/test_cost_calculator.py | 8 +++- .../test_router_vector_store_endpoints.py | 36 ++++++++++++++++++ tests/unit/vector_stores/test_main.py | 6 ++- .../test_vector_store_registry.py | 14 ++++--- 14 files changed, 173 insertions(+), 19 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index dd4b9bbe798..9cbbcafb569 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -52,6 +52,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( parse_prompt_tokens_details, select_cost_metric_for_model, ) +from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.llms.anthropic.cost_calculation import ( cost_per_token as anthropic_cost_per_token, ) @@ -1499,6 +1500,14 @@ def _response_time_ms_for_cost( return 0.0 +def _model_is_none_error(completion_response_text: str, litellm_logging_obj: LitellmLoggingObject | None) -> ValueError: + message: Final = "Model is None and does not exist in passed completion_response." + model_call_details: Final = {} if litellm_logging_obj is None else litellm_logging_obj.model_call_details + if should_redact_message_logging(model_call_details): + return ValueError(message) + return ValueError(f"{message} Passed completion_response={completion_response_text}, model=None") + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1751,7 +1760,7 @@ def completion_cost( else: if model is None: - raise ValueError("Model is None and does not exist in passed completion_response.") + raise _model_is_none_error(f"{completion_response}", litellm_logging_obj) if len(messages) > 0: prompt_tokens = token_counter(model=model, messages=messages) elif len(prompt) > 0: @@ -1765,7 +1774,7 @@ def completion_cost( return A2ACostCalculator.calculate_a2a_cost(litellm_logging_obj=litellm_logging_obj) if model is None: - raise ValueError("Model is None and does not exist in passed completion_response.") + raise _model_is_none_error(f"{completion_response}", litellm_logging_obj) if custom_llm_provider is None: try: model, custom_llm_provider, _, _ = litellm.get_llm_provider( @@ -1985,7 +1994,7 @@ def completion_cost( return get_replicate_completion_pricing(completion_response, response_time_ms) if model is None: - raise ValueError("Model is None and does not exist in passed completion_response.") + raise _model_is_none_error(f"{completion_response}", litellm_logging_obj) if custom_llm_provider is not None and custom_llm_provider == "vertex_ai": # Calculate the prompt characters + response characters diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 4ee52f4d076..49915f93e4b 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -31,6 +31,7 @@ from litellm.llms.vertex_ai.common_utils import ( from litellm.secret_managers.main import str_to_bool from litellm.types.router import BaselineRouteStamp from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -250,7 +251,7 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str): output_item["input"] = redacted_str -_REDACTED_REQUEST_FIELDS: Final = ("query_image", "retrievalQuery") +_REDACTED_REQUEST_FIELDS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS) _NESTED_REQUEST_FIELDS: Final = ("extra_body", "additional_args", "complete_input_dict") diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py index 756796d007a..bca82c99865 100644 --- a/litellm/proxy/vector_store_endpoints/endpoints.py +++ b/litellm/proxy/vector_store_endpoints/endpoints.py @@ -17,6 +17,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import read_raw_json_body +from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER from litellm.proxy.utils import jsonify_object from litellm.proxy.vector_store_endpoints.utils import ( assert_proxy_admin_for_vector_store_index_management, @@ -25,7 +26,7 @@ from litellm.proxy.vector_store_endpoints.utils import ( ) from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse -from litellm.vector_stores.search_query import store_search_params +from litellm.vector_stores.search_query import store_search_params, with_caller_query_params from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry @@ -138,11 +139,19 @@ async def vector_store_search( data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line reject_caller_embedding_selection_params(payload=data, source="the search request body") + caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data) data["vector_store_id"] = vector_store_id # Check for legacy vector store registry (non-managed vector stores) - data = await _update_request_data_with_litellm_managed_vector_store_registry( - data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict + data = dict( # rebind-ok: pre-existing rebinding of the request body + with_caller_query_params( + JSON_OBJECT_ADAPTER.validate_python( + await _update_request_data_with_litellm_managed_vector_store_registry( + data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict + ) + ), + caller_body.get("extra_body"), + ) ) # The managed_vector_stores pre-call hook will handle: diff --git a/litellm/router.py b/litellm/router.py index 9880ca6fdd0..678c75fbaea 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -391,6 +391,7 @@ from litellm.utils import ( provider_rejectable_params, set_live_deployment_replay, ) +from litellm.vector_stores.search_query import store_search_params from .router_utils.pattern_match_deployments import PatternMatchRouter @@ -2307,6 +2308,7 @@ class Router: update, ) + self._vector_store_search_handlers: tuple[Callable[..., object], ...] = (asearch, search) self.avector_store_search = self.factory_function(asearch, call_type="avector_store_search") self.vector_store_search = self.factory_function(search, call_type="vector_store_search") self.vector_store_create = self.factory_function(create, call_type="vector_store_create") @@ -5753,7 +5755,11 @@ class Router: custom_llm_provider: Final = provider_for_generic_call(data) response_kwargs: Final = { - **data, + **( + store_search_params(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if original_generic_function in self._vector_store_search_handlers + else data + ), "caching": self.cache_responses, **kwargs, "model": model_name, @@ -6496,7 +6502,11 @@ class Router: response: Final = original_function( **{ - **data, + **( + store_search_params(_LITELLM_PARAMS_ADAPTER.validate_python(data)) + if original_function in self._vector_store_search_handlers + else data + ), "custom_llm_provider": custom_llm_provider, "caching": self.cache_responses, **kwargs, diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index 41df9cf37ec..e39017f68f3 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -352,6 +352,7 @@ class VectorStoreToolParams: filters: dict | None = None max_num_results: int | None = None ranking_options: dict | None = None + rewrite_query: bool | None = None def to_dict(self) -> dict: """Convert to dict, excluding None values""" @@ -361,6 +362,7 @@ class VectorStoreToolParams: "filters": self.filters, "max_num_results": self.max_num_results, "ranking_options": self.ranking_options, + "rewrite_query": self.rewrite_query, }.items() if v is not None } diff --git a/litellm/vector_stores/main.py b/litellm/vector_stores/main.py index 72c84850e95..6b3075f10e7 100644 --- a/litellm/vector_stores/main.py +++ b/litellm/vector_stores/main.py @@ -13,7 +13,7 @@ import httpx import litellm from litellm.constants import request_timeout -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.vector_store.transformation import ( @@ -464,6 +464,8 @@ def search( ) ) + sent_extra_body: Final = as_str_mapping(extra_body) + # Pre Call logging litellm_logging_obj.update_from_kwargs( kwargs=kwargs, @@ -491,7 +493,9 @@ def search( logging_obj=litellm_logging_obj, embedding_executor=embedding_executor, extra_headers=extra_headers, - extra_body=extra_body, + extra_body=extra_body + if sent_extra_body is None + else {key: value for key, value in sent_extra_body.items() if key != "query_image"}, timeout=timeout or request_timeout, _is_async=_is_async, client=kwargs.get("client"), diff --git a/litellm/vector_stores/search_query.py b/litellm/vector_stores/search_query.py index e514030bf2e..901f3889c7e 100644 --- a/litellm/vector_stores/search_query.py +++ b/litellm/vector_stores/search_query.py @@ -14,6 +14,7 @@ from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS, VectorStoreSearc _QUERY_IMAGE: Final = TypeAdapter(VectorStoreSearchQueryImage) _Query: Final = TypeVar("_Query", bound=str | Sequence[str]) +_CALLER_QUERY_PARAMS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS) @dataclass(frozen=True, slots=True) @@ -118,6 +119,15 @@ def store_search_params(litellm_params: Mapping[str, object]) -> Mapping[str, ob ) +def with_caller_query_params(request: Mapping[str, object], caller_extra_body: object) -> Mapping[str, object]: + caller: Final = as_str_mapping(caller_extra_body) + store: Final = as_str_mapping(request.get("extra_body")) + if caller is None or store is None: + return request + query_params: Final = {key: caller[key] for key in _CALLER_QUERY_PARAMS if key in caller} + return {**request, "extra_body": {**store, **query_params}} + + def resolve_search_query( query: _Query | None, query_image: object, diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index a9b62b7a126..099f35aca30 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -1536,6 +1536,18 @@ def test_redaction_removes_a_query_image_sent_inside_extra_body() -> None: assert SECRET not in json.dumps(details, default=str) +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +def test_redaction_removes_a_raw_retrieval_query_sent_inside_extra_body(retrieval_query_param: str) -> None: + retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}} + details: Final[dict[str, object]] = { + "litellm_params": {"proxy_server_request": {"body": {"extra_body": {retrieval_query_param: retrieval_query}}}}, + } + + perform_redaction(details, result=None) + + assert SECRET not in json.dumps(details, default=str) + + def test_redaction_removes_the_image_from_the_provider_request_copy() -> None: retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}} details: Final[dict[str, object]] = { diff --git a/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py index 9de473fb1f0..d7d9b21ed20 100644 --- a/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py +++ b/tests/unit/llms/mongodb/vector_stores/test_mongodb_transformation.py @@ -220,3 +220,40 @@ async def test_public_sdk_preserves_http_errors_response_and_timeout( else: assert await search() == RESULT executor.call.assert_called_once_with("embedding-alias", "travel policy", {}) + + +QUERY_IMAGE: Final = {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="} + + +def _search_with_extra_body(extra_body: dict[str, object], **search_kwargs: object) -> VectorStoreSearchResponse: + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=RESULT) + + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + return litellm.vector_stores.search( + **BASE_PARAMS, + vector_store_id="policy_index", + query="travel policy", + custom_llm_provider="mongodb", + extra_body=extra_body, + _direct_vector_store_embedding_executor=RecordingEmbeddingExecutor(), + client=HTTPHandler(client=transport), + **search_kwargs, + ) + + +@pytest.mark.parametrize("drop_params_source", ["request", "global"]) +def test_search_with_drop_params_drops_an_extra_body_query_image( + drop_params_source: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global") + drop_kwargs: Final = {"drop_params": True} if drop_params_source == "request" else {} + + assert _search_with_extra_body({"query_image": QUERY_IMAGE}, **drop_kwargs) == RESULT + + +def test_search_with_drop_params_still_rejects_other_extra_body_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + + with pytest.raises(litellm.BadRequestError, match="does not support extra_body overrides"): + _search_with_extra_body({"query_image": QUERY_IMAGE, "other": "value"}) diff --git a/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py index c9fa293317b..74ca6da728c 100644 --- a/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py +++ b/tests/unit/proxy/vector_store_endpoints/test_vector_store_endpoints.py @@ -16,6 +16,7 @@ from litellm.llms.base_llm.vector_store.transformation import ( RouterVectorStoreEmbeddingExecutor, ) from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.vector_stores.search_query import with_caller_query_params from litellm.proxy.vector_store_endpoints.endpoints import ( _update_request_data_with_litellm_managed_vector_store_registry, build_request_data_from_managed_vector_store, @@ -3134,6 +3135,17 @@ def test_a_raw_retrieval_query_in_a_stores_extra_body_never_reaches_the_search_r assert request_data == {"custom_llm_provider": "bedrock", "extra_body": {"kept": "value"}} +@pytest.mark.parametrize("caller_param", ["query_image", "retrievalQuery", "retrieval_query"]) +def test_a_callers_query_survives_a_stores_own_extra_body(caller_param: str): + caller_value: Final = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="} + request_data: Final = with_caller_query_params( + {"custom_llm_provider": "bedrock", "extra_body": {"retrievalConfiguration": "from the store"}}, + {caller_param: caller_value, "retrievalConfiguration": "from the caller", "other": "from the caller"}, + ) + + assert request_data["extra_body"] == {"retrievalConfiguration": "from the store", caller_param: caller_value} + + class TestConfigOwnedVectorStores: """Stores declared under ``vector_store_registry`` in config.yaml are owned by the config file""" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 01205a53ee9..69fb81c6d3e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10270,10 +10270,14 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) -def test_cost_failure_for_an_unpriced_response_does_not_echo_the_response() -> None: +@pytest.mark.parametrize("message_logging_off", [True, False]) +def test_cost_failure_for_an_unpriced_response_echoes_the_response_only_with_message_logging_on( + message_logging_off: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", message_logging_off) image_data: Final = "c2VjcmV0LWltYWdlLWJ5dGVz" with pytest.raises(ValueError, match="Model is None") as exc_info: completion_cost(completion_response={"data": [{"content": [{"source": {"data": image_data}}]}]}) - assert image_data not in str(exc_info.value) + assert (image_data in str(exc_info.value)) is not message_logging_off diff --git a/tests/unit/test_router/test_router_vector_store_endpoints.py b/tests/unit/test_router/test_router_vector_store_endpoints.py index 70621425e11..69a4ad041d7 100644 --- a/tests/unit/test_router/test_router_vector_store_endpoints.py +++ b/tests/unit/test_router/test_router_vector_store_endpoints.py @@ -1,3 +1,4 @@ +import inspect import json from collections.abc import Iterator from typing import Final @@ -160,3 +161,38 @@ async def test_avector_store_retrieve_and_delete_return_provider_responses() -> assert delete_route.call_count == 1 assert retrieved["id"] == "vs_test123" assert deleted["deleted"] is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"]) +@pytest.mark.parametrize("endpoint", ["avector_store_search", "vector_store_search"]) +async def test_a_deployments_raw_retrieval_query_never_reaches_a_routed_vector_store_search( + endpoint: str, retrieval_query_param: str +) -> None: + from_deployment: Final = {"text": "from the deployment"} + router: Final = Router( + model_list=[ + { + "model_name": "kb-route", + "litellm_params": { + "model": "bedrock/kb-route-model", + "aws_access_key_id": "test-access-key", + "aws_secret_access_key": "test-secret-key", + "aws_region_name": "us-east-1", + retrieval_query_param: from_deployment, + "extra_body": {retrieval_query_param: from_deployment}, + }, + } + ] + ) + with respx.mock(assert_all_called=True) as mock: + route: Final = mock.post( + "https://bedrock-agent-runtime.us-east-1.amazonaws.com/knowledgebases/kb-id/retrieve" + ).mock(return_value=httpx.Response(200, json={"retrievalResults": []})) + result: Final = getattr(router, endpoint)( + model="kb-route", vector_store_id="kb-id", query="caller text", custom_llm_provider="bedrock" + ) + if inspect.isawaitable(result): + await result + sent: Final = json.loads(route.calls.last.request.content) + assert sent["retrievalQuery"]["text"] == "caller text" diff --git a/tests/unit/vector_stores/test_main.py b/tests/unit/vector_stores/test_main.py index f648342026d..f8d19707a85 100644 --- a/tests/unit/vector_stores/test_main.py +++ b/tests/unit/vector_stores/test_main.py @@ -175,7 +175,11 @@ def test_search_by_query_image_sends_a_bedrock_image_retrieval_query(query: str ), ( {"custom_llm_provider": "openai", "query_image": QUERY_IMAGE}, - "openai vector stores do not support query_image", + "openai vector stores do not support query_image. To drop it", + ), + ( + {"custom_llm_provider": "openai", "query": "red circle", "query_image": QUERY_IMAGE}, + "openai vector stores do not support query_image. To drop it", ), ( {"custom_llm_provider": "openai", "query": "", "query_image": QUERY_IMAGE, "drop_params": True}, diff --git a/tests/unit/vector_stores/test_vector_store_registry.py b/tests/unit/vector_stores/test_vector_store_registry.py index 0fed3c0cda2..c40f1fe8e95 100644 --- a/tests/unit/vector_stores/test_vector_store_registry.py +++ b/tests/unit/vector_stores/test_vector_store_registry.py @@ -378,11 +378,15 @@ def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_i @pytest.mark.parametrize( - ("tool_field", "tool_value"), - [("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}), ("rewrite_query", True)], + ("tool_field", "tool_value", "searched_params"), + [ + ("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}, {"max_num_results": 3}), + ("rewrite_query", True, {"max_num_results": 3, "rewrite_query": True}), + ("rewrite_query", False, {"max_num_results": 3, "rewrite_query": False}), + ], ) -def test_file_search_tool_with_a_field_the_store_search_does_not_take_still_runs_its_store( - tool_field: str, tool_value: object +def test_file_search_tool_passes_its_store_only_the_fields_a_store_search_takes( + tool_field: str, tool_value: object, searched_params: dict[str, object] ) -> None: now: Final = datetime.now(timezone.utc) registry: Final = VectorStoreRegistry( @@ -393,7 +397,7 @@ def test_file_search_tool_with_a_field_the_store_search_does_not_take_still_runs to_run: Final = registry.pop_vector_stores_to_run(non_default_params={}, tools=tools) assert [store["vector_store_id"] for store in to_run] == ["vs_kb"] - assert to_run[0]["litellm_params"] == {"max_num_results": 3} + assert to_run[0]["litellm_params"] == searched_params @pytest.mark.parametrize(