fix(vector_stores): keep a caller's image query through store and router params

A deployment's retrievalQuery no longer reaches a routed search, a caller's query fields survive a store's own extra_body, and an extra_body image stops before the provider. file_search forwards rewrite_query, both retrieval query spellings are redacted, and the cost error stops echoing the response when message logging is off

(cherry picked from commit c65634d4ad)
This commit is contained in:
mateo-berri 2026-11-10 21:43:53 -08:00
parent 6e7710c5ab
commit dddc86ccb4
14 changed files with 173 additions and 19 deletions

View file

@ -52,6 +52,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
parse_prompt_tokens_details,
select_cost_metric_for_model,
)
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.llms.anthropic.cost_calculation import (
cost_per_token as anthropic_cost_per_token,
)
@ -1499,6 +1500,14 @@ def _response_time_ms_for_cost(
return 0.0
def _model_is_none_error(completion_response_text: str, litellm_logging_obj: LitellmLoggingObject | None) -> ValueError:
message: Final = "Model is None and does not exist in passed completion_response."
model_call_details: Final = {} if litellm_logging_obj is None else litellm_logging_obj.model_call_details
if should_redact_message_logging(model_call_details):
return ValueError(message)
return ValueError(f"{message} Passed completion_response={completion_response_text}, model=None")
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -1751,7 +1760,7 @@ def completion_cost(
else:
if model is None:
raise ValueError("Model is None and does not exist in passed completion_response.")
raise _model_is_none_error(f"{completion_response}", litellm_logging_obj)
if len(messages) > 0:
prompt_tokens = token_counter(model=model, messages=messages)
elif len(prompt) > 0:
@ -1765,7 +1774,7 @@ def completion_cost(
return A2ACostCalculator.calculate_a2a_cost(litellm_logging_obj=litellm_logging_obj)
if model is None:
raise ValueError("Model is None and does not exist in passed completion_response.")
raise _model_is_none_error(f"{completion_response}", litellm_logging_obj)
if custom_llm_provider is None:
try:
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
@ -1985,7 +1994,7 @@ def completion_cost(
return get_replicate_completion_pricing(completion_response, response_time_ms)
if model is None:
raise ValueError("Model is None and does not exist in passed completion_response.")
raise _model_is_none_error(f"{completion_response}", litellm_logging_obj)
if custom_llm_provider is not None and custom_llm_provider == "vertex_ai":
# Calculate the prompt characters + response characters

View file

@ -31,6 +31,7 @@ from litellm.llms.vertex_ai.common_utils import (
from litellm.secret_managers.main import str_to_bool
from litellm.types.router import BaselineRouteStamp
from litellm.types.utils import StandardCallbackDynamicParams
from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
@ -250,7 +251,7 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
output_item["input"] = redacted_str
_REDACTED_REQUEST_FIELDS: Final = ("query_image", "retrievalQuery")
_REDACTED_REQUEST_FIELDS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS)
_NESTED_REQUEST_FIELDS: Final = ("extra_body", "additional_args", "complete_input_dict")

View file

@ -17,6 +17,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.http_parsing_utils import read_raw_json_body
from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER
from litellm.proxy.utils import jsonify_object
from litellm.proxy.vector_store_endpoints.utils import (
assert_proxy_admin_for_vector_store_index_management,
@ -25,7 +26,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
)
from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository
from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse
from litellm.vector_stores.search_query import store_search_params
from litellm.vector_stores.search_query import store_search_params, with_caller_query_params
from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot
from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry
@ -138,11 +139,19 @@ async def vector_store_search(
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
reject_caller_embedding_selection_params(payload=data, source="the search request body")
caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data)
data["vector_store_id"] = vector_store_id
# Check for legacy vector store registry (non-managed vector stores)
data = await _update_request_data_with_litellm_managed_vector_store_registry(
data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict
data = dict( # rebind-ok: pre-existing rebinding of the request body
with_caller_query_params(
JSON_OBJECT_ADAPTER.validate_python(
await _update_request_data_with_litellm_managed_vector_store_registry(
data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict
)
),
caller_body.get("extra_body"),
)
)
# The managed_vector_stores pre-call hook will handle:

View file

@ -391,6 +391,7 @@ from litellm.utils import (
provider_rejectable_params,
set_live_deployment_replay,
)
from litellm.vector_stores.search_query import store_search_params
from .router_utils.pattern_match_deployments import PatternMatchRouter
@ -2307,6 +2308,7 @@ class Router:
update,
)
self._vector_store_search_handlers: tuple[Callable[..., object], ...] = (asearch, search)
self.avector_store_search = self.factory_function(asearch, call_type="avector_store_search")
self.vector_store_search = self.factory_function(search, call_type="vector_store_search")
self.vector_store_create = self.factory_function(create, call_type="vector_store_create")
@ -5753,7 +5755,11 @@ class Router:
custom_llm_provider: Final = provider_for_generic_call(data)
response_kwargs: Final = {
**data,
**(
store_search_params(_LITELLM_PARAMS_ADAPTER.validate_python(data))
if original_generic_function in self._vector_store_search_handlers
else data
),
"caching": self.cache_responses,
**kwargs,
"model": model_name,
@ -6496,7 +6502,11 @@ class Router:
response: Final = original_function(
**{
**data,
**(
store_search_params(_LITELLM_PARAMS_ADAPTER.validate_python(data))
if original_function in self._vector_store_search_handlers
else data
),
"custom_llm_provider": custom_llm_provider,
"caching": self.cache_responses,
**kwargs,

View file

@ -352,6 +352,7 @@ class VectorStoreToolParams:
filters: dict | None = None
max_num_results: int | None = None
ranking_options: dict | None = None
rewrite_query: bool | None = None
def to_dict(self) -> dict:
"""Convert to dict, excluding None values"""
@ -361,6 +362,7 @@ class VectorStoreToolParams:
"filters": self.filters,
"max_num_results": self.max_num_results,
"ranking_options": self.ranking_options,
"rewrite_query": self.rewrite_query,
}.items()
if v is not None
}

View file

@ -13,7 +13,7 @@ import httpx
import litellm
from litellm.constants import request_timeout
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.vector_store.transformation import (
@ -464,6 +464,8 @@ def search(
)
)
sent_extra_body: Final = as_str_mapping(extra_body)
# Pre Call logging
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
@ -491,7 +493,9 @@ def search(
logging_obj=litellm_logging_obj,
embedding_executor=embedding_executor,
extra_headers=extra_headers,
extra_body=extra_body,
extra_body=extra_body
if sent_extra_body is None
else {key: value for key, value in sent_extra_body.items() if key != "query_image"},
timeout=timeout or request_timeout,
_is_async=_is_async,
client=kwargs.get("client"),

View file

@ -14,6 +14,7 @@ from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS, VectorStoreSearc
_QUERY_IMAGE: Final = TypeAdapter(VectorStoreSearchQueryImage)
_Query: Final = TypeVar("_Query", bound=str | Sequence[str])
_CALLER_QUERY_PARAMS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS)
@dataclass(frozen=True, slots=True)
@ -118,6 +119,15 @@ def store_search_params(litellm_params: Mapping[str, object]) -> Mapping[str, ob
)
def with_caller_query_params(request: Mapping[str, object], caller_extra_body: object) -> Mapping[str, object]:
caller: Final = as_str_mapping(caller_extra_body)
store: Final = as_str_mapping(request.get("extra_body"))
if caller is None or store is None:
return request
query_params: Final = {key: caller[key] for key in _CALLER_QUERY_PARAMS if key in caller}
return {**request, "extra_body": {**store, **query_params}}
def resolve_search_query(
query: _Query | None,
query_image: object,

View file

@ -1536,6 +1536,18 @@ def test_redaction_removes_a_query_image_sent_inside_extra_body() -> None:
assert SECRET not in json.dumps(details, default=str)
@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"])
def test_redaction_removes_a_raw_retrieval_query_sent_inside_extra_body(retrieval_query_param: str) -> None:
retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}}
details: Final[dict[str, object]] = {
"litellm_params": {"proxy_server_request": {"body": {"extra_body": {retrieval_query_param: retrieval_query}}}},
}
perform_redaction(details, result=None)
assert SECRET not in json.dumps(details, default=str)
def test_redaction_removes_the_image_from_the_provider_request_copy() -> None:
retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}}
details: Final[dict[str, object]] = {

View file

@ -220,3 +220,40 @@ async def test_public_sdk_preserves_http_errors_response_and_timeout(
else:
assert await search() == RESULT
executor.call.assert_called_once_with("embedding-alias", "travel policy", {})
QUERY_IMAGE: Final = {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="}
def _search_with_extra_body(extra_body: dict[str, object], **search_kwargs: object) -> VectorStoreSearchResponse:
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=RESULT)
with httpx.Client(transport=httpx.MockTransport(respond)) as transport:
return litellm.vector_stores.search(
**BASE_PARAMS,
vector_store_id="policy_index",
query="travel policy",
custom_llm_provider="mongodb",
extra_body=extra_body,
_direct_vector_store_embedding_executor=RecordingEmbeddingExecutor(),
client=HTTPHandler(client=transport),
**search_kwargs,
)
@pytest.mark.parametrize("drop_params_source", ["request", "global"])
def test_search_with_drop_params_drops_an_extra_body_query_image(
drop_params_source: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global")
drop_kwargs: Final = {"drop_params": True} if drop_params_source == "request" else {}
assert _search_with_extra_body({"query_image": QUERY_IMAGE}, **drop_kwargs) == RESULT
def test_search_with_drop_params_still_rejects_other_extra_body_overrides(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "drop_params", True)
with pytest.raises(litellm.BadRequestError, match="does not support extra_body overrides"):
_search_with_extra_body({"query_image": QUERY_IMAGE, "other": "value"})

View file

@ -16,6 +16,7 @@ from litellm.llms.base_llm.vector_store.transformation import (
RouterVectorStoreEmbeddingExecutor,
)
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
from litellm.vector_stores.search_query import with_caller_query_params
from litellm.proxy.vector_store_endpoints.endpoints import (
_update_request_data_with_litellm_managed_vector_store_registry,
build_request_data_from_managed_vector_store,
@ -3134,6 +3135,17 @@ def test_a_raw_retrieval_query_in_a_stores_extra_body_never_reaches_the_search_r
assert request_data == {"custom_llm_provider": "bedrock", "extra_body": {"kept": "value"}}
@pytest.mark.parametrize("caller_param", ["query_image", "retrievalQuery", "retrieval_query"])
def test_a_callers_query_survives_a_stores_own_extra_body(caller_param: str):
caller_value: Final = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}
request_data: Final = with_caller_query_params(
{"custom_llm_provider": "bedrock", "extra_body": {"retrievalConfiguration": "from the store"}},
{caller_param: caller_value, "retrievalConfiguration": "from the caller", "other": "from the caller"},
)
assert request_data["extra_body"] == {"retrievalConfiguration": "from the store", caller_param: caller_value}
class TestConfigOwnedVectorStores:
"""Stores declared under ``vector_store_registry`` in config.yaml are owned by the config file"""

View file

@ -10270,10 +10270,14 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007)
def test_cost_failure_for_an_unpriced_response_does_not_echo_the_response() -> None:
@pytest.mark.parametrize("message_logging_off", [True, False])
def test_cost_failure_for_an_unpriced_response_echoes_the_response_only_with_message_logging_on(
message_logging_off: bool, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "turn_off_message_logging", message_logging_off)
image_data: Final = "c2VjcmV0LWltYWdlLWJ5dGVz"
with pytest.raises(ValueError, match="Model is None") as exc_info:
completion_cost(completion_response={"data": [{"content": [{"source": {"data": image_data}}]}]})
assert image_data not in str(exc_info.value)
assert (image_data in str(exc_info.value)) is not message_logging_off

View file

@ -1,3 +1,4 @@
import inspect
import json
from collections.abc import Iterator
from typing import Final
@ -160,3 +161,38 @@ async def test_avector_store_retrieve_and_delete_return_provider_responses() ->
assert delete_route.call_count == 1
assert retrieved["id"] == "vs_test123"
assert deleted["deleted"] is True
@pytest.mark.asyncio
@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"])
@pytest.mark.parametrize("endpoint", ["avector_store_search", "vector_store_search"])
async def test_a_deployments_raw_retrieval_query_never_reaches_a_routed_vector_store_search(
endpoint: str, retrieval_query_param: str
) -> None:
from_deployment: Final = {"text": "from the deployment"}
router: Final = Router(
model_list=[
{
"model_name": "kb-route",
"litellm_params": {
"model": "bedrock/kb-route-model",
"aws_access_key_id": "test-access-key",
"aws_secret_access_key": "test-secret-key",
"aws_region_name": "us-east-1",
retrieval_query_param: from_deployment,
"extra_body": {retrieval_query_param: from_deployment},
},
}
]
)
with respx.mock(assert_all_called=True) as mock:
route: Final = mock.post(
"https://bedrock-agent-runtime.us-east-1.amazonaws.com/knowledgebases/kb-id/retrieve"
).mock(return_value=httpx.Response(200, json={"retrievalResults": []}))
result: Final = getattr(router, endpoint)(
model="kb-route", vector_store_id="kb-id", query="caller text", custom_llm_provider="bedrock"
)
if inspect.isawaitable(result):
await result
sent: Final = json.loads(route.calls.last.request.content)
assert sent["retrievalQuery"]["text"] == "caller text"

View file

@ -175,7 +175,11 @@ def test_search_by_query_image_sends_a_bedrock_image_retrieval_query(query: str
),
(
{"custom_llm_provider": "openai", "query_image": QUERY_IMAGE},
"openai vector stores do not support query_image",
"openai vector stores do not support query_image. To drop it",
),
(
{"custom_llm_provider": "openai", "query": "red circle", "query_image": QUERY_IMAGE},
"openai vector stores do not support query_image. To drop it",
),
(
{"custom_llm_provider": "openai", "query": "", "query_image": QUERY_IMAGE, "drop_params": True},

View file

@ -378,11 +378,15 @@ def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_i
@pytest.mark.parametrize(
("tool_field", "tool_value"),
[("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}), ("rewrite_query", True)],
("tool_field", "tool_value", "searched_params"),
[
("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}, {"max_num_results": 3}),
("rewrite_query", True, {"max_num_results": 3, "rewrite_query": True}),
("rewrite_query", False, {"max_num_results": 3, "rewrite_query": False}),
],
)
def test_file_search_tool_with_a_field_the_store_search_does_not_take_still_runs_its_store(
tool_field: str, tool_value: object
def test_file_search_tool_passes_its_store_only_the_fields_a_store_search_takes(
tool_field: str, tool_value: object, searched_params: dict[str, object]
) -> None:
now: Final = datetime.now(timezone.utc)
registry: Final = VectorStoreRegistry(
@ -393,7 +397,7 @@ def test_file_search_tool_with_a_field_the_store_search_does_not_take_still_runs
to_run: Final = registry.pop_vector_stores_to_run(non_default_params={}, tools=tools)
assert [store["vector_store_id"] for store in to_run] == ["vs_kb"]
assert to_run[0]["litellm_params"] == {"max_num_results": 3}
assert to_run[0]["litellm_params"] == searched_params
@pytest.mark.parametrize(