mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(vector_stores): keep a caller's image query through store and router params
A deployment's retrievalQuery no longer reaches a routed search, a caller's query fields survive a store's own extra_body, and an extra_body image stops before the provider. file_search forwards rewrite_query, both retrieval query spellings are redacted, and the cost error stops echoing the response when message logging is off
(cherry picked from commit c65634d4ad)
This commit is contained in:
parent
6e7710c5ab
commit
dddc86ccb4
14 changed files with 173 additions and 19 deletions
|
|
@ -52,6 +52,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
parse_prompt_tokens_details,
|
||||
select_cost_metric_for_model,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
cost_per_token as anthropic_cost_per_token,
|
||||
)
|
||||
|
|
@ -1499,6 +1500,14 @@ def _response_time_ms_for_cost(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _model_is_none_error(completion_response_text: str, litellm_logging_obj: LitellmLoggingObject | None) -> ValueError:
|
||||
message: Final = "Model is None and does not exist in passed completion_response."
|
||||
model_call_details: Final = {} if litellm_logging_obj is None else litellm_logging_obj.model_call_details
|
||||
if should_redact_message_logging(model_call_details):
|
||||
return ValueError(message)
|
||||
return ValueError(f"{message} Passed completion_response={completion_response_text}, model=None")
|
||||
|
||||
|
||||
def completion_cost(
|
||||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
|
|
@ -1751,7 +1760,7 @@ def completion_cost(
|
|||
|
||||
else:
|
||||
if model is None:
|
||||
raise ValueError("Model is None and does not exist in passed completion_response.")
|
||||
raise _model_is_none_error(f"{completion_response}", litellm_logging_obj)
|
||||
if len(messages) > 0:
|
||||
prompt_tokens = token_counter(model=model, messages=messages)
|
||||
elif len(prompt) > 0:
|
||||
|
|
@ -1765,7 +1774,7 @@ def completion_cost(
|
|||
return A2ACostCalculator.calculate_a2a_cost(litellm_logging_obj=litellm_logging_obj)
|
||||
|
||||
if model is None:
|
||||
raise ValueError("Model is None and does not exist in passed completion_response.")
|
||||
raise _model_is_none_error(f"{completion_response}", litellm_logging_obj)
|
||||
if custom_llm_provider is None:
|
||||
try:
|
||||
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
||||
|
|
@ -1985,7 +1994,7 @@ def completion_cost(
|
|||
return get_replicate_completion_pricing(completion_response, response_time_ms)
|
||||
|
||||
if model is None:
|
||||
raise ValueError("Model is None and does not exist in passed completion_response.")
|
||||
raise _model_is_none_error(f"{completion_response}", litellm_logging_obj)
|
||||
|
||||
if custom_llm_provider is not None and custom_llm_provider == "vertex_ai":
|
||||
# Calculate the prompt characters + response characters
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.llms.vertex_ai.common_utils import (
|
|||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -250,7 +251,7 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
|
|||
output_item["input"] = redacted_str
|
||||
|
||||
|
||||
_REDACTED_REQUEST_FIELDS: Final = ("query_image", "retrievalQuery")
|
||||
_REDACTED_REQUEST_FIELDS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS)
|
||||
_NESTED_REQUEST_FIELDS: Final = ("extra_body", "additional_args", "complete_input_dict")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
|
|||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
|
||||
from litellm.proxy.common_utils.http_parsing_utils import read_raw_json_body
|
||||
from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER
|
||||
from litellm.proxy.utils import jsonify_object
|
||||
from litellm.proxy.vector_store_endpoints.utils import (
|
||||
assert_proxy_admin_for_vector_store_index_management,
|
||||
|
|
@ -25,7 +26,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
|
|||
)
|
||||
from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository
|
||||
from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse
|
||||
from litellm.vector_stores.search_query import store_search_params
|
||||
from litellm.vector_stores.search_query import store_search_params, with_caller_query_params
|
||||
from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot
|
||||
from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry
|
||||
|
||||
|
|
@ -138,11 +139,19 @@ async def vector_store_search(
|
|||
|
||||
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
reject_caller_embedding_selection_params(payload=data, source="the search request body")
|
||||
caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data)
|
||||
data["vector_store_id"] = vector_store_id
|
||||
|
||||
# Check for legacy vector store registry (non-managed vector stores)
|
||||
data = await _update_request_data_with_litellm_managed_vector_store_registry(
|
||||
data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict
|
||||
data = dict( # rebind-ok: pre-existing rebinding of the request body
|
||||
with_caller_query_params(
|
||||
JSON_OBJECT_ADAPTER.validate_python(
|
||||
await _update_request_data_with_litellm_managed_vector_store_registry(
|
||||
data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict
|
||||
)
|
||||
),
|
||||
caller_body.get("extra_body"),
|
||||
)
|
||||
)
|
||||
|
||||
# The managed_vector_stores pre-call hook will handle:
|
||||
|
|
|
|||
|
|
@ -391,6 +391,7 @@ from litellm.utils import (
|
|||
provider_rejectable_params,
|
||||
set_live_deployment_replay,
|
||||
)
|
||||
from litellm.vector_stores.search_query import store_search_params
|
||||
|
||||
from .router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
|
|
@ -2307,6 +2308,7 @@ class Router:
|
|||
update,
|
||||
)
|
||||
|
||||
self._vector_store_search_handlers: tuple[Callable[..., object], ...] = (asearch, search)
|
||||
self.avector_store_search = self.factory_function(asearch, call_type="avector_store_search")
|
||||
self.vector_store_search = self.factory_function(search, call_type="vector_store_search")
|
||||
self.vector_store_create = self.factory_function(create, call_type="vector_store_create")
|
||||
|
|
@ -5753,7 +5755,11 @@ class Router:
|
|||
custom_llm_provider: Final = provider_for_generic_call(data)
|
||||
|
||||
response_kwargs: Final = {
|
||||
**data,
|
||||
**(
|
||||
store_search_params(_LITELLM_PARAMS_ADAPTER.validate_python(data))
|
||||
if original_generic_function in self._vector_store_search_handlers
|
||||
else data
|
||||
),
|
||||
"caching": self.cache_responses,
|
||||
**kwargs,
|
||||
"model": model_name,
|
||||
|
|
@ -6496,7 +6502,11 @@ class Router:
|
|||
|
||||
response: Final = original_function(
|
||||
**{
|
||||
**data,
|
||||
**(
|
||||
store_search_params(_LITELLM_PARAMS_ADAPTER.validate_python(data))
|
||||
if original_function in self._vector_store_search_handlers
|
||||
else data
|
||||
),
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
"caching": self.cache_responses,
|
||||
**kwargs,
|
||||
|
|
|
|||
|
|
@ -352,6 +352,7 @@ class VectorStoreToolParams:
|
|||
filters: dict | None = None
|
||||
max_num_results: int | None = None
|
||||
ranking_options: dict | None = None
|
||||
rewrite_query: bool | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Convert to dict, excluding None values"""
|
||||
|
|
@ -361,6 +362,7 @@ class VectorStoreToolParams:
|
|||
"filters": self.filters,
|
||||
"max_num_results": self.max_num_results,
|
||||
"ranking_options": self.ranking_options,
|
||||
"rewrite_query": self.rewrite_query,
|
||||
}.items()
|
||||
if v is not None
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm.constants import request_timeout
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.vector_store.transformation import (
|
||||
|
|
@ -464,6 +464,8 @@ def search(
|
|||
)
|
||||
)
|
||||
|
||||
sent_extra_body: Final = as_str_mapping(extra_body)
|
||||
|
||||
# Pre Call logging
|
||||
litellm_logging_obj.update_from_kwargs(
|
||||
kwargs=kwargs,
|
||||
|
|
@ -491,7 +493,9 @@ def search(
|
|||
logging_obj=litellm_logging_obj,
|
||||
embedding_executor=embedding_executor,
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
extra_body=extra_body
|
||||
if sent_extra_body is None
|
||||
else {key: value for key, value in sent_extra_body.items() if key != "query_image"},
|
||||
timeout=timeout or request_timeout,
|
||||
_is_async=_is_async,
|
||||
client=kwargs.get("client"),
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.types.vector_stores import RETRIEVAL_QUERY_PARAMS, VectorStoreSearc
|
|||
|
||||
_QUERY_IMAGE: Final = TypeAdapter(VectorStoreSearchQueryImage)
|
||||
_Query: Final = TypeVar("_Query", bound=str | Sequence[str])
|
||||
_CALLER_QUERY_PARAMS: Final = ("query_image", *RETRIEVAL_QUERY_PARAMS)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -118,6 +119,15 @@ def store_search_params(litellm_params: Mapping[str, object]) -> Mapping[str, ob
|
|||
)
|
||||
|
||||
|
||||
def with_caller_query_params(request: Mapping[str, object], caller_extra_body: object) -> Mapping[str, object]:
|
||||
caller: Final = as_str_mapping(caller_extra_body)
|
||||
store: Final = as_str_mapping(request.get("extra_body"))
|
||||
if caller is None or store is None:
|
||||
return request
|
||||
query_params: Final = {key: caller[key] for key in _CALLER_QUERY_PARAMS if key in caller}
|
||||
return {**request, "extra_body": {**store, **query_params}}
|
||||
|
||||
|
||||
def resolve_search_query(
|
||||
query: _Query | None,
|
||||
query_image: object,
|
||||
|
|
|
|||
|
|
@ -1536,6 +1536,18 @@ def test_redaction_removes_a_query_image_sent_inside_extra_body() -> None:
|
|||
assert SECRET not in json.dumps(details, default=str)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"])
|
||||
def test_redaction_removes_a_raw_retrieval_query_sent_inside_extra_body(retrieval_query_param: str) -> None:
|
||||
retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}}
|
||||
details: Final[dict[str, object]] = {
|
||||
"litellm_params": {"proxy_server_request": {"body": {"extra_body": {retrieval_query_param: retrieval_query}}}},
|
||||
}
|
||||
|
||||
perform_redaction(details, result=None)
|
||||
|
||||
assert SECRET not in json.dumps(details, default=str)
|
||||
|
||||
|
||||
def test_redaction_removes_the_image_from_the_provider_request_copy() -> None:
|
||||
retrieval_query: Final = {"type": "IMAGE", "image": {"format": "jpeg", "inlineContent": SECRET}}
|
||||
details: Final[dict[str, object]] = {
|
||||
|
|
|
|||
|
|
@ -220,3 +220,40 @@ async def test_public_sdk_preserves_http_errors_response_and_timeout(
|
|||
else:
|
||||
assert await search() == RESULT
|
||||
executor.call.assert_called_once_with("embedding-alias", "travel policy", {})
|
||||
|
||||
|
||||
QUERY_IMAGE: Final = {"type": "base64", "media_type": "image/jpeg", "data": "/9j/4AAQSkZJRg=="}
|
||||
|
||||
|
||||
def _search_with_extra_body(extra_body: dict[str, object], **search_kwargs: object) -> VectorStoreSearchResponse:
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=RESULT)
|
||||
|
||||
with httpx.Client(transport=httpx.MockTransport(respond)) as transport:
|
||||
return litellm.vector_stores.search(
|
||||
**BASE_PARAMS,
|
||||
vector_store_id="policy_index",
|
||||
query="travel policy",
|
||||
custom_llm_provider="mongodb",
|
||||
extra_body=extra_body,
|
||||
_direct_vector_store_embedding_executor=RecordingEmbeddingExecutor(),
|
||||
client=HTTPHandler(client=transport),
|
||||
**search_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params_source", ["request", "global"])
|
||||
def test_search_with_drop_params_drops_an_extra_body_query_image(
|
||||
drop_params_source: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", drop_params_source == "global")
|
||||
drop_kwargs: Final = {"drop_params": True} if drop_params_source == "request" else {}
|
||||
|
||||
assert _search_with_extra_body({"query_image": QUERY_IMAGE}, **drop_kwargs) == RESULT
|
||||
|
||||
|
||||
def test_search_with_drop_params_still_rejects_other_extra_body_overrides(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="does not support extra_body overrides"):
|
||||
_search_with_extra_body({"query_image": QUERY_IMAGE, "other": "value"})
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.llms.base_llm.vector_store.transformation import (
|
|||
RouterVectorStoreEmbeddingExecutor,
|
||||
)
|
||||
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.vector_stores.search_query import with_caller_query_params
|
||||
from litellm.proxy.vector_store_endpoints.endpoints import (
|
||||
_update_request_data_with_litellm_managed_vector_store_registry,
|
||||
build_request_data_from_managed_vector_store,
|
||||
|
|
@ -3134,6 +3135,17 @@ def test_a_raw_retrieval_query_in_a_stores_extra_body_never_reaches_the_search_r
|
|||
assert request_data == {"custom_llm_provider": "bedrock", "extra_body": {"kept": "value"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("caller_param", ["query_image", "retrievalQuery", "retrieval_query"])
|
||||
def test_a_callers_query_survives_a_stores_own_extra_body(caller_param: str):
|
||||
caller_value: Final = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}
|
||||
request_data: Final = with_caller_query_params(
|
||||
{"custom_llm_provider": "bedrock", "extra_body": {"retrievalConfiguration": "from the store"}},
|
||||
{caller_param: caller_value, "retrievalConfiguration": "from the caller", "other": "from the caller"},
|
||||
)
|
||||
|
||||
assert request_data["extra_body"] == {"retrievalConfiguration": "from the store", caller_param: caller_value}
|
||||
|
||||
|
||||
class TestConfigOwnedVectorStores:
|
||||
"""Stores declared under ``vector_store_registry`` in config.yaml are owned by the config file"""
|
||||
|
||||
|
|
|
|||
|
|
@ -10270,10 +10270,14 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
|
|||
assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007)
|
||||
|
||||
|
||||
def test_cost_failure_for_an_unpriced_response_does_not_echo_the_response() -> None:
|
||||
@pytest.mark.parametrize("message_logging_off", [True, False])
|
||||
def test_cost_failure_for_an_unpriced_response_echoes_the_response_only_with_message_logging_on(
|
||||
message_logging_off: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "turn_off_message_logging", message_logging_off)
|
||||
image_data: Final = "c2VjcmV0LWltYWdlLWJ5dGVz"
|
||||
|
||||
with pytest.raises(ValueError, match="Model is None") as exc_info:
|
||||
completion_cost(completion_response={"data": [{"content": [{"source": {"data": image_data}}]}]})
|
||||
|
||||
assert image_data not in str(exc_info.value)
|
||||
assert (image_data in str(exc_info.value)) is not message_logging_off
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import inspect
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
|
@ -160,3 +161,38 @@ async def test_avector_store_retrieve_and_delete_return_provider_responses() ->
|
|||
assert delete_route.call_count == 1
|
||||
assert retrieved["id"] == "vs_test123"
|
||||
assert deleted["deleted"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("retrieval_query_param", ["retrievalQuery", "retrieval_query"])
|
||||
@pytest.mark.parametrize("endpoint", ["avector_store_search", "vector_store_search"])
|
||||
async def test_a_deployments_raw_retrieval_query_never_reaches_a_routed_vector_store_search(
|
||||
endpoint: str, retrieval_query_param: str
|
||||
) -> None:
|
||||
from_deployment: Final = {"text": "from the deployment"}
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "kb-route",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/kb-route-model",
|
||||
"aws_access_key_id": "test-access-key",
|
||||
"aws_secret_access_key": "test-secret-key",
|
||||
"aws_region_name": "us-east-1",
|
||||
retrieval_query_param: from_deployment,
|
||||
"extra_body": {retrieval_query_param: from_deployment},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
with respx.mock(assert_all_called=True) as mock:
|
||||
route: Final = mock.post(
|
||||
"https://bedrock-agent-runtime.us-east-1.amazonaws.com/knowledgebases/kb-id/retrieve"
|
||||
).mock(return_value=httpx.Response(200, json={"retrievalResults": []}))
|
||||
result: Final = getattr(router, endpoint)(
|
||||
model="kb-route", vector_store_id="kb-id", query="caller text", custom_llm_provider="bedrock"
|
||||
)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert sent["retrievalQuery"]["text"] == "caller text"
|
||||
|
|
|
|||
|
|
@ -175,7 +175,11 @@ def test_search_by_query_image_sends_a_bedrock_image_retrieval_query(query: str
|
|||
),
|
||||
(
|
||||
{"custom_llm_provider": "openai", "query_image": QUERY_IMAGE},
|
||||
"openai vector stores do not support query_image",
|
||||
"openai vector stores do not support query_image. To drop it",
|
||||
),
|
||||
(
|
||||
{"custom_llm_provider": "openai", "query": "red circle", "query_image": QUERY_IMAGE},
|
||||
"openai vector stores do not support query_image. To drop it",
|
||||
),
|
||||
(
|
||||
{"custom_llm_provider": "openai", "query": "", "query_image": QUERY_IMAGE, "drop_params": True},
|
||||
|
|
|
|||
|
|
@ -378,11 +378,15 @@ def test_vector_store_ids_to_run_reads_every_store_id_a_malformed_vector_store_i
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_field", "tool_value"),
|
||||
[("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}), ("rewrite_query", True)],
|
||||
("tool_field", "tool_value", "searched_params"),
|
||||
[
|
||||
("query_image", {"type": "base64", "media_type": "image/jpeg", "data": "/9j/"}, {"max_num_results": 3}),
|
||||
("rewrite_query", True, {"max_num_results": 3, "rewrite_query": True}),
|
||||
("rewrite_query", False, {"max_num_results": 3, "rewrite_query": False}),
|
||||
],
|
||||
)
|
||||
def test_file_search_tool_with_a_field_the_store_search_does_not_take_still_runs_its_store(
|
||||
tool_field: str, tool_value: object
|
||||
def test_file_search_tool_passes_its_store_only_the_fields_a_store_search_takes(
|
||||
tool_field: str, tool_value: object, searched_params: dict[str, object]
|
||||
) -> None:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
registry: Final = VectorStoreRegistry(
|
||||
|
|
@ -393,7 +397,7 @@ def test_file_search_tool_with_a_field_the_store_search_does_not_take_still_runs
|
|||
to_run: Final = registry.pop_vector_stores_to_run(non_default_params={}, tools=tools)
|
||||
|
||||
assert [store["vector_store_id"] for store in to_run] == ["vs_kb"]
|
||||
assert to_run[0]["litellm_params"] == {"max_num_results": 3}
|
||||
assert to_run[0]["litellm_params"] == searched_params
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue