mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
* fix(responses): log and bill every round of non-stream MCP auto-execute Both model rounds run as internal sub-calls on the request's logging object, and the outer call logs the response the client received with usage, cost, and cost breakdown summed across the rounds. The internal-call gate now also covers the sync success callbacks, so an inner round no longer takes their first-wins slot * fix(caching): let the parent call log and bill an internal sub-call's cache hit A round of non-stream MCP auto-execute that the response cache served fired the cache-hit callbacks on the shared logging object, so the spend log kept round 1's function_call row and dropped the billed row for the answer. Internal sub-call cache hits now fire no callbacks and carry a zero cost, and the summed cost breakdown leaves out rounds served from the cache * fix(logging): keep a parent row unique when its response came from the response cache A RAG query or an MCP turn whose final round is served from the cache hands back the cached response's id, which already belongs to the row that cached it, so SpendLogs dropped the new row as a duplicate. Such a row now gets the cache-hit id suffix, and it is logged as a cache hit when it cost nothing. * fix(logging): skip the zero-cost warning for a response served from the cache at no charge A non-stream MCP auto-execute request whose rounds the response cache served is priced at $0 before the parent marks it a cache hit, so it logged a false 'priced at $0' warning and counted in litellm_zero_cost_requests_total. The diagnostic now treats a response the cache served at no charge the same as a marked cache hit * fix(responses): bill the paid first MCP round when a later step of the request fails * fix(responses): log each MCP tool call under its own id so a failed follow-up keeps its spend row The non-stream auto-execute path passed the request's call id to the tool call, so the tool's spend row and the request's failure row shared one request_id and the failure row was dropped on insert, leaving the paid first round unbilled. * fix(responses): bill round 1 on a client disconnect, keep MCP tool rows in the request's session, and charge a fixed margin once per request * fix(responses): keep bridged MCP auto-execute in one session and trace and continue its conversation The bridged follow-up no longer chains on round 1, so a thinking model keeps its reasoning and the request stays in one session. The tool call log continues the request trace without renaming it, the tools are fetched once, and a later turn finds the MCP turn under the id the client got, skipping the tool rows. * fix(responses): log the non-stream MCP tool listing in the request's session and trace * fix(responses): find a bridged MCP turn by the id the client holds and log its listing in the bridged trace * fix(responses): derive the MCP tool row id from the request's call id The tool call's spend row now uses <call id>_<tool call id>, so it stays linked to the request without colliding with the request's failure row. Also tighten the internal cache-hit test to a fixed clock and typed params. * fix(responses): bill a retried MCP attempt for its own rounds A router retry or fallback reuses the request's logging object, so a failed attempt's stashed round 1 usage made the next attempt's success row log stale tokens. Each non-stream MCP attempt now starts without that stash. * fix(responses): bill every failed MCP attempt's paid first round on the request failure A retry or fallback now sets a failed attempt's stashed round 1 aside instead of dropping it, and the proxy's failure row adds it to the last attempt's own, so a request whose later attempt fails on its first call still bills the round the provider charged. A cached final round also keeps the paid first round's cost breakdown. * fix(responses): keep MCP tool catalog and results out of the bridged next-turn replay * fix(responses): keep MCP tool spend, client body keys, and Langfuse traces working on auto-execute Count the MCP tools an auto-executed Responses API call ran in the tool spend rollup, accept a client body key named previous_response_client_ids without a 500, and point MCP listing and tool spans at the trace Langfuse gives the parent request, including x-litellm-session-id requests * fix(responses): join MCP spans to the Langfuse trace when the request sends body metadata * test(integration): cover non-stream MCP auto-execute billing, failure rows, and bridged follow-ups --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
516 lines
17 KiB
Python
516 lines
17 KiB
Python
"""
|
|
RAG Ingest API for LiteLLM.
|
|
|
|
Provides an all-in-one API for document ingestion:
|
|
Upload -> (OCR) -> Chunk -> Embed -> Vector Store
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
__all__ = ["aingest", "aquery", "ingest", "query"]
|
|
|
|
import asyncio
|
|
import contextvars
|
|
from collections.abc import Coroutine, Mapping
|
|
from functools import partial
|
|
from types import MappingProxyType
|
|
from typing import TYPE_CHECKING, Any, Final
|
|
|
|
import httpx
|
|
from pydantic import ConfigDict, TypeAdapter
|
|
|
|
import litellm
|
|
from litellm._internal_context import internal_sub_call
|
|
from litellm.cost_calculator import vector_store_search_cost
|
|
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
|
from litellm.rag.ingestion.base_ingestion import BaseRAGIngestion
|
|
from litellm.rag.ingestion.bedrock_ingestion import BedrockRAGIngestion
|
|
from litellm.rag.ingestion.gemini_ingestion import GeminiRAGIngestion
|
|
from litellm.rag.ingestion.openai_ingestion import OpenAIRAGIngestion
|
|
from litellm.rag.ingestion.s3_vectors_ingestion import S3VectorsRAGIngestion
|
|
from litellm.rag.ingestion.vertex_ai_ingestion import VertexAIRAGIngestion
|
|
from litellm.rag.rag_query import RAGQuery
|
|
from litellm.types.llms.openai import AllMessageValues
|
|
from litellm.types.rag import (
|
|
RAGIngestOptions,
|
|
RAGIngestResponse,
|
|
)
|
|
from litellm.types.utils import ModelResponse
|
|
from litellm.utils import client
|
|
|
|
if TYPE_CHECKING:
|
|
from litellm import Router
|
|
|
|
|
|
# Registry of provider-specific ingestion classes
|
|
INGESTION_REGISTRY: Final[dict[str, type[BaseRAGIngestion]]] = {
|
|
"openai": OpenAIRAGIngestion,
|
|
"bedrock": BedrockRAGIngestion,
|
|
"gemini": GeminiRAGIngestion,
|
|
"s3_vectors": S3VectorsRAGIngestion,
|
|
"vertex_ai": VertexAIRAGIngestion,
|
|
}
|
|
|
|
# Only these retrieval_config keys are forwarded to vector_stores.asearch as
|
|
# provider-specific params. The explicit allowlist keeps caller-controlled
|
|
# connection overrides (api_base, api_key, ...) away from the search call,
|
|
# where they could redirect store credentials to an attacker-chosen host.
|
|
_FORWARDABLE_RETRIEVAL_CONFIG_KEYS: Final = frozenset(
|
|
{
|
|
"aws_region_name",
|
|
"vector_bucket_name",
|
|
"embedding_model",
|
|
"litellm_embedding_model",
|
|
"litellm_embedding_config",
|
|
"litellm_credential_name",
|
|
}
|
|
)
|
|
|
|
_SEARCH_ARGS_SET_BY_PIPELINE: Final = frozenset(
|
|
{"vector_store_id", "query", "max_num_results", "custom_llm_provider", "router"}
|
|
)
|
|
|
|
|
|
_VECTOR_STORE_OPTIONS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
|
|
|
|
|
|
def _vector_store_provider(ingest_options: Mapping[str, object]) -> object:
|
|
return _VECTOR_STORE_OPTIONS.validate_python(ingest_options.get("vector_store", {})).get("custom_llm_provider")
|
|
|
|
|
|
def get_ingestion_class(provider: str) -> type[BaseRAGIngestion]:
|
|
"""
|
|
Get the ingestion class for a given provider.
|
|
|
|
Args:
|
|
provider: The vector store provider name (e.g., 'openai')
|
|
|
|
Returns:
|
|
The ingestion class for the provider
|
|
|
|
Raises:
|
|
ValueError: If provider is not supported
|
|
"""
|
|
ingestion_class: Final = INGESTION_REGISTRY.get(provider)
|
|
if ingestion_class is None:
|
|
supported: Final = ", ".join(INGESTION_REGISTRY.keys())
|
|
raise ValueError(f"Provider '{provider}' is not supported for RAG ingestion. Supported providers: {supported}")
|
|
return ingestion_class
|
|
|
|
|
|
async def _execute_ingest_pipeline(
|
|
ingest_options: RAGIngestOptions,
|
|
file_data: tuple[str, bytes, str] | None = None,
|
|
file_url: str | None = None,
|
|
file_id: str | None = None,
|
|
router: Router | None = None,
|
|
) -> RAGIngestResponse:
|
|
"""
|
|
Execute the RAG ingest pipeline using provider-specific implementation.
|
|
|
|
Args:
|
|
ingest_options: Configuration for the ingest pipeline
|
|
file_data: Tuple of (filename, content_bytes, content_type)
|
|
file_url: URL to fetch file from
|
|
file_id: Existing file ID to use
|
|
router: Optional LiteLLM router for load balancing
|
|
|
|
Returns:
|
|
RAGIngestResponse with status and IDs
|
|
"""
|
|
# Get provider from vector store config
|
|
vector_store_config: Final = ingest_options.get("vector_store") or {}
|
|
provider: Final = vector_store_config.get("custom_llm_provider", "openai")
|
|
|
|
# Get provider-specific ingestion class
|
|
ingestion_class: Final = get_ingestion_class(provider)
|
|
|
|
# Create ingestion instance
|
|
ingestion: Final = ingestion_class(
|
|
ingest_options=ingest_options,
|
|
router=router,
|
|
)
|
|
|
|
# Execute ingestion pipeline
|
|
return await ingestion.ingest(
|
|
file_data=file_data,
|
|
file_url=file_url,
|
|
file_id=file_id,
|
|
)
|
|
|
|
|
|
####### PUBLIC API ###################
|
|
|
|
|
|
@client
|
|
async def aingest(
|
|
ingest_options: Mapping[str, object],
|
|
file_data: tuple[str, bytes, str] | None = None,
|
|
file: dict[str, str] | None = None,
|
|
file_url: str | None = None,
|
|
file_id: str | None = None,
|
|
timeout: float | httpx.Timeout | None = None,
|
|
**kwargs,
|
|
) -> RAGIngestResponse:
|
|
"""
|
|
Async: Ingest a document into a vector store.
|
|
|
|
Args:
|
|
ingest_options: Configuration for the ingest pipeline
|
|
file_data: Tuple of (filename, content_bytes, content_type)
|
|
file: Dict with {filename, content (base64), content_type} - for JSON API
|
|
file_url: URL to fetch file from
|
|
file_id: Existing file ID to use
|
|
|
|
Example:
|
|
```python
|
|
response = await litellm.aingest(
|
|
ingest_options={
|
|
"vector_store": {
|
|
"custom_llm_provider": "openai",
|
|
"litellm_credential_name": "my-openai-creds", # optional
|
|
}
|
|
},
|
|
file_url="https://example.com/doc.pdf",
|
|
)
|
|
```
|
|
"""
|
|
local_vars: Final = locals()
|
|
try:
|
|
loop: Final = asyncio.get_event_loop()
|
|
kwargs["aingest"] = True
|
|
|
|
func: Final = partial(
|
|
ingest,
|
|
ingest_options=ingest_options,
|
|
file_data=file_data,
|
|
file=file,
|
|
file_url=file_url,
|
|
file_id=file_id,
|
|
timeout=timeout,
|
|
**kwargs,
|
|
)
|
|
|
|
ctx: Final = contextvars.copy_context()
|
|
func_with_context: Final = partial(ctx.run, func)
|
|
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
|
|
|
if asyncio.iscoroutine(init_response):
|
|
response = await init_response
|
|
else:
|
|
response = init_response
|
|
|
|
return response
|
|
except Exception as e:
|
|
raise litellm.exception_type(
|
|
model=None,
|
|
custom_llm_provider=_vector_store_provider(ingest_options),
|
|
original_exception=e,
|
|
completion_kwargs=local_vars,
|
|
extra_kwargs=kwargs,
|
|
)
|
|
|
|
|
|
async def _execute_query_pipeline(
|
|
model: str,
|
|
messages: list[AllMessageValues],
|
|
retrieval_config: Mapping[str, object],
|
|
rerank: dict[str, Any] | None = None,
|
|
stream: bool = False,
|
|
vector_store_params: Mapping[str, object] | None = None,
|
|
**kwargs,
|
|
) -> ModelResponse:
|
|
"""
|
|
Execute the RAG query pipeline.
|
|
"""
|
|
# Extract router from kwargs - use it for completion if available
|
|
# to properly resolve virtual model names
|
|
router: Final[Router | None] = kwargs.pop("router", None)
|
|
|
|
# 1. Extract query from last user message
|
|
query_text: Final = RAGQuery.extract_query_from_messages(messages)
|
|
if not query_text:
|
|
raise ValueError("No query found in messages for RAG query")
|
|
|
|
# 2. Search vector store
|
|
top_level_filters: Final = kwargs.pop("filters", None)
|
|
filters: Final = retrieval_config.get("retrieval_filter") or retrieval_config.get("filters") or top_level_filters
|
|
filter_search_params: Final = MappingProxyType({"filters": filters} if filters else {})
|
|
# Forward allowlisted provider retrieval_config extras (region, embedding
|
|
# model, bucket, credential refs) to the search call; the managed store's
|
|
# params win on conflict.
|
|
provider_search_params: Final = MappingProxyType(
|
|
{k: v for k, v in retrieval_config.items() if k in _FORWARDABLE_RETRIEVAL_CONFIG_KEYS}
|
|
)
|
|
store_search_params: Final = MappingProxyType(
|
|
{
|
|
k: v
|
|
for k, v in (vector_store_params.items() if vector_store_params else ())
|
|
if k not in _SEARCH_ARGS_SET_BY_PIPELINE
|
|
}
|
|
)
|
|
forwarded_search_params: Final = MappingProxyType(
|
|
{**provider_search_params, **kwargs, **filter_search_params, **store_search_params}
|
|
)
|
|
with internal_sub_call():
|
|
search_response: Final = await litellm.vector_stores.asearch(
|
|
vector_store_id=retrieval_config["vector_store_id"],
|
|
query=query_text,
|
|
max_num_results=retrieval_config.get("top_k", 10),
|
|
custom_llm_provider=retrieval_config.get("custom_llm_provider", "openai"),
|
|
router=router,
|
|
**forwarded_search_params,
|
|
)
|
|
|
|
search_provider: Final = retrieval_config.get("custom_llm_provider", "openai")
|
|
try:
|
|
search_cost = (
|
|
sum(
|
|
vector_store_search_cost(
|
|
model=search_provider if "/" in search_provider else None,
|
|
custom_llm_provider=search_provider,
|
|
response=search_response,
|
|
)
|
|
)
|
|
if isinstance(search_provider, str)
|
|
else 0.0
|
|
)
|
|
except Exception: # noqa: BLE001 - cost accounting must never break the query path
|
|
search_cost = 0.0
|
|
|
|
rerank_response = None
|
|
rerank_cost = 0.0
|
|
context_chunks = search_response.get("data", [])
|
|
|
|
# 3. Optional rerank
|
|
if rerank and rerank.get("enabled"):
|
|
documents: Final = RAGQuery.extract_documents_from_search(search_response)
|
|
if documents:
|
|
with internal_sub_call():
|
|
rerank_response = await litellm.arerank(
|
|
model=rerank["model"],
|
|
query=query_text,
|
|
documents=documents,
|
|
top_n=rerank.get("top_n", 5),
|
|
)
|
|
rerank_hidden_params: Final = getattr(rerank_response, "_hidden_params", None)
|
|
if isinstance(rerank_hidden_params, dict):
|
|
rerank_response_cost: Final[float | None] = rerank_hidden_params.get("response_cost")
|
|
rerank_cost = rerank_response_cost or 0.0
|
|
context_chunks = RAGQuery.get_top_chunks_from_rerank(search_response, rerank_response)
|
|
|
|
# 4. Build context message and call completion
|
|
context_message: Final = RAGQuery.build_context_message(context_chunks)
|
|
modified_messages: Final = messages[:-1] + [context_message] + [messages[-1]]
|
|
|
|
# Use router if available to properly resolve virtual model names
|
|
with internal_sub_call():
|
|
if router is not None:
|
|
response = await router.acompletion(
|
|
model=model,
|
|
messages=modified_messages,
|
|
stream=stream,
|
|
**kwargs,
|
|
)
|
|
else:
|
|
response = await litellm.acompletion(
|
|
model=model,
|
|
messages=modified_messages,
|
|
stream=stream,
|
|
**kwargs,
|
|
)
|
|
|
|
# 5. Attach search results to response
|
|
sub_call_cost: Final = search_cost + rerank_cost
|
|
if not stream and isinstance(response, ModelResponse):
|
|
response = RAGQuery.add_search_results_to_response(
|
|
response=response,
|
|
search_results=search_response,
|
|
rerank_results=rerank_response,
|
|
)
|
|
if sub_call_cost > 0:
|
|
hidden_params: Final = getattr(response, "_hidden_params", None)
|
|
if isinstance(hidden_params, dict):
|
|
completion_response_cost: Final[float | None] = hidden_params.get("response_cost")
|
|
if completion_response_cost is not None:
|
|
hidden_params["response_cost"] = completion_response_cost + sub_call_cost
|
|
elif sub_call_cost > 0:
|
|
logging_obj: Final[object] = kwargs.get("litellm_logging_obj")
|
|
if isinstance(logging_obj, LiteLLMLoggingObj):
|
|
logging_obj.model_call_details["additional_response_cost"] = sub_call_cost
|
|
|
|
return response
|
|
|
|
|
|
@client
|
|
async def aquery(
|
|
model: str,
|
|
messages: list[AllMessageValues],
|
|
retrieval_config: Mapping[str, object],
|
|
rerank: dict[str, Any] | None = None,
|
|
stream: bool = False,
|
|
vector_store_params: Mapping[str, object] | None = None,
|
|
**kwargs,
|
|
) -> ModelResponse:
|
|
"""
|
|
Async: Query a RAG pipeline.
|
|
"""
|
|
local_vars: Final = locals()
|
|
try:
|
|
loop: Final = asyncio.get_event_loop()
|
|
kwargs["aquery"] = True
|
|
|
|
func: Final = partial(
|
|
query,
|
|
model=model,
|
|
messages=messages,
|
|
retrieval_config=retrieval_config,
|
|
rerank=rerank,
|
|
stream=stream,
|
|
vector_store_params=vector_store_params,
|
|
**kwargs,
|
|
)
|
|
|
|
ctx: Final = contextvars.copy_context()
|
|
func_with_context: Final = partial(ctx.run, func)
|
|
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
|
|
|
if asyncio.iscoroutine(init_response):
|
|
response = await init_response
|
|
else:
|
|
response = init_response
|
|
|
|
return response
|
|
except Exception as e:
|
|
raise litellm.exception_type(
|
|
model=model,
|
|
custom_llm_provider=retrieval_config.get("custom_llm_provider"),
|
|
original_exception=e,
|
|
completion_kwargs=local_vars,
|
|
extra_kwargs=kwargs,
|
|
)
|
|
|
|
|
|
@client
|
|
def query(
|
|
model: str,
|
|
messages: list[AllMessageValues],
|
|
retrieval_config: Mapping[str, object],
|
|
rerank: dict[str, Any] | None = None,
|
|
stream: bool = False,
|
|
vector_store_params: Mapping[str, object] | None = None,
|
|
**kwargs,
|
|
) -> ModelResponse | Coroutine[None, None, ModelResponse]:
|
|
"""
|
|
Query a RAG pipeline.
|
|
"""
|
|
local_vars: Final = locals()
|
|
try:
|
|
_is_async: Final = kwargs.pop("aquery", False) is True
|
|
|
|
if _is_async:
|
|
return _execute_query_pipeline(
|
|
model=model,
|
|
messages=messages,
|
|
retrieval_config=retrieval_config,
|
|
rerank=rerank,
|
|
stream=stream,
|
|
vector_store_params=vector_store_params,
|
|
**kwargs,
|
|
)
|
|
else:
|
|
return asyncio.get_event_loop().run_until_complete(
|
|
_execute_query_pipeline(
|
|
model=model,
|
|
messages=messages,
|
|
retrieval_config=retrieval_config,
|
|
rerank=rerank,
|
|
stream=stream,
|
|
vector_store_params=vector_store_params,
|
|
**kwargs,
|
|
)
|
|
)
|
|
except Exception as e:
|
|
raise litellm.exception_type(
|
|
model=model,
|
|
custom_llm_provider=retrieval_config.get("custom_llm_provider"),
|
|
original_exception=e,
|
|
completion_kwargs=local_vars,
|
|
extra_kwargs=kwargs,
|
|
)
|
|
|
|
|
|
@client
|
|
def ingest(
|
|
ingest_options: Mapping[str, object],
|
|
file_data: tuple[str, bytes, str] | None = None,
|
|
file: dict[str, str] | None = None,
|
|
file_url: str | None = None,
|
|
file_id: str | None = None,
|
|
timeout: float | httpx.Timeout | None = None,
|
|
**kwargs,
|
|
) -> RAGIngestResponse | Coroutine[None, None, RAGIngestResponse]:
|
|
"""
|
|
Ingest a document into a vector store.
|
|
|
|
Args:
|
|
ingest_options: Configuration for the ingest pipeline
|
|
file_data: Tuple of (filename, content_bytes, content_type)
|
|
file: Dict with {filename, content (base64), content_type} - for JSON API
|
|
file_url: URL to fetch file from
|
|
file_id: Existing file ID to use
|
|
|
|
Example:
|
|
```python
|
|
response = litellm.ingest(
|
|
ingest_options={
|
|
"vector_store": {
|
|
"custom_llm_provider": "openai",
|
|
"litellm_credential_name": "my-openai-creds", # optional
|
|
}
|
|
},
|
|
file_data=("doc.txt", b"Hello world", "text/plain"),
|
|
)
|
|
```
|
|
"""
|
|
import base64
|
|
|
|
local_vars: Final = locals()
|
|
try:
|
|
_is_async: Final = kwargs.pop("aingest", False) is True
|
|
router: Final[Router | None] = kwargs.get("router")
|
|
|
|
# Convert file dict to file_data tuple if provided
|
|
if file is not None and file_data is None:
|
|
filename: Final = file.get("filename", "document")
|
|
content_b64: Final = file.get("content", "")
|
|
content_type: Final = file.get("content_type", "application/octet-stream")
|
|
content_bytes: Final = base64.b64decode(content_b64)
|
|
file_data = (filename, content_bytes, content_type)
|
|
|
|
if _is_async:
|
|
return _execute_ingest_pipeline(
|
|
ingest_options=ingest_options,
|
|
file_data=file_data,
|
|
file_url=file_url,
|
|
file_id=file_id,
|
|
router=router,
|
|
)
|
|
else:
|
|
return asyncio.get_event_loop().run_until_complete(
|
|
_execute_ingest_pipeline(
|
|
ingest_options=ingest_options,
|
|
file_data=file_data,
|
|
file_url=file_url,
|
|
file_id=file_id,
|
|
router=router,
|
|
)
|
|
)
|
|
except Exception as e:
|
|
raise litellm.exception_type(
|
|
model=None,
|
|
custom_llm_provider=_vector_store_provider(ingest_options),
|
|
original_exception=e,
|
|
completion_kwargs=local_vars,
|
|
extra_kwargs=kwargs,
|
|
)
|