mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-19 00:01:29 +00:00
Add benchmark: semantic code search (claude-context) vs. grep for codebase Q&A
Benchmarks two retrieval approaches for answering natural language queries about the LiteLLM codebase: 1. Semantic Search (claude-context style): sentence-transformer embeddings + Milvus vector DB with AST-based code chunking 2. Grep-based Search: ripgrep keyword matching with file ranking and snippet extraction Results across 10 diverse queries (scored by Claude as judge): - Grep: 19.70/20 avg, Semantic: 18.30/20 avg - Grep wins 5/10, Semantic wins 1/10, Ties 4/10 - Semantic uses 3.2x fewer context tokens (3668 vs 11577 avg) - Semantic search is 6.5x faster per query (0.02s vs 0.13s) - But requires 132s upfront indexing cost Co-authored-by: Krrish Dholakia <krrish-berri-2@users.noreply.github.com>
This commit is contained in:
Cursor Agent
parent
c94a8d6514
commit
6413ddf040
3 changed files with 1374 additions and 0 deletions
849
benchmark_mcp_vs_grep.py
Normal file
849
benchmark_mcp_vs_grep.py
Normal file
|
|
@ -0,0 +1,849 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Benchmark: Semantic Code Search (claude-context style) vs. Grep-based Search
|
||||
for answering codebase queries about LiteLLM.
|
||||
|
||||
Compares two retrieval approaches:
|
||||
|
||||
1. **Semantic Search** (claude-context approach): sentence-transformers embeddings +
|
||||
Milvus Lite vector DB with cosine similarity. Code is AST-chunked.
|
||||
|
||||
2. **Grep-based Search**: ripgrep keyword search with file ranking and snippet extraction.
|
||||
|
||||
Both feed retrieved context to Claude to generate answers, scored by Claude-as-judge.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
import anthropic
|
||||
|
||||
ANTHROPIC_API_KEY = os.environ["ANTHROPIC_API_KEY"]
|
||||
CODEBASE_DIR = "/workspace/litellm"
|
||||
MILVUS_DB_PATH = "/workspace/benchmark_milvus.db"
|
||||
COLLECTION_NAME = "litellm_code"
|
||||
EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2"
|
||||
EMBEDDING_DIM = 384
|
||||
ANSWERING_MODEL = "claude-sonnet-4-20250514"
|
||||
JUDGE_MODEL = "claude-sonnet-4-20250514"
|
||||
MAX_CONTEXT_TOKENS = 12000
|
||||
MAX_FILES_TO_INDEX = 500
|
||||
TOP_K_SEMANTIC = 20
|
||||
|
||||
anthropic_client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
|
||||
|
||||
# Lazy-loaded globals
|
||||
_embedding_model = None
|
||||
_tokenizer = None
|
||||
|
||||
|
||||
def get_embedding_model():
|
||||
global _embedding_model
|
||||
if _embedding_model is None:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
_embedding_model = SentenceTransformer(EMBEDDING_MODEL_NAME)
|
||||
return _embedding_model
|
||||
|
||||
|
||||
def count_tokens_approx(text: str) -> int:
|
||||
"""Approximate token count (words * 1.3)."""
|
||||
return int(len(text.split()) * 1.3)
|
||||
|
||||
|
||||
def truncate_to_tokens(text: str, max_tokens: int) -> str:
|
||||
words = text.split()
|
||||
target_words = int(max_tokens / 1.3)
|
||||
return " ".join(words[:target_words])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Benchmark queries with ground truth
|
||||
# ---------------------------------------------------------------------------
|
||||
BENCHMARK_QUERIES = [
|
||||
{
|
||||
"id": "q1",
|
||||
"query": "How does LiteLLM handle streaming responses from different providers?",
|
||||
"category": "architecture",
|
||||
"ground_truth": (
|
||||
"LiteLLM handles streaming through a CustomStreamWrapper class defined in "
|
||||
"litellm/litellm_core_utils/streaming_handler.py (or litellm/utils.py). Each provider's streaming "
|
||||
"response is normalized into OpenAI-compatible chunk format with delta content. "
|
||||
"The main completion() function in litellm/main.py checks for stream=True and "
|
||||
"returns the wrapped stream. Provider-specific implementations handle their "
|
||||
"native streaming formats (SSE for OpenAI, event streams for Anthropic, etc.) "
|
||||
"and convert them to a consistent ModelResponse format."
|
||||
),
|
||||
"key_files": ["litellm/main.py", "litellm/litellm_core_utils/streaming_handler.py"],
|
||||
"grep_terms": ["CustomStreamWrapper", "stream=True", "streaming", "ModelResponseStream"],
|
||||
},
|
||||
{
|
||||
"id": "q2",
|
||||
"query": "What is the Router class and how does it handle load balancing?",
|
||||
"category": "architecture",
|
||||
"ground_truth": (
|
||||
"The Router class is defined in litellm/router.py. It manages multiple model "
|
||||
"deployments and provides load balancing, fallbacks, and retries. It supports "
|
||||
"multiple routing strategies including 'simple-shuffle', 'least-busy', "
|
||||
"'latency-based-routing', 'cost-based-routing', and 'usage-based-routing'. "
|
||||
"The Router maintains a list of model_list deployments and routes requests "
|
||||
"using the selected strategy."
|
||||
),
|
||||
"key_files": ["litellm/router.py", "litellm/router_utils/"],
|
||||
"grep_terms": ["class Router", "routing_strategy", "simple-shuffle", "least-busy", "latency-based-routing"],
|
||||
},
|
||||
{
|
||||
"id": "q3",
|
||||
"query": "How are API keys authenticated in the LiteLLM proxy server?",
|
||||
"category": "proxy",
|
||||
"ground_truth": (
|
||||
"The LiteLLM proxy authenticates API keys through the user_api_key_auth() "
|
||||
"function in litellm/proxy/auth/user_api_key_auth.py. It validates bearer "
|
||||
"tokens against a database of virtual keys stored in the LiteLLM_VerificationToken "
|
||||
"table via Prisma. The proxy supports multiple auth methods: API keys, JWT tokens, "
|
||||
"and OAuth2."
|
||||
),
|
||||
"key_files": ["litellm/proxy/auth/user_api_key_auth.py"],
|
||||
"grep_terms": ["user_api_key_auth", "api_key", "LiteLLM_VerificationToken", "bearer"],
|
||||
},
|
||||
{
|
||||
"id": "q4",
|
||||
"query": "How does litellm.completion() work internally - what is the call flow?",
|
||||
"category": "core",
|
||||
"ground_truth": (
|
||||
"litellm.completion() is defined in litellm/main.py. The flow is: 1) Parse and "
|
||||
"validate input parameters. 2) Determine the provider from the model string. "
|
||||
"3) Apply pre-call hooks and logging callbacks. 4) Transform the request to "
|
||||
"the provider's format. 5) Make the API call. 6) Transform the response back "
|
||||
"to OpenAI format (ModelResponse). 7) Run post-call hooks and logging."
|
||||
),
|
||||
"key_files": ["litellm/main.py"],
|
||||
"grep_terms": ["def completion(", "def acompletion(", "get_llm_provider", "model_response"],
|
||||
},
|
||||
{
|
||||
"id": "q5",
|
||||
"query": "What caching mechanisms does LiteLLM support?",
|
||||
"category": "feature",
|
||||
"ground_truth": (
|
||||
"LiteLLM supports multiple caching backends defined in litellm/caching/. "
|
||||
"The main Cache class supports: InMemoryCache, RedisCache, RedisSemanticCache, "
|
||||
"S3Cache, DiskCache, and QdrantSemanticCache. Caching can be enabled via "
|
||||
"litellm.cache = Cache(type='redis') or via proxy config."
|
||||
),
|
||||
"key_files": ["litellm/caching/", "litellm/caching/caching.py"],
|
||||
"grep_terms": ["class Cache", "InMemoryCache", "RedisCache", "S3Cache", "caching"],
|
||||
},
|
||||
{
|
||||
"id": "q6",
|
||||
"query": "How does LiteLLM transform Anthropic Claude function/tool calling to OpenAI format?",
|
||||
"category": "provider",
|
||||
"ground_truth": (
|
||||
"LiteLLM handles Anthropic tool calling transformation in "
|
||||
"litellm/llms/anthropic/chat/transformation.py (AnthropicConfig class). "
|
||||
"OpenAI-format tools are transformed to Anthropic's tool_use format. "
|
||||
"Tool calls in responses are mapped from Anthropic's content blocks "
|
||||
"with type='tool_use' to OpenAI's tool_calls array."
|
||||
),
|
||||
"key_files": ["litellm/llms/anthropic/chat/transformation.py"],
|
||||
"grep_terms": ["tool_use", "tool_calls", "AnthropicConfig", "function_call", "input_schema"],
|
||||
},
|
||||
{
|
||||
"id": "q7",
|
||||
"query": "How does the proxy handle budget management and spend tracking?",
|
||||
"category": "proxy",
|
||||
"ground_truth": (
|
||||
"Budget management is handled through the proxy's spend tracking system. "
|
||||
"Each API key, user, and team has a max_budget field. The proxy tracks spend "
|
||||
"in the LiteLLM_SpendLogs table. Budget checks happen in auth middleware. "
|
||||
"The spend is calculated using model cost data."
|
||||
),
|
||||
"key_files": ["litellm/proxy/auth/", "litellm/proxy/spend_tracking/"],
|
||||
"grep_terms": ["max_budget", "spend", "LiteLLM_SpendLogs", "track_cost", "budget"],
|
||||
},
|
||||
{
|
||||
"id": "q8",
|
||||
"query": "What database schema does the LiteLLM proxy use and how are migrations handled?",
|
||||
"category": "infrastructure",
|
||||
"ground_truth": (
|
||||
"The proxy uses Prisma ORM with schema in litellm/proxy/schema.prisma. "
|
||||
"Key tables: LiteLLM_VerificationToken, LiteLLM_TeamTable, LiteLLM_UserTable, "
|
||||
"LiteLLM_SpendLogs. Migrations are handled by 'prisma migrate deploy' "
|
||||
"on startup. Supports PostgreSQL and SQLite."
|
||||
),
|
||||
"key_files": ["litellm/proxy/schema.prisma"],
|
||||
"grep_terms": ["schema.prisma", "LiteLLM_VerificationToken", "LiteLLM_TeamTable", "prisma migrate"],
|
||||
},
|
||||
{
|
||||
"id": "q9",
|
||||
"query": "How does LiteLLM implement fallback logic when a model fails?",
|
||||
"category": "reliability",
|
||||
"ground_truth": (
|
||||
"Fallback logic is in the Router class (litellm/router.py). When a request "
|
||||
"fails, the Router retries or falls back to alternative deployments. "
|
||||
"Configured via fallbacks parameter. Failed deployments are placed in cooldown. "
|
||||
"Also supports context_window_fallbacks and content_policy_fallbacks."
|
||||
),
|
||||
"key_files": ["litellm/router.py", "litellm/router_utils/fallback_event_handlers.py"],
|
||||
"grep_terms": ["fallbacks", "cooldown", "context_window_fallbacks", "content_policy_fallbacks", "retry"],
|
||||
},
|
||||
{
|
||||
"id": "q10",
|
||||
"query": "How does LiteLLM support custom callback/logging integrations?",
|
||||
"category": "observability",
|
||||
"ground_truth": (
|
||||
"LiteLLM supports custom callbacks through litellm.callbacks and "
|
||||
"litellm.success_callback/failure_callback. Custom callbacks implement "
|
||||
"CustomLogger from litellm/integrations/custom_logger.py with methods like "
|
||||
"log_success_event(), log_failure_event(). Built-in integrations include "
|
||||
"Langfuse, Datadog, Sentry, Prometheus."
|
||||
),
|
||||
"key_files": ["litellm/integrations/custom_logger.py"],
|
||||
"grep_terms": ["CustomLogger", "success_callback", "failure_callback", "log_success_event", "callbacks"],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Code chunking (AST-based, mirrors claude-context)
|
||||
# ---------------------------------------------------------------------------
|
||||
def chunk_python_file(filepath: str, max_chunk_words: int = 400) -> list[dict]:
|
||||
try:
|
||||
with open(filepath, "r", errors="replace") as f:
|
||||
content = f.read()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if not content.strip():
|
||||
return []
|
||||
|
||||
rel_path = os.path.relpath(filepath, "/workspace")
|
||||
chunks = []
|
||||
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
except SyntaxError:
|
||||
words = len(content.split())
|
||||
if words <= max_chunk_words:
|
||||
chunks.append({"text": f"# File: {rel_path}\n{content}", "file": rel_path, "type": "file"})
|
||||
else:
|
||||
lines = content.split("\n")
|
||||
current_chunk = []
|
||||
current_words = 0
|
||||
for line in lines:
|
||||
lw = len(line.split())
|
||||
if current_words + lw > max_chunk_words and current_chunk:
|
||||
chunk_text = "\n".join(current_chunk)
|
||||
chunks.append({"text": f"# File: {rel_path}\n{chunk_text}", "file": rel_path, "type": "fragment"})
|
||||
current_chunk = []
|
||||
current_words = 0
|
||||
current_chunk.append(line)
|
||||
current_words += lw
|
||||
if current_chunk:
|
||||
chunks.append({"text": f"# File: {rel_path}\n" + "\n".join(current_chunk), "file": rel_path, "type": "fragment"})
|
||||
return chunks
|
||||
|
||||
lines = content.split("\n")
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
||||
start = node.lineno - 1
|
||||
end = node.end_lineno if hasattr(node, "end_lineno") and node.end_lineno else start + 1
|
||||
block = "\n".join(lines[start:end])
|
||||
block_words = len(block.split())
|
||||
|
||||
if block_words <= max_chunk_words:
|
||||
node_type = "class" if isinstance(node, ast.ClassDef) else "function"
|
||||
chunks.append({
|
||||
"text": f"# File: {rel_path} | {node_type}: {node.name}\n{block}",
|
||||
"file": rel_path,
|
||||
"type": node_type,
|
||||
"name": node.name,
|
||||
})
|
||||
else:
|
||||
sub_lines = block.split("\n")
|
||||
current = []
|
||||
current_w = 0
|
||||
for sl in sub_lines:
|
||||
lw = len(sl.split())
|
||||
if current_w + lw > max_chunk_words and current:
|
||||
chunks.append({
|
||||
"text": f"# File: {rel_path} | {node.name} (part)\n" + "\n".join(current),
|
||||
"file": rel_path,
|
||||
"type": "fragment",
|
||||
"name": node.name,
|
||||
})
|
||||
current = []
|
||||
current_w = 0
|
||||
current.append(sl)
|
||||
current_w += lw
|
||||
if current:
|
||||
chunks.append({
|
||||
"text": f"# File: {rel_path} | {node.name} (part)\n" + "\n".join(current),
|
||||
"file": rel_path,
|
||||
"type": "fragment",
|
||||
"name": node.name,
|
||||
})
|
||||
|
||||
if not chunks:
|
||||
words = len(content.split())
|
||||
if words <= max_chunk_words:
|
||||
chunks.append({"text": f"# File: {rel_path}\n{content}", "file": rel_path, "type": "file"})
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Approach 1: Semantic Search (claude-context style)
|
||||
# ---------------------------------------------------------------------------
|
||||
def index_codebase_semantic():
|
||||
from pymilvus import MilvusClient, DataType, CollectionSchema, FieldSchema
|
||||
|
||||
print("\n=== INDEXING CODEBASE FOR SEMANTIC SEARCH ===")
|
||||
|
||||
if os.path.exists(MILVUS_DB_PATH):
|
||||
os.remove(MILVUS_DB_PATH)
|
||||
|
||||
model = get_embedding_model()
|
||||
|
||||
milvus = MilvusClient(MILVUS_DB_PATH)
|
||||
|
||||
schema = CollectionSchema(fields=[
|
||||
FieldSchema(name="id", dtype=DataType.INT64, is_primary=True, auto_id=True),
|
||||
FieldSchema(name="text", dtype=DataType.VARCHAR, max_length=65535),
|
||||
FieldSchema(name="file", dtype=DataType.VARCHAR, max_length=1024),
|
||||
FieldSchema(name="chunk_type", dtype=DataType.VARCHAR, max_length=64),
|
||||
FieldSchema(name="embedding", dtype=DataType.FLOAT_VECTOR, dim=EMBEDDING_DIM),
|
||||
])
|
||||
|
||||
milvus.create_collection(collection_name=COLLECTION_NAME, schema=schema)
|
||||
|
||||
py_files = []
|
||||
for root, dirs, files in os.walk(CODEBASE_DIR):
|
||||
dirs[:] = [d for d in dirs if d not in {"__pycache__", ".git", "node_modules", ".venv", "out"}]
|
||||
for f in files:
|
||||
if f.endswith(".py"):
|
||||
py_files.append(os.path.join(root, f))
|
||||
|
||||
py_files.sort(key=lambda p: os.path.getsize(p), reverse=True)
|
||||
py_files = py_files[:MAX_FILES_TO_INDEX]
|
||||
print(f" Chunking {len(py_files)} Python files...")
|
||||
|
||||
all_chunks = []
|
||||
for fp in py_files:
|
||||
all_chunks.extend(chunk_python_file(fp))
|
||||
|
||||
print(f" Total chunks: {len(all_chunks)}")
|
||||
|
||||
print(f" Generating embeddings with {EMBEDDING_MODEL_NAME} (local)...")
|
||||
t0 = time.time()
|
||||
texts = [c["text"] for c in all_chunks]
|
||||
batch_size = 256
|
||||
all_embeddings = []
|
||||
for i in range(0, len(texts), batch_size):
|
||||
batch = texts[i : i + batch_size]
|
||||
embs = model.encode(batch, show_progress_bar=False, normalize_embeddings=True)
|
||||
all_embeddings.extend(embs.tolist())
|
||||
if (i // batch_size) % 10 == 0:
|
||||
print(f" Embedded {min(i + batch_size, len(texts))}/{len(texts)} chunks...")
|
||||
embed_time = time.time() - t0
|
||||
print(f" Embedding time: {embed_time:.1f}s")
|
||||
|
||||
print(" Inserting into Milvus Lite...")
|
||||
insert_batch = 500
|
||||
for i in range(0, len(all_chunks), insert_batch):
|
||||
batch_chunks = all_chunks[i : i + insert_batch]
|
||||
batch_embs = all_embeddings[i : i + insert_batch]
|
||||
data = []
|
||||
for chunk, emb in zip(batch_chunks, batch_embs):
|
||||
data.append({
|
||||
"text": chunk["text"][:65000],
|
||||
"file": chunk["file"][:1024],
|
||||
"chunk_type": chunk.get("type", "unknown")[:64],
|
||||
"embedding": emb,
|
||||
})
|
||||
milvus.insert(collection_name=COLLECTION_NAME, data=data)
|
||||
|
||||
index_params = milvus.prepare_index_params()
|
||||
index_params.add_index(field_name="embedding", metric_type="COSINE", index_type="FLAT")
|
||||
milvus.create_index(collection_name=COLLECTION_NAME, index_params=index_params)
|
||||
|
||||
milvus.close()
|
||||
print(f" Indexing complete. {len(all_chunks)} chunks indexed.")
|
||||
return len(all_chunks), embed_time
|
||||
|
||||
|
||||
def search_semantic(query: str, top_k: int = TOP_K_SEMANTIC) -> tuple[str, float]:
|
||||
from pymilvus import MilvusClient
|
||||
|
||||
t0 = time.time()
|
||||
model = get_embedding_model()
|
||||
query_emb = model.encode([query], normalize_embeddings=True).tolist()[0]
|
||||
|
||||
milvus = MilvusClient(MILVUS_DB_PATH)
|
||||
results = milvus.search(
|
||||
collection_name=COLLECTION_NAME,
|
||||
data=[query_emb],
|
||||
limit=top_k,
|
||||
output_fields=["text", "file", "chunk_type"],
|
||||
)
|
||||
milvus.close()
|
||||
|
||||
context_parts = []
|
||||
total_tokens = 0
|
||||
seen = set()
|
||||
|
||||
for hits in results:
|
||||
for hit in hits:
|
||||
text = hit["entity"]["text"]
|
||||
h = hashlib.md5(text.encode()).hexdigest()
|
||||
if h in seen:
|
||||
continue
|
||||
seen.add(h)
|
||||
ct = count_tokens_approx(text)
|
||||
if total_tokens + ct > MAX_CONTEXT_TOKENS:
|
||||
break
|
||||
context_parts.append(text)
|
||||
total_tokens += ct
|
||||
|
||||
search_time = time.time() - t0
|
||||
return "\n\n---\n\n".join(context_parts), search_time
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Approach 2: Grep-based Search
|
||||
# ---------------------------------------------------------------------------
|
||||
def search_grep(query: str, grep_terms: list[str] | None = None) -> tuple[str, float]:
|
||||
t0 = time.time()
|
||||
|
||||
if grep_terms is None:
|
||||
grep_terms = [w for w in query.split() if len(w) > 3][:5]
|
||||
|
||||
all_matches: dict[str, list[str]] = {}
|
||||
|
||||
for term in grep_terms:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["rg", "--no-heading", "-n", "--type", "py", "-C", "3", "-m", "10", term, CODEBASE_DIR],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
if result.stdout:
|
||||
for line in result.stdout.split("\n"):
|
||||
if ":" in line:
|
||||
filepath = line.split(":")[0]
|
||||
if os.path.isfile(filepath):
|
||||
rel = os.path.relpath(filepath, "/workspace")
|
||||
if rel not in all_matches:
|
||||
all_matches[rel] = []
|
||||
all_matches[rel].append(line)
|
||||
except (subprocess.TimeoutExpired, Exception):
|
||||
continue
|
||||
|
||||
file_scores = {fp: len(m) for fp, m in all_matches.items()}
|
||||
ranked = sorted(file_scores, key=file_scores.get, reverse=True)
|
||||
|
||||
context_parts = []
|
||||
total_tokens = 0
|
||||
|
||||
for filepath in ranked[:10]:
|
||||
full_path = os.path.join("/workspace", filepath)
|
||||
try:
|
||||
with open(full_path, "r", errors="replace") as f:
|
||||
content = f.read()
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
file_tok = count_tokens_approx(content)
|
||||
if file_tok > 3000:
|
||||
relevant_lines = all_matches.get(filepath, [])
|
||||
line_nums = set()
|
||||
for ml in relevant_lines:
|
||||
parts = ml.split(":")
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
line_nums.add(int(parts[1]))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if line_nums:
|
||||
lines = content.split("\n")
|
||||
extracted = []
|
||||
for ln in sorted(line_nums):
|
||||
start = max(0, ln - 10)
|
||||
end = min(len(lines), ln + 10)
|
||||
extracted.append(f"# Lines {start+1}-{end} of {filepath}\n" + "\n".join(lines[start:end]))
|
||||
content = "\n\n".join(extracted)
|
||||
|
||||
chunk = f"# File: {filepath}\n{content}"
|
||||
ct = count_tokens_approx(chunk)
|
||||
|
||||
if total_tokens + ct > MAX_CONTEXT_TOKENS:
|
||||
remaining = MAX_CONTEXT_TOKENS - total_tokens
|
||||
if remaining > 200:
|
||||
chunk = truncate_to_tokens(chunk, remaining)
|
||||
context_parts.append(chunk)
|
||||
break
|
||||
|
||||
context_parts.append(chunk)
|
||||
total_tokens += ct
|
||||
|
||||
search_time = time.time() - t0
|
||||
return "\n\n---\n\n".join(context_parts), search_time
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM answer generation (Claude)
|
||||
# ---------------------------------------------------------------------------
|
||||
def generate_answer(query: str, context: str) -> tuple[str, float]:
|
||||
t0 = time.time()
|
||||
context_trimmed = truncate_to_tokens(context, MAX_CONTEXT_TOKENS)
|
||||
msg = anthropic_client.messages.create(
|
||||
model=ANSWERING_MODEL,
|
||||
max_tokens=1000,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"You are a code expert answering questions about the LiteLLM codebase. "
|
||||
f"Use the provided code context to give accurate, specific answers. "
|
||||
f"Reference specific files, classes, and functions when possible. Be concise but thorough.\n\n"
|
||||
f"Code context:\n\n{context_trimmed}\n\n---\n\nQuestion: {query}"
|
||||
),
|
||||
}],
|
||||
)
|
||||
answer = msg.content[0].text.strip()
|
||||
return answer, time.time() - t0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM-as-Judge (Claude)
|
||||
# ---------------------------------------------------------------------------
|
||||
def judge_answer(query: str, answer: str, ground_truth: str) -> dict:
|
||||
msg = anthropic_client.messages.create(
|
||||
model=JUDGE_MODEL,
|
||||
max_tokens=500,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"You are an expert judge evaluating answers about a codebase. "
|
||||
"Score the answer on these dimensions (1-5 each):\n"
|
||||
"1. **Accuracy**: Does the answer contain correct information matching the ground truth?\n"
|
||||
"2. **Completeness**: Does it cover all key points from the ground truth?\n"
|
||||
"3. **Specificity**: Does it reference specific files, classes, functions?\n"
|
||||
"4. **Relevance**: Is the answer focused on what was asked?\n\n"
|
||||
"Return a JSON object with keys: accuracy, completeness, specificity, "
|
||||
"relevance, total (sum of the four), and brief_explanation (one sentence).\n\n"
|
||||
f"Question: {query}\n\n"
|
||||
f"Ground Truth Answer:\n{ground_truth}\n\n"
|
||||
f"Generated Answer:\n{answer}\n\n"
|
||||
"Return ONLY valid JSON, no markdown formatting."
|
||||
),
|
||||
}],
|
||||
)
|
||||
text = msg.content[0].text.strip()
|
||||
text = re.sub(r"```json\s*", "", text)
|
||||
text = re.sub(r"```\s*$", "", text)
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return {"accuracy": 0, "completeness": 0, "specificity": 0, "relevance": 0, "total": 0, "brief_explanation": "parse error"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report compilation
|
||||
# ---------------------------------------------------------------------------
|
||||
@dataclass
|
||||
class QueryResult:
|
||||
query_id: str
|
||||
query: str
|
||||
category: str
|
||||
approach: str
|
||||
context_tokens: int
|
||||
search_time_s: float
|
||||
answer_time_s: float
|
||||
total_time_s: float
|
||||
answer: str
|
||||
scores: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def safe_score(r, key):
|
||||
v = r.scores.get(key, 0) if isinstance(r.scores, dict) else 0
|
||||
return v if isinstance(v, (int, float)) else 0
|
||||
|
||||
|
||||
def avg(vals):
|
||||
return sum(vals) / len(vals) if vals else 0
|
||||
|
||||
|
||||
def compile_report(results: list[QueryResult], num_chunks: int, embed_time: float) -> str:
|
||||
sem = [r for r in results if r.approach == "semantic"]
|
||||
grp = [r for r in results if r.approach == "grep"]
|
||||
|
||||
lines = []
|
||||
lines.append("# Benchmark: Semantic Code Search (claude-context) vs. Grep-based Search")
|
||||
lines.append("")
|
||||
lines.append("## Overview")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"This benchmark compares two code retrieval approaches for answering "
|
||||
"natural language questions about the LiteLLM codebase (~1800 Python files, ~125K LOC):"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"1. **Semantic Search** (claude-context style): `all-MiniLM-L6-v2` sentence-transformer embeddings + "
|
||||
"Milvus Lite vector DB with cosine similarity. Code split into semantic chunks using Python AST parsing."
|
||||
)
|
||||
lines.append(
|
||||
"2. **Grep-based Search**: ripgrep keyword search with context lines, "
|
||||
"file ranking by match density, and targeted snippet extraction."
|
||||
)
|
||||
lines.append("")
|
||||
lines.append(
|
||||
f"Both approaches feed retrieved context to `{ANSWERING_MODEL}` to generate answers. "
|
||||
f"Answers are scored by `{JUDGE_MODEL}` as judge against human-written ground truth."
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Setup Details")
|
||||
lines.append("")
|
||||
lines.append("| Parameter | Value |")
|
||||
lines.append("|---|---|")
|
||||
lines.append("| Codebase | LiteLLM (~1800 .py files, ~125K LOC) |")
|
||||
lines.append(f"| Chunks indexed (semantic) | {num_chunks} |")
|
||||
lines.append(f"| Embedding model | {EMBEDDING_MODEL_NAME} (local, 384-dim) |")
|
||||
lines.append(f"| Embedding time (one-time) | {embed_time:.1f}s |")
|
||||
lines.append(f"| Answering model | {ANSWERING_MODEL} |")
|
||||
lines.append(f"| Judge model | {JUDGE_MODEL} |")
|
||||
lines.append(f"| Max context tokens | {MAX_CONTEXT_TOKENS} |")
|
||||
lines.append(f"| Number of queries | {len(BENCHMARK_QUERIES)} |")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Per-Query Results")
|
||||
lines.append("")
|
||||
lines.append("| Query | Category | Approach | Accuracy | Completeness | Specificity | Relevance | Total (/20) | Search Time | Context Tokens |")
|
||||
lines.append("|---|---|---|---|---|---|---|---|---|---|")
|
||||
|
||||
for q in BENCHMARK_QUERIES:
|
||||
sem_r = next(r for r in sem if r.query_id == q["id"])
|
||||
grp_r = next(r for r in grp if r.query_id == q["id"])
|
||||
for r, label in [(sem_r, "Semantic"), (grp_r, "Grep")]:
|
||||
lines.append(
|
||||
f"| {q['id']} | {q['category']} | {label} | "
|
||||
f"{safe_score(r, 'accuracy')} | {safe_score(r, 'completeness')} | "
|
||||
f"{safe_score(r, 'specificity')} | {safe_score(r, 'relevance')} | "
|
||||
f"{safe_score(r, 'total')} | {r.search_time_s:.2f}s | {r.context_tokens} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
|
||||
sem_sc = {k: avg([safe_score(r, k) for r in sem]) for k in ["accuracy", "completeness", "specificity", "relevance", "total"]}
|
||||
grp_sc = {k: avg([safe_score(r, k) for r in grp]) for k in ["accuracy", "completeness", "specificity", "relevance", "total"]}
|
||||
|
||||
lines.append("## Aggregate Scores")
|
||||
lines.append("")
|
||||
lines.append("| Metric | Semantic (avg) | Grep (avg) | Winner |")
|
||||
lines.append("|---|---|---|---|")
|
||||
for key in ["accuracy", "completeness", "specificity", "relevance", "total"]:
|
||||
s, g = sem_sc[key], grp_sc[key]
|
||||
winner = "Semantic" if s > g else ("Grep" if g > s else "Tie")
|
||||
label = key.title() if key != "total" else "**Total (/20)**"
|
||||
lines.append(f"| {label} | {s:.2f} | {g:.2f} | {winner} |")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Performance Comparison")
|
||||
lines.append("")
|
||||
lines.append("| Metric | Semantic (avg) | Grep (avg) |")
|
||||
lines.append("|---|---|---|")
|
||||
lines.append(f"| Search time | {avg([r.search_time_s for r in sem]):.2f}s | {avg([r.search_time_s for r in grp]):.2f}s |")
|
||||
lines.append(f"| Answer gen time | {avg([r.answer_time_s for r in sem]):.2f}s | {avg([r.answer_time_s for r in grp]):.2f}s |")
|
||||
lines.append(f"| Total time per query | {avg([r.total_time_s for r in sem]):.2f}s | {avg([r.total_time_s for r in grp]):.2f}s |")
|
||||
lines.append(f"| Context tokens (avg) | {avg([r.context_tokens for r in sem]):.0f} | {avg([r.context_tokens for r in grp]):.0f} |")
|
||||
lines.append(f"| One-time indexing cost | {embed_time:.1f}s | 0s |")
|
||||
lines.append("")
|
||||
|
||||
sem_wins = sum(1 for q in BENCHMARK_QUERIES if safe_score(next(r for r in sem if r.query_id == q["id"]), "total") > safe_score(next(r for r in grp if r.query_id == q["id"]), "total"))
|
||||
grp_wins = sum(1 for q in BENCHMARK_QUERIES if safe_score(next(r for r in grp if r.query_id == q["id"]), "total") > safe_score(next(r for r in sem if r.query_id == q["id"]), "total"))
|
||||
ties = len(BENCHMARK_QUERIES) - sem_wins - grp_wins
|
||||
|
||||
lines.append("## Head-to-Head")
|
||||
lines.append("")
|
||||
lines.append("| Outcome | Count |")
|
||||
lines.append("|---|---|")
|
||||
lines.append(f"| Semantic wins | {sem_wins}/{len(BENCHMARK_QUERIES)} |")
|
||||
lines.append(f"| Grep wins | {grp_wins}/{len(BENCHMARK_QUERIES)} |")
|
||||
lines.append(f"| Ties | {ties}/{len(BENCHMARK_QUERIES)} |")
|
||||
lines.append("")
|
||||
|
||||
categories = sorted(set(q["category"] for q in BENCHMARK_QUERIES))
|
||||
lines.append("## Performance by Category")
|
||||
lines.append("")
|
||||
lines.append("| Category | Semantic Avg Total | Grep Avg Total | Winner |")
|
||||
lines.append("|---|---|---|---|")
|
||||
for cat in categories:
|
||||
cs = [r for r in sem if r.category == cat]
|
||||
cg = [r for r in grp if r.category == cat]
|
||||
s = avg([safe_score(r, "total") for r in cs])
|
||||
g = avg([safe_score(r, "total") for r in cg])
|
||||
winner = "Semantic" if s > g else ("Grep" if g > s else "Tie")
|
||||
lines.append(f"| {cat} | {s:.1f} | {g:.1f} | {winner} |")
|
||||
lines.append("")
|
||||
|
||||
overall_winner = "Semantic Search (claude-context)" if sem_sc["total"] > grp_sc["total"] else "Grep-based Search"
|
||||
margin = abs(sem_sc["total"] - grp_sc["total"])
|
||||
|
||||
lines.append("## Key Findings")
|
||||
lines.append("")
|
||||
lines.append(f"**Overall winner: {overall_winner}** (margin: {margin:.2f}/20)")
|
||||
lines.append("")
|
||||
|
||||
lines.append("### Semantic Search (claude-context style)")
|
||||
lines.append("**Strengths:**")
|
||||
lines.append("- Understands natural language queries semantically")
|
||||
lines.append("- Finds conceptually related code even without exact keyword matches")
|
||||
lines.append("- Returns focused, relevant code chunks (AST-aware splitting)")
|
||||
lines.append("- Consistent retrieval quality regardless of query phrasing")
|
||||
lines.append("")
|
||||
lines.append("**Weaknesses:**")
|
||||
lines.append(f"- Requires upfront indexing ({embed_time:.0f}s for {num_chunks} chunks)")
|
||||
lines.append("- Each search requires an embedding computation")
|
||||
lines.append("- May miss exact symbol matches if embedding doesn't capture them")
|
||||
lines.append("- Requires API keys for embedding model + vector database (in cloud mode)")
|
||||
lines.append("")
|
||||
|
||||
lines.append("### Grep-based Search")
|
||||
lines.append("**Strengths:**")
|
||||
lines.append("- Zero indexing overhead")
|
||||
lines.append("- Exact matching for known symbols")
|
||||
lines.append("- Fast per-query search (sub-second, no API call)")
|
||||
lines.append("- No external dependencies for search")
|
||||
lines.append("- Returns exact line-level matches with surrounding context")
|
||||
lines.append("")
|
||||
lines.append("**Weaknesses:**")
|
||||
lines.append("- Requires knowing the right keywords")
|
||||
lines.append("- Cannot understand semantic intent")
|
||||
lines.append("- May return irrelevant matches for common terms")
|
||||
lines.append("- Context extraction is heuristic-based")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Methodology Notes")
|
||||
lines.append("")
|
||||
lines.append("- **Semantic search** mirrors claude-context's approach: AST-based code chunking, ")
|
||||
lines.append(" dense embeddings, vector similarity search via Milvus. Uses local `all-MiniLM-L6-v2`")
|
||||
lines.append(f" instead of OpenAI `text-embedding-3-small` (claude-context default).")
|
||||
lines.append(" Note: claude-context also uses BM25 hybrid search which we approximate with dense-only.")
|
||||
lines.append("- **Grep search** mirrors typical coding agent behavior: keyword-based ripgrep, ")
|
||||
lines.append(" file ranking, targeted snippet extraction.")
|
||||
lines.append("- Both use the same context budget (12K tokens) and answering model (Claude).")
|
||||
lines.append("- Scoring: Claude as judge, 4 dimensions x 5 points = 20 max.")
|
||||
lines.append("- The grep approach is given pre-defined search terms (best-case for grep).")
|
||||
lines.append("- 10 queries spanning architecture, proxy, core, features, providers, reliability.")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main benchmark
|
||||
# ---------------------------------------------------------------------------
|
||||
def run_benchmark():
|
||||
print("=" * 80)
|
||||
print("BENCHMARK: Semantic Code Search (claude-context) vs. Grep-based Search")
|
||||
print("=" * 80)
|
||||
print(f"Codebase: LiteLLM ({CODEBASE_DIR})")
|
||||
print(f"Embedding model: {EMBEDDING_MODEL_NAME} (local)")
|
||||
print(f"Answering model: {ANSWERING_MODEL}")
|
||||
print(f"Judge model: {JUDGE_MODEL}")
|
||||
print(f"Max context tokens: {MAX_CONTEXT_TOKENS}")
|
||||
print(f"Queries: {len(BENCHMARK_QUERIES)}")
|
||||
|
||||
num_chunks, embed_time = index_codebase_semantic()
|
||||
|
||||
results: list[QueryResult] = []
|
||||
|
||||
for i, q in enumerate(BENCHMARK_QUERIES):
|
||||
print(f"\n{'─' * 60}")
|
||||
print(f"Query {i+1}/{len(BENCHMARK_QUERIES)}: {q['query'][:80]}...")
|
||||
print(f"Category: {q['category']}")
|
||||
|
||||
# Semantic
|
||||
print(" [Semantic] Searching...")
|
||||
sem_ctx, sem_st = search_semantic(q["query"])
|
||||
sem_ct = count_tokens_approx(sem_ctx)
|
||||
print(f" [Semantic] Context: {sem_ct} tokens, Search: {sem_st:.2f}s")
|
||||
|
||||
print(" [Semantic] Generating answer...")
|
||||
sem_ans, sem_at = generate_answer(q["query"], sem_ctx)
|
||||
sem_total = sem_st + sem_at
|
||||
print(f" [Semantic] Total: {sem_total:.2f}s")
|
||||
|
||||
results.append(QueryResult(
|
||||
query_id=q["id"], query=q["query"], category=q["category"],
|
||||
approach="semantic", context_tokens=sem_ct,
|
||||
search_time_s=round(sem_st, 3), answer_time_s=round(sem_at, 3),
|
||||
total_time_s=round(sem_total, 3), answer=sem_ans,
|
||||
))
|
||||
|
||||
# Grep
|
||||
print(" [Grep] Searching...")
|
||||
grp_ctx, grp_st = search_grep(q["query"], q.get("grep_terms"))
|
||||
grp_ct = count_tokens_approx(grp_ctx)
|
||||
print(f" [Grep] Context: {grp_ct} tokens, Search: {grp_st:.2f}s")
|
||||
|
||||
print(" [Grep] Generating answer...")
|
||||
grp_ans, grp_at = generate_answer(q["query"], grp_ctx)
|
||||
grp_total = grp_st + grp_at
|
||||
print(f" [Grep] Total: {grp_total:.2f}s")
|
||||
|
||||
results.append(QueryResult(
|
||||
query_id=q["id"], query=q["query"], category=q["category"],
|
||||
approach="grep", context_tokens=grp_ct,
|
||||
search_time_s=round(grp_st, 3), answer_time_s=round(grp_at, 3),
|
||||
total_time_s=round(grp_total, 3), answer=grp_ans,
|
||||
))
|
||||
|
||||
# Judge
|
||||
print(f"\n{'=' * 60}")
|
||||
print("JUDGING ANSWERS...")
|
||||
for r in results:
|
||||
q = next(q for q in BENCHMARK_QUERIES if q["id"] == r.query_id)
|
||||
print(f" Judging {r.query_id} ({r.approach})...")
|
||||
r.scores = judge_answer(r.query, r.answer, q["ground_truth"])
|
||||
print(f" Scores: {r.scores}")
|
||||
|
||||
# Report
|
||||
print(f"\n{'=' * 80}")
|
||||
print("RESULTS")
|
||||
print("=" * 80)
|
||||
report = compile_report(results, num_chunks, embed_time)
|
||||
print(report)
|
||||
|
||||
# Save
|
||||
results_data = {
|
||||
"metadata": {
|
||||
"codebase": "litellm",
|
||||
"num_chunks": num_chunks,
|
||||
"embed_time_s": round(embed_time, 1),
|
||||
"embedding_model": EMBEDDING_MODEL_NAME,
|
||||
"answering_model": ANSWERING_MODEL,
|
||||
"judge_model": JUDGE_MODEL,
|
||||
},
|
||||
"results": [asdict(r) for r in results],
|
||||
}
|
||||
with open("/workspace/benchmark_results.json", "w") as f:
|
||||
json.dump(results_data, f, indent=2)
|
||||
with open("/workspace/benchmark_report.md", "w") as f:
|
||||
f.write(report)
|
||||
|
||||
print("\nResults saved to benchmark_results.json and benchmark_report.md")
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_benchmark()
|
||||
133
benchmark_report.md
Normal file
133
benchmark_report.md
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
# Benchmark: Semantic Code Search (claude-context) vs. Grep-based Search
|
||||
|
||||
## Overview
|
||||
|
||||
This benchmark compares two code retrieval approaches for answering natural language questions about the LiteLLM codebase (~1800 Python files, ~125K LOC):
|
||||
|
||||
1. **Semantic Search** (claude-context style): `all-MiniLM-L6-v2` sentence-transformer embeddings + Milvus Lite vector DB with cosine similarity. Code split into semantic chunks using Python AST parsing.
|
||||
2. **Grep-based Search**: ripgrep keyword search with context lines, file ranking by match density, and targeted snippet extraction.
|
||||
|
||||
Both approaches feed retrieved context to `claude-sonnet-4-20250514` to generate answers. Answers are scored by `claude-sonnet-4-20250514` as judge against human-written ground truth.
|
||||
|
||||
## Setup Details
|
||||
|
||||
| Parameter | Value |
|
||||
|---|---|
|
||||
| Codebase | LiteLLM (~1800 .py files, ~125K LOC) |
|
||||
| Chunks indexed (semantic) | 13561 |
|
||||
| Embedding model | all-MiniLM-L6-v2 (local, 384-dim) |
|
||||
| Embedding time (one-time) | 132.3s |
|
||||
| Answering model | claude-sonnet-4-20250514 |
|
||||
| Judge model | claude-sonnet-4-20250514 |
|
||||
| Max context tokens | 12000 |
|
||||
| Number of queries | 10 |
|
||||
|
||||
## Per-Query Results
|
||||
|
||||
| Query | Category | Approach | Accuracy | Completeness | Specificity | Relevance | Total (/20) | Search Time | Context Tokens |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| q1 | architecture | Semantic | 4 | 5 | 5 | 4 | 18 | 0.03s | 3780 |
|
||||
| q1 | architecture | Grep | 5 | 5 | 5 | 5 | 20 | 0.14s | 12006 |
|
||||
| q2 | architecture | Semantic | 5 | 5 | 5 | 5 | 20 | 0.03s | 3471 |
|
||||
| q2 | architecture | Grep | 5 | 5 | 5 | 5 | 20 | 0.12s | 8801 |
|
||||
| q3 | proxy | Semantic | 5 | 5 | 5 | 5 | 20 | 0.02s | 5601 |
|
||||
| q3 | proxy | Grep | 5 | 5 | 5 | 5 | 20 | 0.19s | 12006 |
|
||||
| q4 | core | Semantic | 4 | 4 | 5 | 5 | 18 | 0.02s | 2481 |
|
||||
| q4 | core | Grep | 5 | 4 | 5 | 5 | 19 | 0.08s | 12012 |
|
||||
| q5 | feature | Semantic | 4 | 5 | 5 | 5 | 19 | 0.02s | 4517 |
|
||||
| q5 | feature | Grep | 5 | 5 | 5 | 5 | 20 | 0.12s | 12013 |
|
||||
| q6 | provider | Semantic | 3 | 2 | 3 | 4 | 12 | 0.01s | 1355 |
|
||||
| q6 | provider | Grep | 5 | 5 | 5 | 5 | 20 | 0.13s | 12004 |
|
||||
| q7 | proxy | Semantic | 5 | 5 | 5 | 5 | 20 | 0.01s | 3084 |
|
||||
| q7 | proxy | Grep | 4 | 5 | 5 | 5 | 19 | 0.14s | 12005 |
|
||||
| q8 | infrastructure | Semantic | 4 | 3 | 5 | 4 | 16 | 0.02s | 3998 |
|
||||
| q8 | infrastructure | Grep | 4 | 5 | 5 | 5 | 19 | 0.10s | 10910 |
|
||||
| q9 | reliability | Semantic | 5 | 5 | 5 | 5 | 20 | 0.02s | 5012 |
|
||||
| q9 | reliability | Grep | 5 | 5 | 5 | 5 | 20 | 0.13s | 12009 |
|
||||
| q10 | observability | Semantic | 5 | 5 | 5 | 5 | 20 | 0.02s | 3381 |
|
||||
| q10 | observability | Grep | 5 | 5 | 5 | 5 | 20 | 0.13s | 12006 |
|
||||
|
||||
## Aggregate Scores
|
||||
|
||||
| Metric | Semantic (avg) | Grep (avg) | Winner |
|
||||
|---|---|---|---|
|
||||
| Accuracy | 4.40 | 4.80 | Grep |
|
||||
| Completeness | 4.40 | 4.90 | Grep |
|
||||
| Specificity | 4.80 | 5.00 | Grep |
|
||||
| Relevance | 4.70 | 5.00 | Grep |
|
||||
| **Total (/20)** | 18.30 | 19.70 | Grep |
|
||||
|
||||
## Performance Comparison
|
||||
|
||||
| Metric | Semantic (avg) | Grep (avg) |
|
||||
|---|---|---|
|
||||
| Search time | 0.02s | 0.13s |
|
||||
| Answer gen time | 16.91s | 20.18s |
|
||||
| Total time per query | 16.93s | 20.31s |
|
||||
| Context tokens (avg) | 3668 | 11577 |
|
||||
| One-time indexing cost | 132.3s | 0s |
|
||||
|
||||
## Head-to-Head
|
||||
|
||||
| Outcome | Count |
|
||||
|---|---|
|
||||
| Semantic wins | 1/10 |
|
||||
| Grep wins | 5/10 |
|
||||
| Ties | 4/10 |
|
||||
|
||||
## Performance by Category
|
||||
|
||||
| Category | Semantic Avg Total | Grep Avg Total | Winner |
|
||||
|---|---|---|---|
|
||||
| architecture | 19.0 | 20.0 | Grep |
|
||||
| core | 18.0 | 19.0 | Grep |
|
||||
| feature | 19.0 | 20.0 | Grep |
|
||||
| infrastructure | 16.0 | 19.0 | Grep |
|
||||
| observability | 20.0 | 20.0 | Tie |
|
||||
| provider | 12.0 | 20.0 | Grep |
|
||||
| proxy | 20.0 | 19.5 | Semantic |
|
||||
| reliability | 20.0 | 20.0 | Tie |
|
||||
|
||||
## Key Findings
|
||||
|
||||
**Overall winner: Grep-based Search** (margin: 1.40/20)
|
||||
|
||||
### Semantic Search (claude-context style)
|
||||
**Strengths:**
|
||||
- Understands natural language queries semantically
|
||||
- Finds conceptually related code even without exact keyword matches
|
||||
- Returns focused, relevant code chunks (AST-aware splitting)
|
||||
- Consistent retrieval quality regardless of query phrasing
|
||||
|
||||
**Weaknesses:**
|
||||
- Requires upfront indexing (132s for 13561 chunks)
|
||||
- Each search requires an embedding computation
|
||||
- May miss exact symbol matches if embedding doesn't capture them
|
||||
- Requires API keys for embedding model + vector database (in cloud mode)
|
||||
|
||||
### Grep-based Search
|
||||
**Strengths:**
|
||||
- Zero indexing overhead
|
||||
- Exact matching for known symbols
|
||||
- Fast per-query search (sub-second, no API call)
|
||||
- No external dependencies for search
|
||||
- Returns exact line-level matches with surrounding context
|
||||
|
||||
**Weaknesses:**
|
||||
- Requires knowing the right keywords
|
||||
- Cannot understand semantic intent
|
||||
- May return irrelevant matches for common terms
|
||||
- Context extraction is heuristic-based
|
||||
|
||||
## Methodology Notes
|
||||
|
||||
- **Semantic search** mirrors claude-context's approach: AST-based code chunking,
|
||||
dense embeddings, vector similarity search via Milvus. Uses local `all-MiniLM-L6-v2`
|
||||
instead of OpenAI `text-embedding-3-small` (claude-context default).
|
||||
Note: claude-context also uses BM25 hybrid search which we approximate with dense-only.
|
||||
- **Grep search** mirrors typical coding agent behavior: keyword-based ripgrep,
|
||||
file ranking, targeted snippet extraction.
|
||||
- Both use the same context budget (12K tokens) and answering model (Claude).
|
||||
- Scoring: Claude as judge, 4 dimensions x 5 points = 20 max.
|
||||
- The grep approach is given pre-defined search terms (best-case for grep).
|
||||
- 10 queries spanning architecture, proxy, core, features, providers, reliability.
|
||||
392
benchmark_results.json
Normal file
392
benchmark_results.json
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
{
|
||||
"metadata": {
|
||||
"codebase": "litellm",
|
||||
"num_chunks": 13561,
|
||||
"embed_time_s": 132.3,
|
||||
"embedding_model": "all-MiniLM-L6-v2",
|
||||
"answering_model": "claude-sonnet-4-20250514",
|
||||
"judge_model": "claude-sonnet-4-20250514"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"query_id": "q1",
|
||||
"query": "How does LiteLLM handle streaming responses from different providers?",
|
||||
"category": "architecture",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 3780,
|
||||
"search_time_s": 0.028,
|
||||
"answer_time_s": 21.269,
|
||||
"total_time_s": 21.297,
|
||||
"answer": "LiteLLM handles streaming responses from different providers through a comprehensive streaming architecture with several key components:\n\n## Core Streaming Infrastructure\n\n### 1. CustomStreamWrapper\nThe main streaming handler is `CustomStreamWrapper` in `litellm/litellm_core_utils/streaming_handler.py`, which provides a unified interface for all streaming responses:\n\n- **Dual Interface**: Implements both `__next__()` for sync and `__anext__()` for async streaming\n- **Provider Agnostic**: Works with any provider that LiteLLM supports\n- **Error Handling**: Maps provider-specific exceptions to OpenAI-compatible formats and handles fallbacks\n\n### 2. Provider-Specific Stream Wrappers\nDifferent providers can implement custom streaming through provider-specific configurations:\n\n```python\n# Example from LangGraphConfig\nasync def get_async_custom_stream_wrapper(self, ...):\n # Make async streaming request\n response = await client.post(api_base, headers=headers, data=json.dumps(data), stream=True)\n \n # Create iterator for SSE stream\n completion_stream = self.get_streaming_response(model=model, raw_response=response)\n \n return CustomStreamWrapper(\n completion_stream=completion_stream,\n model=model,\n custom_llm_provider=custom_llm_provider,\n logging_obj=logging_obj,\n )\n```\n\n## Streaming Detection and Processing\n\n### Stream Detection\nLiteLLM detects streaming requests through multiple methods:\n\n```python\ndef _is_streaming_request(self, data: dict, is_streaming_request: Optional[bool] = False) -> bool:\n if is_streaming_request is True:\n return True\n if \"stream\" in data and data[\"stream\"] is True:\n return True\n return False\n```\n\n### Post-Stream Processing\nProviders can apply custom transformations after streaming:\n\n```python\ndef _apply_post_stream_processing(stream: \"CustomStreamWrapper\", model: str, custom_llm_provider: str):\n provider_config = ProviderConfigManager.get_provider_chat_config(model=model, provider=LlmProviders(custom_llm_provider))\n if provider_config is not None:\n return provider_config.post_stream_processing(stream)\n return stream\n```\n\n## WebSocket Streaming for HTTP-Only Providers\n\n### ManagedResponsesWebSocketHandler\nFor providers without native WebSocket support, LiteLLM provides `ManagedResponsesWebSocketHandler`:\n\n- **HTTP-to-WebSocket Bridge**: Converts HTTP streaming to WebSocket transport\n- **Session Management**: Supports multi-turn conversations with in-memory history tracking\n- **Universal Compatibility**: Makes any HTTP-accessible provider available over WebSocket\n\n```python\nclass ManagedResponsesWebSocketHandler:\n \"\"\"\n Handles Responses API WebSocket mode for providers that do not expose a native wss:// responses endpoint.\n - Listens for response.create events from the client\n - Makes HTTP streaming calls via litellm.aresponses(stream=True) \n - Serialises and forwards every streaming event back over the WebSocket\n \"\"\"\n```\n\n## Streaming Hooks and Callbacks\n\n### Multiple Hook Points\nLiteLLM provides several hooks for modifying streaming responses:\n\n1. **Per-Chunk Hooks**: `async_post_call_streaming_hook` - modifies individual chunks\n2. **Iterator Hooks**: `async_post_call_streaming_iterator_hook` - modifies the entire response iterator\n3. **Deployment Hooks**: `async_post_call_streaming_deployment_hook` - called just before chunks are returned\n\n### Guardrails Integration\nStreaming responses can be processed through guardrails:\n\n```python\nasync def async_post_call_streaming_iterator_hook(self, response, user_api_key_dict, request_data):\n current_response = response\n for callback in litellm.callbacks:\n if isinstance(callback, CustomGuardrail):\n current_response = unified_guardrail.async_post_call_streaming_iterator_hook(\n user_api_key_dict=user_api_key_dict,\n request_data=request_data,",
|
||||
"scores": {
|
||||
"accuracy": 4,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 4,
|
||||
"total": 18,
|
||||
"brief_explanation": "The answer is highly detailed and accurate with extensive specific references, though it goes significantly beyond the scope of the basic question about streaming response handling."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q1",
|
||||
"query": "How does LiteLLM handle streaming responses from different providers?",
|
||||
"category": "architecture",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12006,
|
||||
"search_time_s": 0.145,
|
||||
"answer_time_s": 19.959,
|
||||
"total_time_s": 20.104,
|
||||
"answer": "Based on the code context, LiteLLM handles streaming responses from different providers through a sophisticated multi-layered architecture:\n\n## Core Streaming Architecture\n\n### 1. **CustomStreamWrapper** - Universal Stream Interface\nThe `CustomStreamWrapper` class in `litellm/litellm_core_utils/streaming_handler.py` serves as the universal interface for all streaming responses:\n\n```python\nclass CustomStreamWrapper:\n def __init__(\n self, completion_stream, model, logging_obj,\n custom_llm_provider: Optional[str] = None,\n stream_options=None, make_call: Optional[Callable] = None,\n _response_headers: Optional[dict] = None,\n ):\n```\n\nKey features:\n- **Provider-agnostic**: Works with any LLM provider\n- **Dual iteration support**: Implements both `__iter__()` and `__aiter__()` for sync/async streaming\n- **Timeout protection**: Built-in `_check_max_streaming_duration()` method\n- **Logging integration**: Tracks chunks and calculates tokens for stream options\n\n### 2. **Provider-Specific Stream Decoders**\n\n#### **Bedrock Streaming** (`litellm/llms/bedrock/chat/invoke_handler.py`)\nMultiple specialized decoders handle different Bedrock providers:\n\n- **`AWSEventStreamDecoder`**: Base decoder for AWS event streams using `botocore.eventstream.EventStreamBuffer`\n- **`AmazonAnthropicClaudeStreamDecoder`**: Extends base decoder with Anthropic-specific chunk parsing\n- **`AmazonDeepSeekR1StreamDecoder`**: Handles DeepSeek R1 model responses\n- **`MockResponseIterator`**: For AI21 models that don't support native streaming\n\n```python\nclass AWSEventStreamDecoder:\n def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[Union[GChunk, ModelResponseStream, dict]]:\n from botocore.eventstream import EventStreamBuffer\n event_stream_buffer = EventStreamBuffer()\n for chunk in iterator:\n event_stream_buffer.add_data(chunk)\n for event in event_stream_buffer:\n message = self._parse_message_from_event(event)\n if message:\n _data = json.loads(message)\n yield self._chunk_parser(chunk_data=_data)\n```\n\n#### **Anthropic Streaming** (`litellm/llms/anthropic/chat/handler.py`)\nThe `ModelResponseIterator` class handles Anthropic's Server-Sent Events (SSE):\n\n- **JSON accumulation**: Handles partial JSON chunks with `_handle_accumulated_json_chunk()`\n- **Tool call tracking**: Manages streaming tool calls and function calls\n- **Content blocks**: Tracks multiple content blocks in responses\n\n```python\ndef _parse_sse_data(self, str_line: str) -> Optional[ModelResponseStream]:\n data_str = str_line[5:] # Remove \"data:\" prefix\n if self.chunk_type == \"accumulated_json\":\n return self._handle_accumulated_json_chunk(data_str)\n try:\n data_json = json.loads(data_str)\n return self.chunk_parser(chunk=data_json)\n except json.JSONDecodeError:\n # Switch to accumulation mode for partial JSON\n return self._handle_accumulated_json_chunk(data_str)\n```\n\n#### **Vertex AI/Gemini Streaming** (`litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py`)\nHandles both standard and accumulated JSON streaming:\n\n```python\ndef chunk_parser(self, chunk: dict) -> Optional[\"ModelResponseStream\"]:\n from litellm.types.utils import ModelResponseStream\n processed_chunk = GenerateContentResponseBody(**chunk)\n response_id = processed_chunk.get(\"responseId\")\n model_response = ModelResponseStream(choices=[], id=response_id)\n \n # Check for content filtering\n blocked_response = VertexGeminiConfig._check_prompt_level_content_filter",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer provides highly accurate, complete, and specific information about LiteLLM's streaming architecture, matching the ground truth with extensive detail on CustomStreamWrapper, provider-specific implementations, and file locations."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q2",
|
||||
"query": "What is the Router class and how does it handle load balancing?",
|
||||
"category": "architecture",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 3471,
|
||||
"search_time_s": 0.025,
|
||||
"answer_time_s": 18.581,
|
||||
"total_time_s": 18.606,
|
||||
"answer": "Based on the code context, the **Router class** is LiteLLM's central routing component that manages multiple model deployments and implements various load balancing strategies.\n\n## Core Router Functionality\n\nThe Router class handles load balancing through several key mechanisms:\n\n### 1. **Routing Strategies**\nThe Router supports multiple routing strategies defined in `litellm/types/router.py`:\n\n```python\nclass RoutingStrategy(enum.Enum):\n LEAST_BUSY = \"least-busy\"\n LATENCY_BASED = \"latency-based-routing\"\n COST_BASED = \"cost-based-routing\"\n USAGE_BASED_ROUTING_V2 = \"usage-based-routing-v2\"\n USAGE_BASED_ROUTING = \"usage-based-routing\"\n PROVIDER_BUDGET_LIMITING = \"provider-budget-routing\"\n```\n\n### 2. **Specialized Router Types**\nThe Router manages multiple specialized routing strategies:\n\n- **Auto Router**: Semantic routing using embeddings\n- **Complexity Router**: Rule-based routing that classifies requests by complexity in <1ms\n- **Adaptive Router**: Bandit-based routing that learns from performance feedback\n- **Quality Router**: Routes based on quality requirements\n\n### 3. **Auto Router Deployments**\nAuto routers are identified by model names starting with `\"auto_router/\"`:\n\n```python\ndef _is_auto_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:\n if litellm_params.model.startswith(\"auto_router/complexity_router\"):\n return False # Handled by complexity_router\n if litellm_params.model.startswith(\"auto_router/adaptive_router\"):\n return False # Handled by adaptive_router\n if litellm_params.model.startswith(\"auto_router/\"):\n return True\n return False\n```\n\n### 4. **Adaptive Router Load Balancing**\nThe most sophisticated load balancing is handled by the **AdaptiveRouter**:\n\n- Uses **Thompson Sampling** (bandit algorithms) to learn optimal routing\n- Tracks performance via `BanditCell` with alpha/beta parameters\n- Maintains session state and owner caches\n- Classifies requests by type and routes accordingly\n- Supports quality tier preferences and cost considerations\n\n```python\nclass AdaptiveRouter:\n def __init__(self, router_name: str, config: AdaptiveRouterConfig, \n model_to_prefs: Dict[str, AdaptiveRouterPreferences], \n model_to_cost: Dict[str, float]):\n self._cells: Dict[Tuple[RequestType, str], BanditCell] = {}\n self._session_states: Dict[Tuple[str, str], SessionState] = {}\n```\n\n### 5. **Deployment Management**\nThe Router maintains:\n- `model_list`: List of available deployments\n- `model_name_to_deployment_indices`: Fast lookup mapping\n- Separate dictionaries for each router type (`adaptive_routers`, `complexity_routers`, etc.)\n\n### 6. **Load Balancing for Guardrails**\nThe Router also supports load balancing for guardrails:\n\n```python\ndef _should_use_guardrail_load_balancing(self, guardrail_name: str) -> bool:\n matching = [g for g in llm_router.guardrail_list \n if g.get(\"guardrail_name\") == guardrail_name]\n return len(matching) > 1\n```\n\n### 7. **Strategy-Specific Implementations**\nDifferent strategies use different approaches:\n- **Lowest Cost**: Routes to cheapest available option\n- **Lowest Latency**: Uses cached latency data with TTL and buffers\n- **Complexity Router**: Sub-1ms classification using weighted scoring across dimensions like token count, code presence, and reasoning markers\n\nThe Router architecture allows multiple routing strategies to coexist, with each deployment type handled by its specialized router instance while maintaining a unified interface for load balancing across model deployments.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer is highly accurate, comprehensive, and specific, providing detailed code examples and covering all routing strategies mentioned in the ground truth plus additional relevant implementation details."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q2",
|
||||
"query": "What is the Router class and how does it handle load balancing?",
|
||||
"category": "architecture",
|
||||
"approach": "grep",
|
||||
"context_tokens": 8801,
|
||||
"search_time_s": 0.117,
|
||||
"answer_time_s": 21.579,
|
||||
"total_time_s": 21.696,
|
||||
"answer": "Based on the code context, the **Router class** is the core load balancing component in LiteLLM that distributes requests across multiple model deployments. Here's how it works:\n\n## What is the Router Class?\n\nThe `Router` class (defined in `litellm/router.py`) is a sophisticated load balancer that manages multiple LLM deployments and routes requests intelligently based on various strategies, health checks, and constraints.\n\n## Key Components\n\n### Class Definition\n```python\nclass Router:\n model_names: set = set()\n cache_responses: Optional[bool] = False\n default_cache_time_seconds: int = 1 * 60 * 60 # 1 hour\n leastbusy_logger: Optional[LeastBusyLoggingHandler] = None\n lowesttpm_logger: Optional[LowestTPMLoggingHandler] = None\n```\n\n## Load Balancing Strategies\n\nThe Router supports multiple routing strategies defined in the `RoutingStrategy` enum:\n\n1. **\"simple-shuffle\"** (default) - Randomly picks deployments\n2. **\"least-busy\"** - Routes to deployment with lowest ongoing requests \n3. **\"usage-based-routing-v2\"** - Routes to deployment with lowest TPM usage\n4. **\"latency-based-routing\"** - Routes to deployment with lowest latency\n5. **\"cost-based-routing\"** - Routes to deployment with lowest cost per token\n\n## How Load Balancing Works\n\n### 1. Strategy Selection\nThe routing strategy is set during initialization:\n```python\ndef routing_strategy_init(self, routing_strategy: Union[RoutingStrategy, str], routing_strategy_args: dict):\n # Validates and sets the routing strategy\n```\n\n### 2. Deployment Selection Process\nThe Router uses different methods for selecting deployments:\n\n#### Async Deployment Selection\n```python\nasync def async_get_available_deployments(self, model_group: str, healthy_deployments: list, ...):\n if self.routing_strategy == \"usage-based-routing-v2\":\n deployment = await self.lowesttpm_logger_v2.async_get_available_deployments(...)\n elif self.routing_strategy == \"latency-based-routing\":\n deployment = await self.lowestlatency_logger.async_get_available_deployments(...)\n elif self.routing_strategy == \"simple-shuffle\":\n return simple_shuffle(...)\n elif self.routing_strategy == \"least-busy\":\n deployment = await self.leastbusy_logger.async_get_available_deployments(...)\n```\n\n### 3. Health Checks & Pre-call Validation\nBefore routing, the Router performs several checks:\n- **Health monitoring** - Tracks deployment health status\n- **Rate limiting** - Enforces TPM/RPM limits per deployment\n- **Cooldown management** - Temporarily excludes failing deployments\n- **Budget limiting** - Filters deployments exceeding budget limits\n\n### 4. Failure Handling & Cooldowns\n```python\n# Cooldown configuration\nallowed_fails: Optional[int] = None # Number of failures before cooldown\ncooldown_time: Optional[float] = None # Cooldown duration in seconds\n```\n\n## Advanced Load Balancing Features\n\n### Usage-Based Routing (TPM/RPM)\nThe `LowestTPMLoggingHandler_v2` class tracks token usage:\n```python\ndef pre_call_check(self, deployment: Dict) -> Optional[Dict]:\n # Checks if deployment is within RPM/TPM limits\n # Raises RateLimitError if over limit\n```\n\n### Weighted Selection\nFor simple-shuffle, the Router supports weighted selection based on deployment configuration.\n\n### Fallback Mechanisms\nThe Router supports multiple fallback types:\n- General fallbacks\n- Context window fallbacks \n- Content policy fallbacks\n\n## Configuration Example\n\n```python\nrouter = Router(\n model_list=[...], # List of deployments\n routing_strategy=\"least-busy\",\n routing_strategy_args={},\n allowed_fails=3,\n cooldown_time=60,\n retry_after=10,\n enable_pre_call_checks=True\n)\n```",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer is highly accurate, comprehensive, and includes specific code examples and file references that exceed the ground truth requirements while maintaining complete relevance to the question."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q3",
|
||||
"query": "How are API keys authenticated in the LiteLLM proxy server?",
|
||||
"category": "proxy",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 5601,
|
||||
"search_time_s": 0.019,
|
||||
"answer_time_s": 19.914,
|
||||
"total_time_s": 19.933,
|
||||
"answer": "Based on the code context, API keys in the LiteLLM proxy server are authenticated through a comprehensive multi-layered process. Here's how it works:\n\n## Key Extraction\n\nThe authentication process begins by extracting API keys from multiple possible sources in `auth/user_api_key_auth.py`:\n\n1. **Custom LiteLLM headers** (`x-litellm-api-key`) - takes precedence\n2. **Authorization header** (Bearer tokens)\n3. **Custom header names** (configurable via `litellm_key_header_name`)\n4. **Provider-specific headers** (Azure, Anthropic, Google AI Studio)\n\n```python\napi_key, passed_in_key = get_api_key(\n custom_litellm_key_header=custom_litellm_key_header,\n api_key=api_key,\n azure_api_key_header=azure_api_key_header,\n anthropic_api_key_header=anthropic_api_key_header,\n # ... other headers\n)\n```\n\n## Authentication Methods\n\nThe proxy supports multiple authentication methods:\n\n### 1. **Standard API Keys**\n- Keys starting with `sk-` are hashed using the `hash_token()` function\n- Validated against the database (`litellm_verificationtoken` table)\n- Support key blocking, expiration, and budget limits\n\n### 2. **JWT Tokens**\n- Enabled via `enable_jwt_auth: true` in proxy config\n- Detected using `jwt_handler.is_jwt(token=api_key)`\n- Can be routed to OAuth2 validation in certain cases\n\n### 3. **OAuth2 Authentication**\n- Enabled via `enable_oauth2_auth: true`\n- Used for upstream provider authentication\n- Supports token passthrough for MCP servers\n\n### 4. **Custom Authentication**\n- Enterprise custom auth functions\n- User-defined auth functions via `user_custom_auth`\n\n## Realtime Endpoint Authentication\n\nFor realtime endpoints (`/realtime/calls`), there's a special authentication flow in `realtime_endpoints/endpoints.py`:\n\n```python\n# Extract encrypted ephemeral key from Authorization header\nencrypted_token = auth_header.removeprefix(\"Bearer \").strip()\ndecrypted_token_value = decrypt_value_helper(\n value=encrypted_token,\n key=\"realtime_calls_auth\",\n)\n\n# Check token expiry\nif time.time() > expires_at:\n return Response(content=json.dumps({\"error\": \"Token has expired\"}))\n```\n\n## Key Management Operations\n\nThe proxy provides comprehensive key management through `management_endpoints/key_management_endpoints.py`:\n\n- **Generate keys**: `generate_key_fn()` - Creates new API keys with configurable limits\n- **Block keys**: `block_key()` - Blocks keys from making requests\n- **Update keys**: `update_key_fn()` - Modifies key parameters\n- **Regenerate keys**: `regenerate_key_fn()` - Creates new keys while optionally keeping old ones valid\n\n## Security Features\n\n1. **Token Hashing**: All API keys are hashed before storage using `_safe_hash_litellm_api_key()`\n2. **Header Cleaning**: The `clean_headers()` function removes sensitive auth headers before forwarding requests\n3. **Admin Access Control**: Admin-only operations require special permissions\n4. **Audit Logging**: All key operations are logged for accountability\n\n## Key Validation Process\n\nThe `UserAPIKeyAuth` object contains the validated key information:\n- User ID, team ID, organization ID\n- Budget limits and spending tracking\n- Model access permissions\n- Rate limiting configuration\n- Metadata and guardrails\n\nThe authentication flow ensures that only valid, non-blocked, non-expired keys with appropriate permissions can access the proxy endpoints.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer is highly accurate, comprehensive, and specific, covering all authentication methods mentioned in the ground truth plus extensive additional relevant details with precise file and function references."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q3",
|
||||
"query": "How are API keys authenticated in the LiteLLM proxy server?",
|
||||
"category": "proxy",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12006,
|
||||
"search_time_s": 0.194,
|
||||
"answer_time_s": 28.247,
|
||||
"total_time_s": 28.441,
|
||||
"answer": "Based on the provided code context, API keys are authenticated in the LiteLLM proxy server through a comprehensive multi-step process handled primarily by the `user_api_key_auth` function in `litellm/proxy/auth/user_api_key_auth.py`. Here's how the authentication works:\n\n## Authentication Flow\n\n### 1. **API Key Extraction**\nThe system extracts API keys from multiple header formats:\n- `Authorization: Bearer <token>`\n- `x-api-key: <token>` (custom LiteLLM header)\n- `api-key: <token>` (Azure format)\n- `x-api-key: <token>` (Anthropic format)\n- `x-goog-api-key: <token>` (Google AI Studio)\n- AWS Signature V4 format (for LangChain compatibility)\n\nThe `_get_bearer_token()` function normalizes these different formats:\n\n```python\ndef _get_bearer_token(api_key: str):\n if api_key.startswith(\"Bearer \"):\n api_key = api_key.replace(\"Bearer \", \"\")\n elif api_key.startswith(\"Basic \"):\n api_key = api_key.replace(\"Basic \", \"\")\n # ... handles other formats\n```\n\n### 2. **Master Key Authentication**\nThe system first checks if the provided key is a master key:\n```python\nif is_master_key_valid:\n # Substitute a stable alias for the raw master key\n _user_api_key_obj = await _return_user_api_key_auth_obj(\n user_obj=None,\n user_role=LitellmUserRoles.PROXY_ADMIN,\n api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,\n # ...\n )\n```\n\n### 3. **Token Validation and Lookup**\nFor non-master keys:\n\n1. **Hash the token**: If the key starts with \"sk-\", it's hashed for security\n2. **Database lookup**: The hashed token is looked up in the `LiteLLM_VerificationToken` table via the `get_key_object()` function\n3. **Cache check**: The system first checks the `UserApiKeyCache` for performance\n\n```python\nif api_key.startswith(\"sk-\"):\n api_key = hash_token(token=api_key)\n\nvalid_token = await get_key_object(\n hashed_token=api_key,\n prisma_client=prisma_client,\n user_api_key_cache=user_api_key_cache,\n # ...\n)\n```\n\n### 4. **JWT Token Handling**\nThe system also supports JWT tokens and OAuth2:\n- JWT tokens are validated using the `JWTHandler` class\n- OAuth2 tokens are handled via the `Oauth2Handler`\n- Token routing decisions are made based on configuration\n\n### 5. **Authorization Checks**\nAfter authentication, the system performs authorization checks via `_run_centralized_common_checks()`:\n\n- **Model access validation**: Ensures the key can access requested models\n- **Budget limits**: Checks spending limits and quotas\n- **Rate limiting**: Validates TPM/RPM limits\n- **Team/organization permissions**: Verifies team and org-level access\n- **Guardrail validation**: Runs any configured guardrails\n\n### 6. **User Object Creation**\nFinally, the system creates a `UserAPIKeyAuth` object containing:\n- User ID and role information\n- Team and organization details\n- Budget and rate limit information\n- Model access permissions\n- End-user parameters\n\n## Key Security Features\n\n1. **Token Hashing**: API keys are hashed before storage/lookup\n2. **Caching**: Validated tokens are cached for performance\n3. **View-based Queries**: Uses database views like `LiteLLM_VerificationTokenView` for efficient lookups\n4. **Multiple Auth Methods**: Supports various authentication schemes (Bearer, API-Key, etc.)\n5. **Comprehensive Validation**: Includes budget, rate limiting, and permission checks\n\nThe authentication system is designed to be both secure and performant, with extensive caching and support for multiple authentication patterns commonly used in AI/ML applications.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer perfectly matches the ground truth with comprehensive detail, correctly identifies the key function and file, mentions the database table and Prisma, covers all authentication methods, and provides extensive specific implementation details."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q4",
|
||||
"query": "How does litellm.completion() work internally - what is the call flow?",
|
||||
"category": "core",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 2481,
|
||||
"search_time_s": 0.024,
|
||||
"answer_time_s": 15.661,
|
||||
"total_time_s": 15.685,
|
||||
"answer": "Based on the code context, here's how `litellm.completion()` works internally:\n\n## Core Call Flow\n\n### 1. **Request Preparation & Validation**\n- The completion request starts by validating environment variables and headers through provider-specific config classes (e.g., `AnthropicConfig().validate_environment()`)\n- Request parameters are transformed using `ProviderConfigManager.get_provider_chat_config()` and `config.transform_request()`\n\n### 2. **Pre-Call Logging & Hooks**\n- `logging_obj.pre_call()` is invoked with input messages, API key, and request data\n- Custom hooks like `SkillsInjectionHook.async_pre_call_hook()` process special parameters (e.g., container.skills)\n\n### 3. **Routing Logic**\nThe completion flow branches based on several factors:\n\n**Async vs Sync:**\n```python\nif acompletion is True:\n # Routes to async handlers\n return self.acompletion_function() or self.acompletion_stream_function()\nelse:\n # Routes to sync handlers\n```\n\n**Streaming vs Non-Streaming:**\n```python\nif stream is True:\n # Returns CustomStreamWrapper for streaming responses\n return CustomStreamWrapper(completion_stream=completion_stream, ...)\nelse:\n # Makes direct HTTP call for non-streaming\n```\n\n### 4. **HTTP Request Execution**\n- For sync calls: Uses `make_sync_call()` or direct `client.post()` via `HTTPHandler`\n- For async calls: Uses `AsyncHTTPHandler` \n- Request data is sent as JSON to the provider's API base URL\n\n### 5. **Response Processing**\n- Raw responses are transformed back to LiteLLM format using `config.transform_response()`\n- Streaming responses are wrapped in `CustomStreamWrapper` which implements `__iter__()` and `__next__()`\n- The wrapper handles chunk processing and conversion to standard completion objects\n\n### 6. **Error Handling**\nProvider-specific exceptions are caught and transformed:\n```python\nexcept Exception as e:\n raise AnthropicError(message=error_text, status_code=status_code, headers=error_headers)\n```\n\n### 7. **Post-Call Processing**\n- Success/failure logging through `_client_async_logging_helper()`\n- Post-call hooks process responses (e.g., code execution for skills)\n\n## Key Components\n\n- **`litellm/llms/anthropic/chat/handler.py`**: Main completion handler\n- **`litellm/litellm_core_utils/streaming_handler.py`**: Manages streaming responses\n- **`litellm/utils.py`**: Contains stream wrapper classes with `__next__()` and `__anext__()` methods\n- **`ProviderConfigManager`**: Handles request/response transformation between LiteLLM and provider formats\n\nThe architecture uses a provider-agnostic interface that transforms requests to provider-specific formats, makes HTTP calls, and transforms responses back to a unified LiteLLM format.",
|
||||
"scores": {
|
||||
"accuracy": 4,
|
||||
"completeness": 4,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 18,
|
||||
"brief_explanation": "The answer provides accurate technical details with excellent specificity including file paths and code examples, though it focuses heavily on one provider (Anthropic) rather than the general flow through litellm/main.py as mentioned in the ground truth."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q4",
|
||||
"query": "How does litellm.completion() work internally - what is the call flow?",
|
||||
"category": "core",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12012,
|
||||
"search_time_s": 0.084,
|
||||
"answer_time_s": 19.952,
|
||||
"total_time_s": 20.035,
|
||||
"answer": "Based on the code context, here's how `litellm.completion()` works internally:\n\n## Main Entry Point\n\nThe `completion()` function in `litellm/main.py` is the primary entry point. It handles both sync and async calls by detecting if it's called from an async context and routing appropriately.\n\n## Core Call Flow\n\n### 1. **Provider Detection & Model Resolution**\n```python\nmodel, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(\n model=model,\n custom_llm_provider=custom_llm_provider,\n api_base=api_base,\n api_key=api_key,\n)\n```\nThe `get_llm_provider()` function from `litellm.litellm_core_utils.get_llm_provider_logic` determines which LLM provider to use based on the model name.\n\n### 2. **Parameter Processing & Validation**\n- Processes OpenAI-compatible parameters\n- Handles provider-specific parameter mapping\n- Validates environment variables and API keys\n- Sets up headers and authentication\n\n### 3. **Message Processing**\nThe system processes messages through several transformation steps:\n- **Tool Call ID Processing**: Removes Gemini thought signatures from tool call IDs for non-Gemini providers\n- **Provider-specific transformations**: Each provider has its own transformation logic\n\n### 4. **Provider Routing**\nBased on the `custom_llm_provider`, the call is routed to the appropriate provider handler:\n\n```python\n# Examples from the code:\nif custom_llm_provider == \"openai\":\n # Route to OpenAI handler\nelif custom_llm_provider == \"anthropic\": \n # Route to Anthropic handler\nelif custom_llm_provider == \"azure\":\n # Route to Azure handler\n# ... etc for each provider\n```\n\n### 5. **Response Processing**\nAfter getting the raw response from the provider:\n- **Response Transformation**: Convert provider-specific response to standardized `ModelResponse` format using `convert_to_model_response_object()`\n- **Streaming Handling**: For streaming responses, wrap in `CustomStreamWrapper`\n- **Post-processing**: Apply rules, JSON schema validation, etc.\n\n### 6. **Caching Integration** \nFrom `litellm/caching/caching_handler.py`:\n- Check cache before making API call\n- Store responses in cache after successful calls\n- Handle cache hits by converting cached results back to `ModelResponse` objects\n\n### 7. **Router Integration**\nWhen using `Router` (from `litellm/router.py`):\n- **Load Balancing**: Select deployment based on routing strategy\n- **Fallback Handling**: Automatic failover to backup deployments\n- **Streaming Fallbacks**: Special handling for mid-stream failures via `_completion_streaming_iterator()`\n\n## Key Components\n\n### Streaming Handler\nThe `CustomStreamWrapper` in `litellm/litellm_core_utils/streaming_handler.py` manages streaming responses:\n- Handles both sync and async iteration\n- Maintains response metadata across chunks\n- Provides fallback capabilities for the router\n\n### Exception Handling\n- Provider-specific exceptions are mapped to standardized LiteLLM exceptions\n- Retry logic and fallback mechanisms are built-in\n- Detailed error messages preserve original provider context\n\n### Logging & Observability\n- Comprehensive logging throughout the call flow\n- Integration with various observability providers\n- Cost tracking and usage analytics\n\n## Mock Responses\nFor testing, the system supports mock responses via `mock_completion_streaming_obj()` and related functions, allowing simulation of provider responses without actual API calls.\n\nThis architecture provides a unified interface while maintaining provider-specific optimizations and handling the complexities of different LLM APIs transparently.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 4,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 19,
|
||||
"brief_explanation": "The answer is highly accurate and specific with detailed code references and function names, though it goes beyond the basic call flow to include additional architectural details not in the ground truth."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q5",
|
||||
"query": "What caching mechanisms does LiteLLM support?",
|
||||
"category": "feature",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 4517,
|
||||
"search_time_s": 0.019,
|
||||
"answer_time_s": 13.888,
|
||||
"total_time_s": 13.906,
|
||||
"answer": "Based on the code context, LiteLLM supports several caching mechanisms:\n\n## Primary Cache Types\n\nLiteLLM supports multiple cache backends through the `Cache` class in `litellm/caching/caching.py`:\n\n1. **Local Cache** (`LiteLLMCacheType.LOCAL`) - Default in-memory caching\n2. **Redis Cache** - External Redis-based caching\n3. **Redis Semantic Cache** - Semantic caching using Redis with embeddings\n4. **Qdrant Semantic Cache** - Vector-based semantic caching using Qdrant\n5. **S3 Cache** - AWS S3-based caching\n6. **Azure Blob Cache** - Azure Blob Storage caching \n7. **GCS Cache** - Google Cloud Storage caching\n8. **Disk Cache** - File system-based caching\n\n## Dual Cache System\n\nLiteLLM implements a sophisticated **DualCache** system (`litellm/caching/dual_cache.py`) that:\n\n- Updates both Redis and in-memory cache simultaneously\n- Ensures data consistency between local and remote caches\n- Provides fallback when Redis is unavailable\n- Supports configurable TTL for both cache layers\n\n## Supported Operations\n\nThe caching system supports various LiteLLM operations:\n\n```python\nsupported_call_types = [\n \"completion\", \"acompletion\", \n \"embedding\", \"aembedding\",\n \"atranscription\", \"transcription\", \n \"atext_completion\", \"text_completion\",\n \"arerank\", \"rerank\",\n \"responses\", \"aresponses\"\n]\n```\n\n## Streaming Cache Support\n\nLiteLLM provides specialized streaming response caching through:\n\n- `async_cache_streaming_response()` - Async streaming cache handling\n- `cache_streaming_response()` - Sync streaming cache handling \n- `LLMCachingHandler` - Manages streaming chunk assembly and caching\n\n## Cache Configuration\n\nThe `Cache` class constructor supports extensive configuration including:\n\n- TTL settings for different cache layers\n- Redis cluster configuration\n- Semantic search parameters (embedding models, similarity thresholds)\n- Cloud storage credentials and paths\n- Cache modes (default_on/default_off)\n\n## Cache Management\n\nKey management functions include:\n\n- `disable_cache()` - Completely disables caching and removes callbacks\n- `add_cache()` - Adds results to cache\n- `get_cache()` - Retrieves cached results\n- `set_cache()` - Sets cache values with TTL support\n\nThe caching system integrates deeply with LiteLLM's callback system and logging infrastructure to provide transparent caching across all supported LLM operations.",
|
||||
"scores": {
|
||||
"accuracy": 4,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 19,
|
||||
"brief_explanation": "The answer is highly detailed and accurate, covering all ground truth cache types plus additional ones like Azure Blob and GCS, with excellent specificity referencing specific files and classes."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q5",
|
||||
"query": "What caching mechanisms does LiteLLM support?",
|
||||
"category": "feature",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12013,
|
||||
"search_time_s": 0.122,
|
||||
"answer_time_s": 15.599,
|
||||
"total_time_s": 15.721,
|
||||
"answer": "Based on the code context, LiteLLM supports several caching mechanisms:\n\n## Primary Cache Types\n\nLiteLLM supports the following cache types defined in `litellm/caching/caching.py`:\n\n1. **Local/In-Memory Cache** (`LiteLLMCacheType.LOCAL`)\n - Uses `InMemoryCache` class\n - Stores cached responses in memory\n\n2. **Redis Cache** (`LiteLLMCacheType.REDIS`)\n - Uses `RedisCache` class for single Redis instances\n - Uses `RedisClusterCache` for Redis clusters\n - Supports Redis startup nodes configuration\n\n3. **Semantic Caches**\n - **Redis Semantic Cache** (`LiteLLMCacheType.REDIS_SEMANTIC`)\n - Uses `RedisSemanticCache` class\n - Supports similarity-based caching with configurable thresholds\n - **Qdrant Semantic Cache** (`LiteLLMCacheType.QDRANT_SEMANTIC`)\n - Uses `QdrantSemanticCache` class\n - Requires API base, API key, collection name, and similarity threshold\n\n4. **Cloud Storage Caches**\n - **S3 Cache** (`LiteLLMCacheType.S3`)\n - Uses `S3Cache` class\n - Supports AWS S3 configuration\n - **Azure Blob Cache**\n - Uses `AzureBlobCache` class\n - **Google Cloud Storage Cache**\n - Uses `GCSCache` class\n\n5. **Disk Cache**\n - Uses `DiskCache` class\n - Stores cached responses on local filesystem\n\n## Cache Configuration\n\n### Basic Usage\n```python\nfrom litellm import Cache\n\n# Local cache\ncache = Cache(type=\"local\")\n\n# Redis cache\ncache = Cache(\n type=\"redis\",\n host=\"localhost\",\n port=6379,\n password=\"your_password\"\n)\n```\n\n### Cache Modes\nThe `Cache` class supports two modes defined in `CacheMode`:\n- `default_on`: Cache is always enabled\n- `default_off`: Cache is opt-in per request\n\n## Advanced Caching Features\n\n### Dual Cache\nThe codebase uses `DualCache` which combines Redis and in-memory caching:\n```python\nfrom litellm.caching.caching import DualCache, InMemoryCache\n\ndual_cache = DualCache(\n redis_cache=redis_cache,\n in_memory_cache=InMemoryCache()\n)\n```\n\n### Caching Groups\nLiteLLM supports caching across model groups through the `caching_groups` parameter, allowing you to cache responses across different models in the same group.\n\n### TTL Configuration\n- `default_in_memory_ttl`: TTL for in-memory cache\n- `default_redis_ttl`: TTL for Redis cache\n- `ttl`: General TTL setting\n\n### Router Integration\nThe `Router` class integrates caching through the `cache_responses` parameter and supports Redis configuration for distributed caching across multiple instances.\n\n### Context Caching\nLiteLLM also supports provider-specific context caching (like Vertex AI's context caching) for optimizing repeated prompts with shared context.\n\nAll caching mechanisms are handled through the `Cache` class in `litellm/caching/caching.py`, with the `LLMCachingHandler` managing cache operations for LLM API requests.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer is highly accurate, comprehensive, and provides specific implementation details with file paths, class names, and code examples that go beyond the ground truth while covering all mentioned cache types."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q6",
|
||||
"query": "How does LiteLLM transform Anthropic Claude function/tool calling to OpenAI format?",
|
||||
"category": "provider",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 1355,
|
||||
"search_time_s": 0.015,
|
||||
"answer_time_s": 11.804,
|
||||
"total_time_s": 11.819,
|
||||
"answer": "Based on the code context provided, I can see some information about how LiteLLM handles transformations between Anthropic and OpenAI formats, but the specific details of transforming Anthropic Claude function/tool calling to OpenAI format are not fully shown in the provided code snippets.\n\nHere's what I can determine from the available code:\n\n## Key Components for Anthropic-OpenAI Transformation\n\n1. **Main Transformation Class**: The transformation is handled by `LiteLLMAnthropicMessagesAdapter` class, which is referenced in `litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py`.\n\n2. **Translatable Parameters**: The `translatable_anthropic_params()` function identifies which Anthropic parameters need translation to OpenAI format:\n ```python\n return [\n \"messages\",\n \"metadata\", \n \"system\",\n \"tool_choice\",\n \"tools\",\n \"thinking\",\n \"output_format\",\n \"output_config\",\n ]\n ```\n\n3. **Tool Name Mapping**: The code shows support for handling tool name truncation issues. OpenAI has a 64-character limit for tool names, so there's a `tool_name_mapping` parameter that maps truncated names back to original names:\n ```python\n def translate_completion_output_params(\n self, response: ModelResponse, \n tool_name_mapping: Optional[Dict[str, str]] = None\n ) -> Optional[AnthropicMessagesResponse]:\n ```\n\n4. **Bidirectional Translation**: The adapter supports both directions:\n - **Response Translation**: `translate_openai_response_to_anthropic()` converts OpenAI responses back to Anthropic format\n - **Streaming Support**: `translate_completion_output_params_streaming()` handles streaming responses\n\n## Tool Choice Transformation\n\nFrom the `LiteLLMCompletionResponsesConfig` class, we can see how tool_choice is normalized using `_transform_tool_choice()`, which handles various formats and converts them to OpenAI-compatible strings like \"auto\", \"none\", or \"required\".\n\nHowever, the specific implementation details of how Anthropic's tool calling format is converted to OpenAI's function calling format (parameter mapping, schema transformation, etc.) are not visible in the provided code snippets. The actual transformation logic would be in the `LiteLLMAnthropicMessagesAdapter.translate_openai_response_to_anthropic()` method and its counterpart for request transformation, which are not included in the code context.",
|
||||
"scores": {
|
||||
"accuracy": 3,
|
||||
"completeness": 2,
|
||||
"specificity": 3,
|
||||
"relevance": 4,
|
||||
"total": 12,
|
||||
"brief_explanation": "The answer correctly identifies some transformation components and provides accurate details about tool name mapping and bidirectional translation, but misses the specific file mentioned in ground truth and lacks details about the actual transformation process from Anthropic's tool_use content blocks to OpenAI's tool_calls array."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q6",
|
||||
"query": "How does LiteLLM transform Anthropic Claude function/tool calling to OpenAI format?",
|
||||
"category": "provider",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12004,
|
||||
"search_time_s": 0.132,
|
||||
"answer_time_s": 17.817,
|
||||
"total_time_s": 17.949,
|
||||
"answer": "Based on the code context, LiteLLM transforms Anthropic Claude function/tool calling to OpenAI format through several key transformation methods in the `AnthropicConfig` class:\n\n## Tool Use to OpenAI Format Conversion\n\nThe main transformation happens in the `convert_tool_use_to_openai_format` static method:\n\n```python\n@staticmethod\ndef convert_tool_use_to_openai_format(\n anthropic_tool_content: Dict[str, Any], \n index: int,\n) -> ChatCompletionToolCallChunk:\n \"\"\"\n Convert Anthropic tool_use format to OpenAI ChatCompletionToolCallChunk format.\n \n Args:\n anthropic_tool_content: Anthropic tool_use content block with format:\n {\"type\": \"tool_use\", \"id\": \"...\", \"name\": \"...\", \"input\": {...}}\n index: The index of this tool call\n \"\"\"\n tool_call = ChatCompletionToolCallChunk(\n id=anthropic_tool_content[\"id\"],\n type=\"function\",\n function=ChatCompletionToolCallFunctionChunk(\n name=anthropic_tool_content[\"name\"],\n # arguments would be derived from anthropic_tool_content[\"input\"]\n )\n )\n```\n\n## Tool Choice Mapping\n\nThe `_map_tool_choice` method converts OpenAI tool choice formats to Anthropic's format:\n\n- **\"auto\"** \u2192 `AnthropicMessagesToolChoice(type=\"auto\")`\n- **\"required\"** \u2192 `AnthropicMessagesToolChoice(type=\"any\")` \n- **\"none\"** \u2192 `AnthropicMessagesToolChoice(type=\"none\")`\n- **Specific function** \u2192 `AnthropicMessagesToolChoice(type=\"tool\", name=tool_name)`\n\n## Tool Schema Transformation\n\nThe `_map_tool_helper` method handles tool parameter transformation:\n\n1. **Schema Normalization**: Ensures `input_schema.type` is \"object\" for Anthropic compatibility\n2. **Parameter Filtering**: Filters parameters to only include fields allowed by `AnthropicInputSchema`\n3. **Required Fields**: Adds required fields like `properties` if missing\n\n```python\n# Anthropic requires input_schema.type to be \"object\"\nif _input_schema.get(\"type\") != \"object\":\n _input_schema = dict(_input_schema)\n _input_schema[\"type\"] = \"object\"\n \nif \"properties\" not in _input_schema:\n _input_schema[\"properties\"] = {}\n```\n\n## Tool Result Handling\n\nFor tool results (function outputs), the transformation involves:\n\n1. **Tool Use ID Sanitization**: Uses `_sanitize_anthropic_tool_use_id` to ensure IDs match Anthropic's pattern `^[a-zA-Z0-9_-]+$`\n2. **Tool Result Format**: Converts to Anthropic's `tool_result` format:\n\n```python\nanthropic_tool_result = AnthropicMessagesToolResultParam(\n type=\"tool_result\",\n tool_use_id=sanitized_tool_use_id,\n content=anthropic_content,\n)\n```\n\n## Parallel Tool Use Handling\n\nAnthropic uses an inverted flag system:\n- OpenAI's `parallel_tool_calls: true` \u2192 Anthropic's `disable_parallel_tool_use: false`\n- OpenAI's `parallel_tool_calls: false` \u2192 Anthropic's `disable_parallel_tool_use: true`\n\n## Schema Filtering\n\nThe `filter_anthropic_output_schema` method recursively processes JSON schemas to remove unsupported fields and ensure compatibility, automatically adding `additionalProperties=false` for object schemas as required by Anthropic.\n\nThis comprehensive transformation system allows LiteLLM to seamlessly convert between OpenAI's function calling format and Anthropic's tool use format while maintaining compatibility and proper error handling.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer provides highly accurate, complete, and specific details about LiteLLM's transformation process with extensive code examples and method references that align perfectly with the ground truth."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q7",
|
||||
"query": "How does the proxy handle budget management and spend tracking?",
|
||||
"category": "proxy",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 3084,
|
||||
"search_time_s": 0.014,
|
||||
"answer_time_s": 18.522,
|
||||
"total_time_s": 18.536,
|
||||
"answer": "Based on the code context, the LiteLLM proxy implements a comprehensive budget management and spend tracking system with several key components:\n\n## Budget Management Architecture\n\n### 1. **Budget Types and Hierarchy**\nThe proxy supports multiple budget levels:\n- **API Keys**: Individual key budgets (`spend:key:{token}`)\n- **Teams**: Team-level budgets (`spend:team:{team_id}`)\n- **Users**: User-level budgets (`spend:user:{user_id}`)\n- **End Users**: End-user budgets (`spend:end_user:{user_id}`)\n- **Organizations**: Organization budgets\n- **Tags**: Tag-based budgets (Enterprise feature)\n- **Providers**: Provider-specific budgets\n- **Models**: Model-specific budgets\n\n### 2. **Budget Counter System**\nThe system uses `_BudgetCounter` objects to track spending:\n```python\nclass _BudgetCounter:\n counter_key: str\n max_budget: float\n fallback_spend: float\n entity_type: str\n entity_id: str\n source_cache_key: Optional[str] = None\n spend_log_entity_id: Optional[str] = None\n window_start: Optional[datetime] = None\n```\n\n## Spend Tracking Mechanisms\n\n### 1. **Cross-Pod Counter System**\n- Uses Redis-first caching with in-memory fallback\n- Counter keys follow pattern: `spend:{type}:{id}`\n- Function `get_current_spend()` provides consistent spend retrieval\n\n### 2. **Budget Reservation System**\nFor request processing, the proxy implements budget reservation:\n- **Pre-request**: Reserves estimated cost against budgets\n- **Post-request**: Reconciles actual cost vs reserved amount\n- **Error handling**: Releases reservations on failures\n\nKey functions in `budget_reservation.py`:\n- `_reserve_counter()`: Reserves budget for a request\n- `reconcile_budget_reservation()`: Updates actual vs reserved costs\n- `release_budget_reservation()`: Releases unused reservations\n\n### 3. **Budget Enforcement**\nBudget checks occur at multiple points:\n- `_check_end_user_budget()`: Validates end-user budgets\n- `_team_max_budget_check()`: Enforces team budget limits\n- Budget validation happens before request processing\n\n## Budget Management Endpoints\n\n### 1. **Budget CRUD Operations**\n- `new_budget()`: Creates new budget objects with validation\n- `update_budget()`: Updates existing budgets\n- `provider_budgets()`: Returns provider budget status\n\n### 2. **Budget Configuration**\nBudgets support:\n- **Duration-based resets**: \"30d\", \"1h\", etc.\n- **Soft/hard limits**: `max_budget` and `soft_budget`\n- **Rate limiting**: TPM/RPM limits\n- **Model-specific budgets**: Per-model budget allocation\n\n## Budget Reset System\n\nThe `ResetBudgetJob` class handles automatic budget resets:\n```python\n# Resets spend counters and updates budget_reset_at\nawait ResetBudgetJob._reset_budget_for_team(team, current_time)\nawait ResetBudgetJob._reset_budget_for_user(user, current_time)\n```\n\n## Error Handling and Reliability\n\n### 1. **Fallback Mechanisms**\n- If Redis fails, falls back to cached database values\n- `_release_applied_entries_best_effort()` handles cleanup on failures\n\n### 2. **Counter Invalidation**\n- `_invalidate_spend_counter()` clears corrupted counters\n- Automatic counter invalidation on reservation failures\n\n## Key Features\n\n1. **Real-time tracking**: Immediate budget enforcement\n2. **Multi-level budgets**: Hierarchical budget structure\n3. **Automatic resets**: Time-based budget renewal\n4. **Enterprise features**: Advanced tagging and organization budgets\n5. **Resilient design**: Graceful degradation on infrastructure failures\n\nThe system ensures accurate spend tracking while maintaining high availability through its layered caching approach and comprehensive error handling.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer is highly accurate, comprehensive, and provides extensive specific details about classes, functions, and implementation while directly addressing the budget management and spend tracking question."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q7",
|
||||
"query": "How does the proxy handle budget management and spend tracking?",
|
||||
"category": "proxy",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12005,
|
||||
"search_time_s": 0.138,
|
||||
"answer_time_s": 20.625,
|
||||
"total_time_s": 20.763,
|
||||
"answer": "Based on the code context, the LiteLLM proxy implements comprehensive budget management and spend tracking through several interconnected systems:\n\n## Budget Management Architecture\n\n### 1. **Multi-Level Budget System**\nThe proxy supports budgets at multiple levels:\n\n- **Proxy-level**: Global budget for the entire proxy (`litellm.max_budget`)\n- **Organization-level**: Budget limits for organizations\n- **Team-level**: Budget limits for teams (`max_budget`, `soft_budget`)\n- **User-level**: Individual user budget limits\n- **API Key-level**: Per-key budget limits\n- **Team Member-level**: Individual team member budgets within teams\n\n### 2. **Budget Configuration Classes**\n\nThe system uses several key classes for budget management:\n\n```python\nclass BudgetLimitEntry(LiteLLMPydanticObjectBase):\n \"\"\"A single budget window with its own limit and independent reset schedule.\"\"\"\n budget_duration: str # e.g. \"24h\", \"7d\", \"30d\"\n max_budget: float # max spend in USD for this window\n reset_at: Optional[datetime] = None\n```\n\n### 3. **Budget Tracking Components**\n\n#### **Database Tables**\n- **LiteLLM_SpendLogs**: Records all spending transactions with fields like:\n - `spend`: Cost of the request\n - `total_tokens`, `prompt_tokens`, `completion_tokens`\n - `api_key`, `user_id`, `team_id`\n - `startTime`, `endTime`, `completionStartTime`\n - `model`, `api_base`, `call_type`\n\n#### **Caching System**\n- **`user_api_key_cache`**: Caches user/key spend data for fast access\n- **`spend_counter_cache`**: Tracks real-time spend counters\n- **Redis integration**: For distributed spend tracking\n\n## Spend Tracking Implementation\n\n### 1. **Real-time Spend Updates**\nThe proxy uses several mechanisms for tracking spend:\n\n```python\n# From proxy_server.py\nasync def _warm_global_spend_cache(\n litellm_proxy_admin_name: str,\n user_api_key_cache: UserApiKeyCache,\n prisma_client: PrismaClient,\n)\n```\n\n### 2. **Budget Validation**\nBudget limits are enforced at request time through validation functions:\n\n```python\n# From team_endpoints.py\nasync def _check_org_team_limits(\n org_table: LiteLLM_OrganizationTable,\n data: Union[NewTeamRequest, UpdateTeamRequest],\n prisma_client: PrismaClient,\n) -> None:\n # Validate team budget against organization's max_budget\n if (\n data.max_budget is not None\n and org_table.litellm_budget_table is not None\n and org_table.litellm_budget_table.max_budget is not None\n and data.max_budget > org_table.litellm_budget_table.max_budget\n ):\n raise HTTPException(...)\n```\n\n### 3. **Prometheus Metrics Integration**\nThe system exports budget and spend metrics to Prometheus:\n\n```python\n# From prometheus.py\nself.litellm_spend_metric = self._counter_factory(\n \"litellm_spend_metric\",\n \"Total spend on LLM requests\",\n)\n\nself.litellm_remaining_team_budget_metric = self._gauge_factory(\n \"litellm_remaining_team_budget_metric\",\n \"Remaining budget for team\",\n)\n```\n\n## Budget Management Features\n\n### 1. **Team Member Budget Handler**\n```python\nclass TeamMemberBudgetHandler:\n @staticmethod\n async def create_team_member_budget_table(\n data: Union[NewTeamRequest, LiteLLM_TeamTable],\n team_member_budget: Optional[float] = None,\n team_member_rpm_limit: Optional[int] = None,\n team_member_tpm_limit: Optional[int] = None,\n team_member_budget_duration: Optional[str] = None,\n ) ->",
|
||||
"scores": {
|
||||
"accuracy": 4,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 19,
|
||||
"brief_explanation": "The answer is highly accurate, comprehensive, and specific with detailed code examples and class references, though it provides more extensive detail than the concise ground truth."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q8",
|
||||
"query": "What database schema does the LiteLLM proxy use and how are migrations handled?",
|
||||
"category": "infrastructure",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 3998,
|
||||
"search_time_s": 0.022,
|
||||
"answer_time_s": 16.004,
|
||||
"total_time_s": 16.026,
|
||||
"answer": "Based on the code context, here's how the LiteLLM proxy handles database schema and migrations:\n\n## Database Schema\n\nThe LiteLLM proxy uses **Prisma** as its ORM and database management system. The schema includes several key tables:\n\n### Core Tables:\n- **`litellm_proxymodeltable`** - Stores model configurations and deployments\n- **`litellm_organizationtable`** - Stores organization data with `organization_alias` field\n- **`litellm_teamtable`** - Stores team information including model access lists\n- **`litellm_usertable`** - User management\n- **Various audit and logging tables**\n\n### Entity Types:\nThe system tracks these entity types (from `Litellm_EntityType` enum):\n- `KEY`, `USER`, `END_USER`, `TEAM`, `TEAM_MEMBER`\n- `ORGANIZATION`, `PROJECT`, `TAG`, `AGENT`\n- `PROXY` (global proxy level entity)\n\n## Migration Handling\n\nMigrations are managed through the `PrismaManager` class in `litellm/proxy/db/prisma_client.py`:\n\n### Setup Methods:\nThe `setup_database()` function provides two approaches:\n\n1. **Database Push (Default)**:\n```python\nsubprocess.run([\n \"prisma\", \"db\", \"push\", \n \"--accept-data-loss\", \n \"--skip-generate\"\n], timeout=60, check=True)\n```\n\n2. **Migration Deploy (Enterprise)**:\n```python\nif use_migrate:\n from litellm_proxy_extras.utils import ProxyExtrasDBManager\n return ProxyExtrasDBManager.setup_database(\n use_migrate=use_migrate,\n use_v2_resolver=use_v2_resolver,\n )\n```\n\n### Migration Features:\n\n- **Retry Logic**: Up to 4 attempts with random delays (5-15 seconds) on failure\n- **Timeout Handling**: 60-second timeout per attempt\n- **V2 Resolver**: Optional `use_v2_resolver` parameter to avoid schema thrashing during rolling deploys\n- **Directory Management**: Automatically changes to the correct Prisma directory (`PrismaManager._get_prisma_dir()`)\n\n### Enterprise vs Standard:\n- **Standard**: Uses `prisma db push` for schema synchronization\n- **Enterprise**: Uses `prisma migrate deploy` through `ProxyExtrasDBManager` for proper versioned migrations\n\n### Database Connection:\nThe system uses a connection pool with configurable limits:\n```python\nclass LiteLLMDatabaseConnectionPool(Enum):\n database_connection_pool_limit = 10\n database_connection_pool_timeout = 60\n```\n\nThe schema supports advanced features like pass-through endpoints, SCIM integration, skills management, and comprehensive audit logging, all managed through Prisma's type-safe database operations.",
|
||||
"scores": {
|
||||
"accuracy": 4,
|
||||
"completeness": 3,
|
||||
"specificity": 5,
|
||||
"relevance": 4,
|
||||
"total": 16,
|
||||
"brief_explanation": "Answer is highly detailed and specific with correct technical information, but mentions different table names than ground truth and includes extensive additional details beyond what was asked."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q8",
|
||||
"query": "What database schema does the LiteLLM proxy use and how are migrations handled?",
|
||||
"category": "infrastructure",
|
||||
"approach": "grep",
|
||||
"context_tokens": 10910,
|
||||
"search_time_s": 0.102,
|
||||
"answer_time_s": 19.55,
|
||||
"total_time_s": 19.652,
|
||||
"answer": "Based on the code context, I can provide details about LiteLLM proxy's database schema and migration handling:\n\n## Database Schema\n\nLiteLLM proxy uses **Prisma** as its ORM with PostgreSQL as the primary database. The main database tables include:\n\n### Core Tables\n- **`LiteLLM_VerificationToken`** - API keys/tokens with spending tracking, rate limits, and permissions\n- **`LiteLLM_TeamTable`** - Teams with budgets, model access, and member management\n- **`LiteLLM_UserTable`** - User accounts and profiles\n- **`LiteLLM_SpendLogs`** - Detailed spending/usage logs for analytics\n- **`LiteLLM_OrganizationTable`** - Organizations containing teams\n- **`LiteLLM_ProjectTable`** - Projects for organizing resources\n\n### Supporting Tables\n- **`LiteLLM_TeamMembership`** - User-team relationships with roles\n- **`LiteLLM_BudgetTable`** - Budget configurations and limits\n- **`LiteLLM_ModelTable`** - Model configurations and aliases\n- **`LiteLLM_ObjectPermissionTable`** - Fine-grained permissions\n\n## Database Views\n\nThe system creates several views for performance optimization:\n\n```sql\n-- Main view joining tokens with team data\nCREATE VIEW \"LiteLLM_VerificationTokenView\" AS \nSELECT v.*, t.spend AS team_spend, t.max_budget AS team_max_budget, \n t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit \nFROM \"LiteLLM_VerificationToken\" v \nLEFT JOIN \"LiteLLM_TeamTable\" t ON v.team_id = t.team_id;\n\n-- Analytics views\nCREATE VIEW \"MonthlyGlobalSpend\" AS ...\nCREATE VIEW \"Last30dKeysBySpend\" AS ...\nCREATE VIEW \"Last30dModelsBySpend\" AS ...\n```\n\n## Migration Handling\n\n### 1. Prisma Migrations\nThe proxy uses Prisma's migration system. In `litellm/proxy/proxy_server.py`:\n\n```python\n# Database migration check on startup\ntry:\n await prisma_client.db.query_raw(\"SELECT 1\")\nexcept Exception as e:\n if \"P3018\" in str(e) or \"P3009\" in str(e):\n verbose_proxy_logger.debug(\"CRITICAL: DATABASE MIGRATION FAILED\")\n verbose_proxy_logger.debug(\"Your database is in a 'dirty' state.\")\n verbose_proxy_logger.debug(\"FIX: Run 'prisma migrate resolve --applied <migration_name>'\")\n raise e\n```\n\n### 2. View Creation\nViews are automatically created/updated in `litellm/proxy/utils.py`:\n\n```python\nasync def check_view_exists(self):\n \"\"\"\n Checks if views exist and creates them if missing.\n Views include LiteLLM_VerificationTokenView, MonthlyGlobalSpend, etc.\n \"\"\"\n expected_views = [\n \"LiteLLM_VerificationTokenView\",\n \"MonthlyGlobalSpend\", \n \"Last30dKeysBySpend\",\n \"Last30dModelsBySpend\",\n # ... more views\n ]\n```\n\nThe system uses `litellm/proxy/db/create_views.py` to handle view creation with error handling for missing relations.\n\n### 3. Schema Evolution\n- **Backward compatibility**: The code handles missing columns/tables gracefully\n- **View recreation**: Views are recreated if schema changes affect them\n- **Error handling**: Specific error codes (P3018, P3009) indicate migration issues\n\n### 4. Migration Commands\nBased on error messages, administrators should run:\n```bash\nprisma migrate resolve --applied <migration_name> # For dirty state\nprisma db push # For schema mismatches \nprisma migrate deploy # For production deployments\n```\n\nThe database schema is designed for multi-tenant SaaS usage with comprehensive audit trails, spending controls, and role-based access management.",
|
||||
"scores": {
|
||||
"accuracy": 4,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 19,
|
||||
"brief_explanation": "Highly detailed and accurate answer that covers all ground truth points plus extensive additional relevant information with specific code examples and file references."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q9",
|
||||
"query": "How does LiteLLM implement fallback logic when a model fails?",
|
||||
"category": "reliability",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 5012,
|
||||
"search_time_s": 0.018,
|
||||
"answer_time_s": 17.469,
|
||||
"total_time_s": 17.486,
|
||||
"answer": "LiteLLM implements a comprehensive fallback system that automatically switches to alternative models when the primary model fails. Here's how it works:\n\n## Fallback Types\n\nLiteLLM supports three types of fallbacks configured in the router:\n\n1. **General fallbacks** (`fallbacks`): Standard fallbacks for any error\n2. **Context window fallbacks** (`context_window_fallbacks`): Specific fallbacks for context window exceeded errors\n3. **Content policy fallbacks** (`content_policy_fallbacks`): Specific fallbacks for content policy violations\n\n## Core Fallback Flow\n\n### 1. Exception Handling and Routing\n\nIn `litellm/router.py`, the main fallback logic is implemented in `async_function_with_fallbacks_common_utils()`:\n\n```python\n# First, check for specific fallback types\nif isinstance(e, litellm.ContextWindowExceededError):\n if context_window_fallbacks is not None:\n context_window_fallback_model_group = self._get_fallback_model_group_from_fallbacks(\n fallbacks=context_window_fallbacks,\n model_group=model_group,\n )\n # Use context window specific fallbacks\n \nelif isinstance(e, litellm.ContentPolicyViolationError):\n if content_policy_fallbacks is not None:\n content_policy_fallback_model_group = self._get_fallback_model_group_from_fallbacks(\n fallbacks=content_policy_fallbacks,\n model_group=model_group,\n )\n # Use content policy specific fallbacks\n```\n\n### 2. Fallback Model Selection\n\nThe `get_fallback_model_group()` function in `litellm/router.py` determines which fallback models to use:\n\n```python\n(fallback_model_group, generic_fallback_idx) = get_fallback_model_group(\n fallbacks=fallbacks, # e.g., [{\"gpt-3.5-turbo\": [\"claude-3-haiku\"]}]\n model_group=model_group,\n)\n\n# If no specific fallback found, check for generic fallback\nif fallback_model_group is None and generic_fallback_idx is not None:\n fallback_model_group = fallbacks[generic_fallback_idx][\"*\"]\n```\n\n### 3. Executing Fallbacks\n\nOnce a fallback model group is identified, the system:\n\n```python\ninput_kwargs.update({\n \"fallback_model_group\": fallback_model_group,\n \"original_model_group\": original_model_group,\n})\n\nresponse = await run_async_fallback(*args, **input_kwargs)\n```\n\n## Streaming Fallbacks\n\nFor streaming responses, LiteLLM handles mid-stream failures in `_acompletion_streaming_iterator()`:\n\n```python\nexcept MidStreamFallbackError as e:\n # Use the router's fallback system\n if e.is_pre_first_chunk or not e.generated_content:\n # No content generated - retry with original messages\n initial_kwargs[\"messages\"] = messages\n else:\n # Add continuation prompt for partial responses\n initial_kwargs[\"messages\"] = messages + [\n {\"role\": \"system\", \"content\": \"Continue from: \"},\n {\"role\": \"assistant\", \"content\": e.generated_content, \"prefix\": True},\n ]\n \n fallback_response = await self.async_function_with_fallbacks_common_utils(...)\n```\n\n## Configuration Management\n\n### Creating Fallbacks\n\nThe `create_fallback()` function in `fallback_management_endpoints.py` allows dynamic fallback configuration:\n\n```python\n# Validate models exist in router\nif data.model not in model_names:\n raise HTTPException(status_code=404, detail=\"Model not found\")\n\n# Update appropriate fallback list based on type\nfallback_key = \"fallbacks\"\nif data.fallback_type == \"context_window\":\n fallback_key = \"context_window_fallbacks\"\nelif data.fallback_type == \"content_policy\":\n fallback_key = \"content_policy_fallbacks\"\n\n# Save to",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer accurately describes the fallback logic in the Router class with specific code examples, covers all fallback types mentioned in the ground truth, and provides comprehensive implementation details."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q9",
|
||||
"query": "How does LiteLLM implement fallback logic when a model fails?",
|
||||
"category": "reliability",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12009,
|
||||
"search_time_s": 0.127,
|
||||
"answer_time_s": 19.294,
|
||||
"total_time_s": 19.421,
|
||||
"answer": "Based on the code context, LiteLLM implements fallback logic through a comprehensive system in both the main completion functions and the Router class. Here's how it works:\n\n## Fallback System Overview\n\nLiteLLM implements three types of fallbacks:\n\n1. **General fallbacks** - For any type of failure\n2. **Context window fallbacks** - For context length exceeded errors\n3. **Content policy fallbacks** - For content policy violations\n\n## Main Completion Function Fallbacks\n\nIn `litellm/main.py`, the basic fallback logic is implemented:\n\n```python\nfallbacks = fallbacks or litellm.model_fallbacks\nif fallbacks is not None:\n response = await async_completion_with_fallbacks(\n **completion_kwargs, \n kwargs={\"fallbacks\": fallbacks, **kwargs}\n )\n```\n\nThe system uses dedicated fallback utility functions:\n- `async_completion_with_fallbacks()` for async operations\n- `completion_with_fallbacks()` for sync operations\n\n## Router-Level Fallback Implementation\n\nThe Router class in `litellm/router.py` provides more sophisticated fallback handling:\n\n### 1. Fallback Configuration\nThe Router accepts multiple fallback configurations:\n\n```python\nfallbacks: List = [], # General fallbacks\ncontext_window_fallbacks: List = [],\ncontent_policy_fallbacks: List = [],\ndefault_fallbacks: Optional[List[str]] = None, # Generic fallbacks across all deployments\n```\n\n### 2. Fallback Processing Logic\nWhen a request fails, the Router follows this process:\n\n1. **Determines the model group** from the failed request\n2. **Retrieves appropriate fallbacks** based on failure type:\n ```python\n fallbacks: Optional[List] = initial_kwargs.get(\"fallbacks\", self.fallbacks)\n context_window_fallbacks: Optional[List] = initial_kwargs.get(\n \"context_window_fallbacks\", self.context_window_fallbacks\n )\n content_policy_fallbacks: Optional[List] = initial_kwargs.get(\n \"content_policy_fallbacks\", self.content_policy_fallbacks\n )\n ```\n\n3. **Calls the common fallback handler**:\n ```python\n fallback_response = await self.async_function_with_fallbacks_common_utils(\n e=e,\n disable_fallbacks=False,\n fallbacks=fallbacks,\n context_window_fallbacks=context_window_fallbacks,\n content_policy_fallbacks=content_policy_fallbacks,\n model_group=model_group,\n args=(),\n kwargs=initial_kwargs,\n )\n ```\n\n### 3. Streaming Response Handling\nFor streaming responses, the Router can seamlessly continue from where it left off:\n\n```python\nif e.is_pre_first_chunk or not e.generated_content:\n # No content generated - retry with original messages\n initial_kwargs[\"messages\"] = messages\nelse:\n # Content was generated - add continuation prompt\n initial_kwargs[\"messages\"] = messages + [\n {\n \"role\": \"system\",\n \"content\": \"You are a helpful assistant... respond in continuation of this text:\",\n }\n ]\n```\n\n## Fallback Types and Management\n\nThe system supports different fallback types through the `FallbackCreateRequest` model in `litellm/types/management_endpoints/router_settings_endpoints.py`:\n\n```python\nfallback_type: Literal[\"general\", \"context_window\", \"content_policy\"] = Field(\n default=\"general\",\n description=\"Type of fallback: 'general' (default), 'context_window', or 'content_policy'\",\n)\n```\n\n## Configuration and Limits\n\nThe Router provides several configuration options for fallback behavior:\n\n- `max_fallbacks`: Maximum number of fallbacks to try (default: 5)\n- `retry_after`: Minimum time to wait before retrying\n- `num_retries`: Number of retries per deployment\n- `allowed_fails`: Number of failures before putting a deployment in cooldown\n\n## Error Handling Integration\n\nThe fallback system integrates with:\n- **Cooldown mechanisms** - Failed deployments are temporarily disabled\n- **Load bal",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer provides highly accurate and comprehensive coverage of LiteLLM's fallback logic with extensive code examples and specific references to files, classes, and functions, going well beyond the ground truth requirements."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q10",
|
||||
"query": "How does LiteLLM support custom callback/logging integrations?",
|
||||
"category": "observability",
|
||||
"approach": "semantic",
|
||||
"context_tokens": 3381,
|
||||
"search_time_s": 0.019,
|
||||
"answer_time_s": 16.034,
|
||||
"total_time_s": 16.053,
|
||||
"answer": "LiteLLM provides comprehensive support for custom callback/logging integrations through several key mechanisms:\n\n## Core Architecture\n\n**LoggingCallbackManager**: The central class (`litellm/litellm_core_utils/logging_callback_manager.py`) manages all callbacks, preventing duplicates and maintaining reasonable limits (MAX_CALLBACKS = 30).\n\n## Callback Types and Registration\n\nLiteLLM supports multiple callback types with automatic routing:\n\n### 1. **Input Callbacks**\n```python\n# Auto-routes async callbacks to _async_input_callback\nadd_litellm_input_callback(callback)\n```\n\n### 2. **Success/Failure Callbacks**\n```python\n# Success callbacks - auto-routes async ones\nadd_litellm_success_callback(callback)\nadd_litellm_async_success_callback(callback)\n\n# Failure callbacks - auto-routes async ones \nadd_litellm_failure_callback(callback)\nadd_litellm_async_failure_callback(callback)\n```\n\n### 3. **Service Callbacks**\n```python\nadd_litellm_service_callback(callback)\n```\n\n## Custom Logger Interface\n\n**CustomLogger Base Class** (`litellm/integrations/custom_logger.py`):\n- Inherit from `CustomLogger` to create custom integrations\n- Implement methods like `log_pre_api_call()` for different events\n- Built-in error handling via `handle_callback_failure()`\n\n## Dynamic Callbacks\n\nThe system supports dynamic callback processing:\n\n```python\n# From _process_dynamic_callback_list\ndef _process_dynamic_callback_list(self, callback_list, dynamic_callbacks_type):\n # Initializes CustomLogger compatible callbacks\n # Handles string callbacks from _known_custom_logger_compatible_callbacks\n```\n\n## Built-in Integrations\n\nLiteLLM includes many pre-built integrations (from `get_custom_logger_compatible_class`):\n- **Observability**: Langfuse, Arize, OpenTelemetry, Datadog, Prometheus\n- **Storage**: S3, GCS, Azure Blob Storage\n- **Analytics**: Braintrust, Langsmith, DeepEval\n- **Security**: Presidio, Lakera AI, OpenAI Moderations\n\n## Async Support\n\nThe system automatically detects and routes async callbacks:\n\n```python\ndef _is_async_callable(self, callback) -> bool:\n # Auto-detection of async callbacks for proper routing\n```\n\n## Configuration\n\n### Proxy Integration\nVia `initialize_callbacks_on_proxy()`:\n- Configure callbacks in `config.yaml`\n- Supports callback-specific parameters\n- Enterprise features for premium users\n\n### String-based Configuration\n```python\n# Known callbacks can be referenced by string\nif callback in litellm._known_custom_logger_compatible_callbacks:\n # Initialize and add to appropriate callback lists\n```\n\n## Error Handling\n\n- Non-blocking logging to prevent API call failures\n- Prometheus metrics for callback failures\n- Comprehensive exception handling in callback execution\n\nThis architecture allows developers to easily integrate custom logging solutions while maintaining performance and reliability of the core LLM functionality.",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer is extremely comprehensive and accurate, providing detailed information about LiteLLM's callback system with specific file paths, class names, and code examples that go well beyond the ground truth while maintaining complete accuracy."
|
||||
}
|
||||
},
|
||||
{
|
||||
"query_id": "q10",
|
||||
"query": "How does LiteLLM support custom callback/logging integrations?",
|
||||
"category": "observability",
|
||||
"approach": "grep",
|
||||
"context_tokens": 12006,
|
||||
"search_time_s": 0.131,
|
||||
"answer_time_s": 19.17,
|
||||
"total_time_s": 19.301,
|
||||
"answer": "LiteLLM provides comprehensive support for custom callback and logging integrations through several key mechanisms:\n\n## Core Architecture\n\n### 1. **LoggingCallbackManager Class**\nLocated in `litellm/litellm_core_utils/logging_callback_manager.py`, this centralized manager handles:\n- Adding/removing callbacks without duplicates\n- Auto-routing async callbacks to appropriate lists\n- Maintaining reasonable callback limits (MAX_CALLBACKS = 30)\n- Managing different callback types (input, success, failure, async variants)\n\n```python\nclass LoggingCallbackManager:\n def add_litellm_success_callback(self, callback: Union[CustomLogger, str, Callable])\n def add_litellm_failure_callback(self, callback: Union[CustomLogger, str, Callable])\n def add_litellm_async_success_callback(self, callback: Union[CustomLogger, Callable, str])\n # ... other methods\n```\n\n### 2. **CustomLogger Base Class**\nThe main interface for custom integrations, supporting:\n- Synchronous and asynchronous event handlers\n- Pre-call checks and filtering\n- Standard logging payload redaction\n- Dynamic callback processing\n\n## Callback Types Supported\n\n### **String-based Callbacks**\n```python\nlitellm.success_callback = [\"langfuse\", \"wandb\", \"slack\"]\n```\n\n### **Custom Logger Classes**\n```python\nclass MyCustomLogger(CustomLogger):\n def log_success_event(self, kwargs, response_obj, start_time, end_time):\n # Custom logging logic\n pass\n \n async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):\n # Async logging logic\n pass\n```\n\n### **Function Callbacks**\n```python\ndef my_callback(kwargs, response_obj, start_time, end_time):\n # Custom function logic\n pass\n\nlitellm.success_callback = [my_callback]\n```\n\n## Dynamic Callback Support\n\nThe `LiteLLMLoggingObject` class in `litellm_core_utils/litellm_logging.py` supports dynamic callbacks:\n\n```python\n# Dynamic callbacks can be passed per request\ndynamic_success_callbacks: Optional[List[Union[str, Callable, CustomLogger]]] = None\ndynamic_failure_callbacks: Optional[List[Union[str, Callable, CustomLogger]]] = None\ndynamic_async_success_callbacks: Optional[List[Union[str, Callable, CustomLogger]]] = None\n```\n\n## Built-in Integrations\n\nLiteLLM includes many pre-built integrations:\n- **Observability**: Langfuse, Weights & Biases, ArizeAI, DeepEval\n- **Alerting**: Slack, email (SendGrid, SMTP, Resend), PagerDuty\n- **Databases**: PostgreSQL, DynamoDB, SQS\n- **Analytics**: OpenMeter, MLflow\n- **Generic API**: Configurable webhook integration\n\n## Generic API Integration\n\nFor custom webhooks, LiteLLM provides the `GenericAPILogger`:\n\n```python\n# Via callback_settings\nlitellm.callback_settings = {\n \"my_webhook\": {\n \"callback_type\": \"generic_api\",\n \"endpoint\": \"https://my-webhook.com/endpoint\",\n \"headers\": {\"Authorization\": \"Bearer token\"},\n \"event_types\": [\"success\", \"failure\"],\n \"max_retries\": 3,\n \"timeout\": 30\n }\n}\nlitellm.success_callback = [\"my_webhook\"]\n```\n\n## Enterprise Features\n\nWhen available, LiteLLM supports enterprise callbacks:\n- **EnterpriseCallbackControls**: Advanced callback management\n- **Enhanced email providers**: SendGrid, SMTP, Resend\n- **PagerDuty integration**: For incident management\n\n## Router Integration\n\nThe Router class automatically integrates with the callback system:\n\n```python\nrouter = Router(\n model_list=[...],\n optional_callbacks=[custom_logger_instance]\n)\n# Router automatically registers deployment tracking callbacks\n```\n\n## Safety Features\n\n1. **Duplicate Prevention**: Callbacks are deduplicated based on type and configuration\n2. **Rate",
|
||||
"scores": {
|
||||
"accuracy": 5,
|
||||
"completeness": 5,
|
||||
"specificity": 5,
|
||||
"relevance": 5,
|
||||
"total": 20,
|
||||
"brief_explanation": "The answer provides comprehensive, accurate details about LiteLLM's callback system with specific file paths, class names, code examples, and covers all key aspects from the ground truth plus additional valuable implementation details."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue