From 20c3a0ffe4d34c39e6e02897c7ec17f3340b539f Mon Sep 17 00:00:00 2001 From: Lucky Lodhi Date: Sun, 18 Jan 2026 18:27:27 +0000 Subject: [PATCH 01/15] fixed litellm params --- litellm/main.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 969cf55a3d6..fb7b0bf8b4a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -565,6 +565,10 @@ async def acompletion( model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None), + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) fallbacks = fallbacks or litellm.model_fallbacks @@ -1291,6 +1295,10 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) if not _should_allow_input_examples( @@ -4368,6 +4376,10 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None), + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) # Await normally @@ -4553,6 +4565,10 @@ def embedding( # noqa: PLR0915 custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) if dynamic_api_key is not None: @@ -5688,6 +5704,10 @@ def text_completion( # noqa: PLR0915 model=model, # type: ignore custom_llm_provider=custom_llm_provider, api_base=api_base, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, # type: ignore + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) if custom_llm_provider == "huggingface": @@ -5978,6 +5998,10 @@ async def amoderation( custom_llm_provider=custom_llm_provider, api_base=optional_params.api_base, api_key=optional_params.api_key, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model or "", + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) except litellm.BadRequestError: # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / not recognized @@ -6043,7 +6067,12 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: func_with_context = partial(ctx.run, func) _, custom_llm_provider, _, _ = get_llm_provider( - model=model, api_base=kwargs.get("api_base", None) + model=model, + api_base=kwargs.get("api_base", None), + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) # Await normally @@ -6147,6 +6176,10 @@ def transcription( custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) # type: ignore if dynamic_api_key is not None: @@ -6322,7 +6355,12 @@ async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent: func_with_context = partial(ctx.run, func) _, custom_llm_provider, _, _ = get_llm_provider( - model=model, api_base=kwargs.get("api_base", None) + model=model, + api_base=kwargs.get("api_base", None), + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) # Await normally @@ -6373,7 +6411,13 @@ def speech( # noqa: PLR0915 model_info = kwargs.get("model_info", None) shared_session = kwargs.get("shared_session", None) model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + ), ) # type: ignore kwargs.pop("tags", []) @@ -6883,6 +6927,10 @@ async def ahealth_check( custom_llm_provider=custom_llm_provider_from_params, api_base=api_base_from_params, api_key=api_key_from_params, + litellm_params=litellm.types.router.LiteLLM_Params( + model=model, + use_litellm_proxy=model_params.get("use_litellm_proxy", False), + ), ) if model in litellm.model_cost and mode is None: mode = litellm.model_cost[model].get("mode") From 0862373b38f08ab97a25f1a7596834f9cec6c4ae Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Mon, 19 Jan 2026 08:22:18 -0800 Subject: [PATCH 02/15] docs: add note about no limits on users/keys/teams in LiteLLM OSS (#19367) Co-authored-by: Cursor Agent --- docs/my-website/docs/proxy/deploy.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 5686e9fd835..7393e73ba87 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -4,6 +4,10 @@ import Image from '@theme/IdealImage'; # Docker, Helm, Terraform +:::info No Limits on LiteLLM OSS +There are **no limits** on the number of users, keys, or teams you can create on LiteLLM OSS. +::: + You can find the Dockerfile to build litellm proxy [here](https://github.com/BerriAI/litellm/blob/main/Dockerfile) > Note: Production requires at least 4 CPU cores and 8 GB RAM. From 0cd7763d5f299c2f32b16a38667c39aff40e6139 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Mon, 19 Jan 2026 08:38:38 -0800 Subject: [PATCH 03/15] Add health check scripts and parallel execution support (#19295) - Add health_check_client.py for monitoring model availability - Add health_check_client_README.md with usage documentation - Add health_check_requirements.txt for dependencies - Add run_parallel_health_checks.ps1 (PowerShell version) - Add run_parallel_health_checks.sh (Bash version) - Organize all scripts under scripts/health_check/ directory --- docker/Dockerfile.health_check | 16 + scripts/health_check/health_check_client.py | 406 ++++++++++++++++++ .../health_check_client_README.md | 246 +++++++++++ .../health_check_requirements.txt | 2 + .../run_parallel_health_checks.ps1 | 69 +++ .../run_parallel_health_checks.sh | 79 ++++ 6 files changed, 818 insertions(+) create mode 100644 docker/Dockerfile.health_check create mode 100644 scripts/health_check/health_check_client.py create mode 100644 scripts/health_check/health_check_client_README.md create mode 100644 scripts/health_check/health_check_requirements.txt create mode 100644 scripts/health_check/run_parallel_health_checks.ps1 create mode 100644 scripts/health_check/run_parallel_health_checks.sh diff --git a/docker/Dockerfile.health_check b/docker/Dockerfile.health_check new file mode 100644 index 00000000000..de62e4bd729 --- /dev/null +++ b/docker/Dockerfile.health_check @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Copy health check script and requirements +COPY scripts/health_check/health_check_client.py /app/health_check_client.py +COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt + +# Install dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Make script executable +RUN chmod +x /app/health_check_client.py + +# Set entrypoint +ENTRYPOINT ["python", "/app/health_check_client.py"] diff --git a/scripts/health_check/health_check_client.py b/scripts/health_check/health_check_client.py new file mode 100644 index 00000000000..337c754a75c --- /dev/null +++ b/scripts/health_check/health_check_client.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +""" +LiteLLM Health Check Client + +A sentinel health check tool that tests all configured models on a LiteLLM proxy. +Similar to HRT's health check system, this script: +- Can read models from YAML config file (like HRT) or fetch from proxy API +- Sends a simple test request to each model concurrently +- Reports health status for each model +- Supports both chat/completion and embedding models +""" + +import asyncio +import json +import os +import sys +import time +from typing import Dict, List, Optional, Tuple + +import httpx +import yaml + + +class LiteLLMHealthCheckClient: + """Client for health checking LiteLLM proxy models.""" + + def __init__( + self, + base_url: str, + api_key: str, + timeout: int = 120, # Match Go implementation's 120s timeout + completion_prompt: str = "Say this is a test", # Match Go implementation + embedding_text: str = "This is a test for vectorization.", # Match Go implementation + ): + """ + Initialize the health check client. + + Args: + base_url: Base URL of the LiteLLM proxy (e.g., https://litellm.example.com) + api_key: API key for authentication + timeout: Request timeout in seconds (default: 120, matching Go implementation) + completion_prompt: Test prompt for chat/completion models + embedding_text: Test text for embedding models + """ + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.timeout = timeout + self.completion_prompt = completion_prompt + self.embedding_text = embedding_text + self.headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + def load_models_from_yaml(self, yaml_path: str) -> List[Dict]: + """ + Load models from a YAML config file (similar to Go implementation). + + Args: + yaml_path: Path to the YAML config file + + Returns: + List of model dictionaries with 'id' and 'mode' keys + """ + try: + with open(yaml_path, "r") as f: + config = yaml.safe_load(f) + + model_list = config.get("model_list", []) + models = [] + + for entry in model_list: + model_name = entry.get("model_name", "") + litellm_params = entry.get("litellm_params", {}) + model_info = litellm_params.get("model_info", {}) + mode = model_info.get("mode", "") + + # Use model_name as the ID (this is what gets sent to the API) + models.append( + { + "id": model_name, + "mode": mode.lower() if mode else "", + "provider": model_info.get("provider", ""), + } + ) + + return models + except Exception as e: + print(f"Error loading models from YAML file {yaml_path}: {e}", file=sys.stderr) + return [] + + async def fetch_models(self, client: httpx.AsyncClient) -> List[Dict]: + """ + Fetch all available models from the proxy API. + + Returns: + List of model dictionaries with 'id' and 'mode' keys + """ + try: + # Try /v1/models first (OpenAI-compatible endpoint) + response = await client.get( + f"{self.base_url}/v1/models", + headers=self.headers, + timeout=self.timeout, + ) + response.raise_for_status() + data = response.json() + models_data = data.get("data", []) + models = [] + for m in models_data: + models.append({"id": m["id"], "mode": "", "provider": ""}) + return models + except Exception as e: + print(f"Error fetching models from /v1/models: {e}", file=sys.stderr) + # Fallback to /model/info endpoint which has more details + try: + response = await client.get( + f"{self.base_url}/model/info", + headers=self.headers, + timeout=self.timeout, + ) + response.raise_for_status() + data = response.json() + if isinstance(data, dict) and "data" in data: + models_data = data["data"] + elif isinstance(data, list): + models_data = data + else: + models_data = [] + + models = [] + for m in models_data: + model_info = m.get("model_info", {}) + mode = model_info.get("mode", "") + models.append( + { + "id": m.get("model_name", m.get("id", "unknown")), + "mode": mode.lower() if mode else "", + "provider": model_info.get("provider", ""), + } + ) + return models + except Exception as e2: + print(f"Error fetching models from /model/info: {e2}", file=sys.stderr) + return [] + + async def check_model_health( + self, client: httpx.AsyncClient, model: Dict + ) -> Tuple[str, Dict]: + """ + Check health of a single model by sending a test request. + + Args: + client: HTTP client + model: Model dictionary with 'id' and 'mode' keys + + Returns: + Tuple of (model_id, result_dict) + """ + model_id = model["id"] + mode = model.get("mode", "") + + start_time = time.time() + result = { + "model": model_id, + "healthy": False, + "error": None, + "response_time_ms": None, + "mode": mode, + } + + try: + # Determine if this is an embedding model + # Check mode first (from config), then fall back to name-based detection + is_embedding = ( + mode == "embedding" + or any( + keyword in model_id.lower() + for keyword in ["embedding", "embed", "text-embedding"] + ) + ) + + if is_embedding: + # Test embedding endpoint (matching Go implementation) + embedding_response = await client.post( + f"{self.base_url}/v1/embeddings", + headers=self.headers, + json={ + "model": model_id, + "input": self.embedding_text, + }, + timeout=self.timeout, + ) + embedding_response.raise_for_status() + embedding_data = embedding_response.json() + dimensions = 0 + if "data" in embedding_data and len(embedding_data["data"]) > 0: + dimensions = len(embedding_data["data"][0].get("embedding", [])) + + result["healthy"] = True + result["mode"] = "embedding" + result["dimensions"] = dimensions + else: + # Test chat completion endpoint (matching Go implementation) + completion_response = await client.post( + f"{self.base_url}/v1/chat/completions", + headers=self.headers, + json={ + "model": model_id, + "messages": [ + {"role": "user", "content": self.completion_prompt} + ], + "max_tokens": 10, # Minimal tokens for health check + }, + timeout=self.timeout, + ) + completion_response.raise_for_status() + completion_data = completion_response.json() + response_text = "" + if "choices" in completion_data and len(completion_data["choices"]) > 0: + response_text = ( + completion_data["choices"][0] + .get("message", {}) + .get("content", "") + ) + + result["healthy"] = True + result["mode"] = "chat" + result["response_text"] = response_text[:100] # Truncate for display + + elapsed_ms = (time.time() - start_time) * 1000 + result["response_time_ms"] = round(elapsed_ms, 2) + + except httpx.HTTPStatusError as e: + result["error"] = f"HTTP {e.response.status_code}: {e.response.text[:200]}" + except httpx.TimeoutException: + result["error"] = f"Request timeout after {self.timeout}s" + except Exception as e: + result["error"] = str(e)[:200] + + return model_id, result + + async def run_health_checks( + self, + models: Optional[List[Dict]] = None, + models_only: Optional[List[str]] = None, + ) -> Dict[str, Dict]: + """ + Run health checks on all models concurrently. + + Args: + models: Optional list of models to check. If None, fetches from proxy. + models_only: Optional list of model IDs to check. If set, only these + models are health-checked (must exist in the models list). + + Returns: + Dictionary mapping model_id to health check result + """ + async with httpx.AsyncClient() as client: + if models is None: + models = await self.fetch_models(client) + + if not models: + print("No models found to health check", file=sys.stderr) + return {} + + if models_only: + allowlist = {m.strip() for m in models_only if m and m.strip()} + models = [m for m in models if m.get("id") in allowlist] + print( + f"Filtering to only check {len(models)} models: {', '.join(sorted(allowlist))}", + file=sys.stderr, + ) + if not models: + print( + "No models matched LITELLM_MODELS_ONLY filter", + file=sys.stderr, + ) + return {} + + print(f"Running health checks on {len(models)} models...", file=sys.stderr) + + # Run all health checks concurrently + tasks = [self.check_model_health(client, model) for model in models] + results_list = await asyncio.gather(*tasks, return_exceptions=True) + + # Convert to dictionary format + results = {} + for result in results_list: + if isinstance(result, Exception): + print( + f"Exception in health check task: {result}", file=sys.stderr + ) + continue + # Type narrowing: after checking it's not an Exception, it's a Tuple + if isinstance(result, tuple) and len(result) == 2: + model_id, result_dict = result + results[model_id] = result_dict + + return results + + def print_results(self, results: Dict[str, Dict], json_output: bool = False): + """ + Print health check results. + + Args: + results: Dictionary of health check results + json_output: If True, output as JSON + """ + if json_output: + print(json.dumps(results, indent=2)) + return + + healthy_count = sum(1 for r in results.values() if r.get("healthy")) + unhealthy_count = len(results) - healthy_count + + # Print detailed results for each model (matching Go output format) + print(f"\n{'='*60}", file=sys.stderr) + print(f"Starting health check queries\n", file=sys.stderr) + + for model_id, result in results.items(): + if result.get("healthy"): + if result.get("mode") == "embedding": + dimensions = result.get("dimensions", 0) + print( + f"---- {model_id} ----\n✅ Success. " + f"Generated embedding vector with {dimensions} dimensions.\n\n", + file=sys.stderr, + ) + else: + response_text = result.get("response_text", "") + print( + f"---- {model_id} ----\n✅ Success. " + f"Response:\n{response_text}\n\n", + file=sys.stderr, + ) + else: + error = result.get("error", "Unknown error") + print(f"---- {model_id} ----\n❌ ERROR: {error}\n\n", file=sys.stderr) + + print(f"{'='*60}", file=sys.stderr) + print(f"Health Check Summary", file=sys.stderr) + print(f"{'='*60}", file=sys.stderr) + print(f"Total models: {len(results)}", file=sys.stderr) + print(f"Healthy: {healthy_count}", file=sys.stderr) + print(f"Unhealthy: {unhealthy_count}", file=sys.stderr) + print(f"{'='*60}\n", file=sys.stderr) + + # Exit with non-zero code if any models are unhealthy + if unhealthy_count > 0: + sys.exit(1) + else: + sys.exit(0) + + +async def main(): + """Main entry point.""" + base_url = os.environ.get("LITELLM_BASE_URL", "http://localhost:4000") + api_key = os.environ.get("LITELLM_API_KEY", "sk-1234") + yaml_path = os.environ.get("LITELLM_MODELS_YAML") + + if not base_url: + print("Error: LITELLM_BASE_URL environment variable not set", file=sys.stderr) + sys.exit(1) + + if not api_key: + print("Error: LITELLM_API_KEY environment variable not set", file=sys.stderr) + sys.exit(1) + + timeout = int(os.environ.get("LITELLM_TIMEOUT", "120")) # Match Go's 120s default + completion_prompt = os.environ.get( + "LITELLM_COMPLETION_PROMPT", "Say this is a test" + ) + embedding_text = os.environ.get( + "LITELLM_EMBEDDING_TEXT", "This is a test for vectorization." + ) + json_output = os.environ.get("LITELLM_JSON_OUTPUT", "").lower() == "true" + # Optional: only health-check these model IDs (comma-separated). E.g.: + # LITELLM_MODELS_ONLY=claude-3.7-sonnet,claude-3.5-sonnet,claude-4.5-haiku + models_only_raw = os.environ.get("LITELLM_MODELS_ONLY", "") + models_only = [m.strip() for m in models_only_raw.split(",") if m.strip()] or None + + client = LiteLLMHealthCheckClient( + base_url=base_url, + api_key=api_key, + timeout=timeout, + completion_prompt=completion_prompt, + embedding_text=embedding_text, + ) + + # Load models from YAML if provided, otherwise fetch from API + models = None + if yaml_path: + models = client.load_models_from_yaml(yaml_path) + if models: + print( + f"Successfully loaded {len(models)} models from {yaml_path}", + file=sys.stderr, + ) + + results = await client.run_health_checks(models=models, models_only=models_only) + client.print_results(results, json_output=json_output) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/health_check/health_check_client_README.md b/scripts/health_check/health_check_client_README.md new file mode 100644 index 00000000000..e3132499e04 --- /dev/null +++ b/scripts/health_check/health_check_client_README.md @@ -0,0 +1,246 @@ +# LiteLLM Health Check Client + +A health check tool for testing all configured models on a LiteLLM proxy. Tests each model with completion/embedding requests and reports health status, errors, and response times. + +## Features + +- **YAML Config Support**: Reads models from YAML config file OR fetches from proxy API +- **Smart Mode Detection**: Detects embedding vs chat models from config or model name +- **Concurrent Testing**: Tests all models concurrently using asyncio +- **Containerized**: Docker image for easy deployment +- **Parallel Execution**: Supports parallel execution for stress testing +- **Configurable**: Customizable timeouts (default 120s) and test prompts + +## Quick Start + +### As a Python Script + +**Option 1: Fetch models from proxy API** +```bash +export LITELLM_BASE_URL="https://litellm.example.com" +export LITELLM_API_KEY="your-api-key" +python scripts/health_check/health_check_client.py +``` + +**Option 2: Use YAML config file** +```bash +export LITELLM_BASE_URL="https://litellm.example.com" +export LITELLM_API_KEY="your-api-key" +export LITELLM_MODELS_YAML="/path/to/config.yaml" +python scripts/health_check/health_check_client.py +``` + +### As a Docker Container + +1. Build the Docker image: + +```bash +docker build -f docker/Dockerfile.health_check -t litellm/litellm-health-check:latest . +``` + +2. Run a single health check: + +```bash +docker run --rm \ + -e LITELLM_BASE_URL="https://litellm.example.com" \ + -e LITELLM_API_KEY="your-api-key" \ + litellm/litellm-health-check:latest +``` + +### Parallel Execution (Stress Testing) + +Run multiple health check containers in parallel: + +**PowerShell:** +```powershell +$env:LITELLM_BASE_URL="https://litellm.example.com" +$env:LITELLM_API_KEY="your-api-key" +.\scripts\health_check\run_parallel_health_checks.ps1 16 +``` + +**Bash/Shell:** +```bash +export LITELLM_BASE_URL="https://litellm.example.com" +export LITELLM_API_KEY="your-api-key" +./scripts/health_check/run_parallel_health_checks.sh 16 +``` + + +## Configuration + +### Environment Variables + +- `LITELLM_BASE_URL` (required): Base URL of the LiteLLM proxy + - Example: `https://litellm.example.com` +- `LITELLM_API_KEY` (required): API key for authentication +- `LITELLM_MODELS_YAML` (optional): Path to YAML config file with model_list + - If provided, reads models from YAML instead of fetching from API + - Example: `/path/to/config.yaml` +- `LITELLM_TIMEOUT` (optional): Request timeout in seconds (default: 120) +- `LITELLM_COMPLETION_PROMPT` (optional): Test prompt for chat/completion models (default: "Say this is a test") +- `LITELLM_EMBEDDING_TEXT` (optional): Test text for embedding models (default: "This is a test for vectorization.") +- `LITELLM_JSON_OUTPUT` (optional): Output results as JSON (default: false) + +## Output + +### Standard Output (Human-Readable) + +Example output format: + +``` +============================================================ +Starting health check queries + +---- gpt-4o ---- +✅ Success. Response: +This is a test + +---- text-embedding-3-small ---- +✅ Success. Generated embedding vector with 1536 dimensions. + +---- gpt-5-codex ---- +❌ ERROR: HTTP 503: Service unavailable + +============================================================ +Health Check Summary +============================================================ +Total models: 47 +Healthy: 45 +Unhealthy: 2 +============================================================ +``` + +Exit code: `0` if all models are healthy, `1` if any models are unhealthy. + +### JSON Output + +When `LITELLM_JSON_OUTPUT=true`, outputs JSON: + +```json +{ + "gpt-4o": { + "model": "gpt-4o", + "healthy": true, + "error": null, + "response_time_ms": 245.67, + "mode": "chat", + "response_text": "This is a test" + }, + "text-embedding-3-small": { + "model": "text-embedding-3-small", + "healthy": true, + "error": null, + "response_time_ms": 123.45, + "mode": "embedding", + "dimensions": 1536 + } +} +``` + +## How It Works + +1. **Model Discovery**: + - If `LITELLM_MODELS_YAML` is set: Reads models from YAML config file + - Otherwise: Queries `/v1/models` (OpenAI-compatible) or `/model/info` to get all configured models +2. **Mode Detection**: + - Checks `mode` field from YAML config, or falls back to model name patterns (embedding, embed, text-embedding) +3. **Concurrent Testing**: + - Chat models: `POST /v1/chat/completions` with configurable prompt (default: "Say this is a test") + - Embedding models: `POST /v1/embeddings` with configurable text (default: "This is a test for vectorization.") +4. **Reporting**: Health status, errors, response times, and response details are reported + +## Use Cases + +### 1. Regular Health Monitoring + +Run as a cron job or scheduled task: + +```bash +# Cron job: Run every 5 minutes +*/5 * * * * /path/to/health_check.sh +``` + +### 2. Load/Stress Testing + +Run multiple health checks in parallel: + +**PowerShell:** +```powershell +.\scripts\health_check\run_parallel_health_checks.ps1 16 +``` + +### 3. CI/CD Integration + +Add to your deployment pipeline: + +```yaml +# GitHub Actions example +- name: Health Check + run: | + docker run --rm \ + -e LITELLM_BASE_URL="${{ secrets.LITELLM_BASE_URL }}" \ + -e LITELLM_API_KEY="${{ secrets.LITELLM_API_KEY }}" \ + litellm/litellm-health-check:latest +``` + +### 4. Kubernetes Deployment + +Deploy as a CronJob: + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: litellm-health-check +spec: + schedule: "*/5 * * * *" # Every 5 minutes + jobTemplate: + spec: + template: + spec: + containers: + - name: health-check + image: litellm/litellm-health-check:latest + env: + - name: LITELLM_BASE_URL + value: "https://litellm.example.com" + - name: LITELLM_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: api-key + restartPolicy: OnFailure +``` + +## Troubleshooting + +### No Models Found + +- Verify `LITELLM_BASE_URL` is correct +- Check that the API key has permissions to list models +- Ensure the proxy is running and accessible +- If using YAML, verify `LITELLM_MODELS_YAML` path is correct + +### Timeout Errors + +- Increase `LITELLM_TIMEOUT` for slower models (default is 120s) +- Check network connectivity to the proxy +- Verify proxy isn't overloaded + +### Authentication Errors + +- Verify `LITELLM_API_KEY` is correct +- Check API key has not expired +- Ensure the key has necessary permissions + +## Dependencies + +- Python 3.11+ +- httpx (for async HTTP requests) +- pyyaml (for YAML config file support) +- Docker or Podman (for containerized execution) +- PowerShell (for parallel execution script on Windows) + +## License + +Same as LiteLLM project. diff --git a/scripts/health_check/health_check_requirements.txt b/scripts/health_check/health_check_requirements.txt new file mode 100644 index 00000000000..c9d2650c884 --- /dev/null +++ b/scripts/health_check/health_check_requirements.txt @@ -0,0 +1,2 @@ +httpx>=0.24.0 +pyyaml>=6.0 diff --git a/scripts/health_check/run_parallel_health_checks.ps1 b/scripts/health_check/run_parallel_health_checks.ps1 new file mode 100644 index 00000000000..856e7f20ec9 --- /dev/null +++ b/scripts/health_check/run_parallel_health_checks.ps1 @@ -0,0 +1,69 @@ +# Parallel LiteLLM Health Check Runner (PowerShell version) +# +# This script runs multiple health check containers in parallel. +# +# Usage: +# $env:LITELLM_BASE_URL="https://litellm.example.com" +# $env:LITELLM_API_KEY="your-api-key" +# .\run_parallel_health_checks.ps1 [num_parallel_jobs] [image_name] +# +# Defaults: +# - num_parallel_jobs: 16 +# - image_name: litellm/litellm-health-check:latest + +param( + [int]$NumParallelJobs = 16, + [string]$ImageName = "litellm/litellm-health-check:latest", + [string]$ContainerRuntime = "docker" +) + +# Set defaults for environment variables if not provided +if (-not $env:LITELLM_BASE_URL) { + $env:LITELLM_BASE_URL = "https://litellm-perf-cache-and-router.onrender.com" + Write-Warning "LITELLM_BASE_URL not set, using default: $env:LITELLM_BASE_URL" +} + +if (-not $env:LITELLM_API_KEY) { + $env:LITELLM_API_KEY = "sk-1234" + Write-Warning "LITELLM_API_KEY not set, using default: $env:LITELLM_API_KEY" +} + +# Check if container runtime is available +$runtimeExists = Get-Command $ContainerRuntime -ErrorAction SilentlyContinue +if (-not $runtimeExists) { + Write-Error "Error: $ContainerRuntime is not installed" + exit 1 +} + +Write-Host "Running $NumParallelJobs parallel health check containers..." -ForegroundColor Yellow +Write-Host "Using image: $ImageName" -ForegroundColor Yellow +Write-Host "Container runtime: $ContainerRuntime" -ForegroundColor Yellow +Write-Host "LiteLLM Base URL: $env:LITELLM_BASE_URL" -ForegroundColor Cyan +Write-Host "" +Write-Host "NOTE: This will run continuously. Press Ctrl+C to stop." -ForegroundColor Red +Write-Host "" +Write-Host "Troubleshooting:" -ForegroundColor Yellow +Write-Host " - If you see 'All connection attempts failed', check:" -ForegroundColor Yellow +Write-Host " 1. Is the LiteLLM proxy running on the expected port?" -ForegroundColor Yellow +Write-Host " 2. Set LITELLM_BASE_URL to the correct URL (e.g., http://host.docker.internal:PORT)" -ForegroundColor Yellow +Write-Host " 3. On Linux, you may need to use the host IP instead of host.docker.internal" -ForegroundColor Yellow +Write-Host "" + +# Run parallel health checks +# This creates an infinite loop that keeps spawning containers +# Each container tests all models, then exits, and a new one starts +while ($true) { + # Start up to NumParallelJobs containers in parallel + 1..$NumParallelJobs | ForEach-Object -Parallel { + $runtime = $using:ContainerRuntime + $imageName = $using:ImageName + $baseUrl = $env:LITELLM_BASE_URL + $apiKey = $env:LITELLM_API_KEY + + & $runtime run --rm ` + -e LITELLM_BASE_URL="$baseUrl" ` + -e LITELLM_API_KEY="$apiKey" ` + -e LITELLM_JSON_OUTPUT="true" ` + $imageName + } -ThrottleLimit $NumParallelJobs +} diff --git a/scripts/health_check/run_parallel_health_checks.sh b/scripts/health_check/run_parallel_health_checks.sh new file mode 100644 index 00000000000..9b6c5d9f393 --- /dev/null +++ b/scripts/health_check/run_parallel_health_checks.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Parallel LiteLLM Health Check Runner (Bash version) +# +# This script runs multiple health check containers in parallel. +# +# Usage: +# export LITELLM_BASE_URL="https://litellm.example.com" +# export LITELLM_API_KEY="your-api-key" +# ./run_parallel_health_checks.sh [num_parallel_jobs] [image_name] [container_runtime] +# +# Defaults: +# - num_parallel_jobs: 16 +# - image_name: litellm/litellm-health-check:latest +# - container_runtime: docker + +set -e + +# Default values +NUM_PARALLEL_JOBS="${1:-16}" +IMAGE_NAME="${2:-litellm/litellm-health-check:latest}" +CONTAINER_RUNTIME="${3:-docker}" + +# Set defaults for environment variables if not provided +if [ -z "$LITELLM_BASE_URL" ]; then + export LITELLM_BASE_URL="https://litellm-perf-cache-and-router.onrender.com" + echo "Warning: LITELLM_BASE_URL not set, using default: $LITELLM_BASE_URL" >&2 +fi + +if [ -z "$LITELLM_API_KEY" ]; then + export LITELLM_API_KEY="sk-1234" + echo "Warning: LITELLM_API_KEY not set, using default: $LITELLM_API_KEY" >&2 +fi + +# Check if container runtime is available +if ! command -v "$CONTAINER_RUNTIME" &> /dev/null; then + echo "Error: $CONTAINER_RUNTIME is not installed" >&2 + exit 1 +fi + +# Print configuration +echo "Running $NUM_PARALLEL_JOBS parallel health check containers..." +echo "Using image: $IMAGE_NAME" +echo "Container runtime: $CONTAINER_RUNTIME" +echo "LiteLLM Base URL: $LITELLM_BASE_URL" +echo "" +echo "NOTE: This will run continuously. Press Ctrl+C to stop." +echo "" +echo "Troubleshooting:" +echo " - If you see 'All connection attempts failed', check:" +echo " 1. Is the LiteLLM proxy running on the expected port?" +echo " 2. Set LITELLM_BASE_URL to the correct URL (e.g., http://host.docker.internal:PORT)" +echo " 3. On Linux, you may need to use the host IP instead of host.docker.internal" +echo "" + +# Function to run a single health check container +run_health_check() { + "$CONTAINER_RUNTIME" run --rm \ + -e LITELLM_BASE_URL="$LITELLM_BASE_URL" \ + -e LITELLM_API_KEY="$LITELLM_API_KEY" \ + -e LITELLM_JSON_OUTPUT="true" \ + "$IMAGE_NAME" +} + +# Run parallel health checks +# This creates an infinite loop that keeps spawning containers +# Each container tests all models, then exits, and a new one starts +while true; do + # Start containers in parallel using background jobs + pids=() + for ((i=1; i<=NUM_PARALLEL_JOBS; i++)); do + run_health_check & + pids+=($!) + done + + # Wait for all background jobs to complete + for pid in "${pids[@]}"; do + wait "$pid" 2>/dev/null || true + done +done From c4013a34b8949c61ae498e2b6ab675052179e2f0 Mon Sep 17 00:00:00 2001 From: Lucky Lodhi Date: Mon, 19 Jan 2026 17:20:18 +0000 Subject: [PATCH 04/15] fix tool call for ollama - #19357 --- litellm/litellm_core_utils/prompt_templates/common_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a8b8b207de4..7790fb83361 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1071,9 +1071,9 @@ def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[s """ message_content = message.get("content") if "reasoning_content" in message: - return message["reasoning_content"], message["content"] + return message["reasoning_content"], message_content elif "reasoning" in message: - return message["reasoning"], message["content"] + return message["reasoning"], message_content elif isinstance(message_content, str): return _parse_content_for_reasoning(message_content) return None, message_content From 74d3b1129068a96ebaac69bf06475dcde8fe1f85 Mon Sep 17 00:00:00 2001 From: Lucky Lodhi Date: Mon, 19 Jan 2026 17:38:29 +0000 Subject: [PATCH 05/15] undid changes --- litellm/main.py | 56 ++++--------------------------------------------- 1 file changed, 4 insertions(+), 52 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index fb7b0bf8b4a..ae27b4145b3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -565,10 +565,6 @@ async def acompletion( model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None), - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) fallbacks = fallbacks or litellm.model_fallbacks @@ -1295,10 +1291,6 @@ def completion( # type: ignore # noqa: PLR0915 custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) if not _should_allow_input_examples( @@ -4376,10 +4368,6 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None), - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) # Await normally @@ -4565,10 +4553,6 @@ def embedding( # noqa: PLR0915 custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) if dynamic_api_key is not None: @@ -5704,10 +5688,6 @@ def text_completion( # noqa: PLR0915 model=model, # type: ignore custom_llm_provider=custom_llm_provider, api_base=api_base, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, # type: ignore - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) if custom_llm_provider == "huggingface": @@ -5998,10 +5978,6 @@ async def amoderation( custom_llm_provider=custom_llm_provider, api_base=optional_params.api_base, api_key=optional_params.api_key, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model or "", - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) except litellm.BadRequestError: # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / not recognized @@ -6067,12 +6043,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: func_with_context = partial(ctx.run, func) _, custom_llm_provider, _, _ = get_llm_provider( - model=model, - api_base=kwargs.get("api_base", None), - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), + model=model, api_base=kwargs.get("api_base", None) ) # Await normally @@ -6176,10 +6147,6 @@ def transcription( custom_llm_provider=custom_llm_provider, api_base=api_base, api_key=api_key, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), ) # type: ignore if dynamic_api_key is not None: @@ -6355,12 +6322,7 @@ async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent: func_with_context = partial(ctx.run, func) _, custom_llm_provider, _, _ = get_llm_provider( - model=model, - api_base=kwargs.get("api_base", None), - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), + model=model, api_base=kwargs.get("api_base", None) ) # Await normally @@ -6411,13 +6373,7 @@ def speech( # noqa: PLR0915 model_info = kwargs.get("model_info", None) shared_session = kwargs.get("shared_session", None) model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - ), + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base ) # type: ignore kwargs.pop("tags", []) @@ -6927,10 +6883,6 @@ async def ahealth_check( custom_llm_provider=custom_llm_provider_from_params, api_base=api_base_from_params, api_key=api_key_from_params, - litellm_params=litellm.types.router.LiteLLM_Params( - model=model, - use_litellm_proxy=model_params.get("use_litellm_proxy", False), - ), ) if model in litellm.model_cost and mode is None: mode = litellm.model_cost[model].get("mode") @@ -7303,4 +7255,4 @@ def __getattr__(name: str) -> Any: global _encoding_cache _encoding_cache = _encoding return _encoding - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") \ No newline at end of file From e817aa713ea94d3b7872904eeee584348e9be135 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 19 Jan 2026 10:16:18 -0800 Subject: [PATCH 06/15] [Fix] Claude Code x Bedrock Invoke fails with `advanced-tool-use-2025-11-20` (#19373) * _filter_unsupported_beta_headers_for_bedrock * test_bedrock_sonnet_4_5_with_advanced_tool_use_beta_header --- .../anthropic_claude3_transformation.py | 91 ++++++++++++++++++- .../test_bedrock_tool_use_beta_header.py | 69 ++++++++++++++ 2 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index fa5002fcad8..293ee1caaf0 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -50,6 +50,12 @@ class AmazonAnthropicClaudeMessagesConfig( DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31" + # Beta header patterns that are not supported by Bedrock Invoke API + # These will be filtered out to prevent 400 "invalid beta flag" errors + UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [ + "advanced-tool-use", # Bedrock Invoke doesn't support advanced-tool-use beta headers + ] + def __init__(self, **kwargs): BaseAnthropicMessagesConfig.__init__(self, **kwargs) AmazonInvokeConfig.__init__(self, **kwargs) @@ -114,7 +120,7 @@ class AmazonAnthropicClaudeMessagesConfig( """ Remove `ttl` field from cache_control in messages. Bedrock doesn't support the ttl field in cache_control. - + Args: anthropic_messages_request: The request dictionary to modify in-place """ @@ -129,6 +135,75 @@ class AmazonAnthropicClaudeMessagesConfig( if isinstance(cache_control, dict) and "ttl" in cache_control: cache_control.pop("ttl", None) + def _supports_extended_thinking_on_bedrock(self, model: str) -> bool: + """ + Check if the model supports extended thinking beta headers on Bedrock. + + On 3rd-party platforms (e.g., Amazon Bedrock), extended thinking is only + supported on: Claude Opus 4.5, Claude Opus 4.1, Opus 4, or Sonnet 4. + + Ref: https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking + + Args: + model: The model name + + Returns: + True if the model supports extended thinking on Bedrock + """ + model_lower = model.lower() + + # Supported models on Bedrock for extended thinking + supported_patterns = [ + "opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5", # Opus 4.5 + "opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1", # Opus 4.1 + "opus-4", "opus_4", # Opus 4 + "sonnet-4", "sonnet_4", # Sonnet 4 + ] + + return any(pattern in model_lower for pattern in supported_patterns) + + def _filter_unsupported_beta_headers_for_bedrock( + self, model: str, beta_set: set + ) -> None: + """ + Remove beta headers that are not supported on Bedrock for the given model. + + Extended thinking beta headers are only supported on specific Claude 4+ models. + Advanced tool use headers are not supported on Bedrock Invoke API. + This prevents 400 "invalid beta flag" errors on Bedrock. + + Note: Bedrock Invoke API fails with a 400 error when unsupported beta headers + are sent, returning: {"message":"invalid beta flag"} + + Args: + model: The model name + beta_set: The set of beta headers to filter in-place + """ + beta_headers_to_remove = set() + + # 1. Filter out beta headers that are universally unsupported on Bedrock Invoke + for beta in beta_set: + for unsupported_pattern in self.UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS: + if unsupported_pattern in beta.lower(): + beta_headers_to_remove.add(beta) + break + + # 2. Filter out extended thinking headers for models that don't support them + extended_thinking_patterns = [ + "extended-thinking", + "interleaved-thinking", + ] + if not self._supports_extended_thinking_on_bedrock(model): + for beta in beta_set: + for pattern in extended_thinking_patterns: + if pattern in beta.lower(): + beta_headers_to_remove.add(beta) + break + + # Remove all filtered headers + for beta in beta_headers_to_remove: + beta_set.discard(beta) + def _get_tool_search_beta_header_for_bedrock( self, model: str, @@ -139,15 +214,15 @@ class AmazonAnthropicClaudeMessagesConfig( ) -> None: """ Adjust tool search beta header for Bedrock. - + Bedrock requires a different beta header for tool search on Opus 4 models when tool search is used without programmatic tool calling or input examples. - + Note: On Amazon Bedrock, server-side tool search is only supported on Claude Opus 4 with the `tool-search-tool-2025-10-19` beta header. - + Ref: https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-search-tool - + Args: model: The model name tool_search_used: Whether tool search is used @@ -228,6 +303,12 @@ class AmazonAnthropicClaudeMessagesConfig( beta_set=beta_set, ) + # Filter out unsupported beta headers for Bedrock (e.g., advanced-tool-use, extended-thinking on non-Opus/Sonnet 4 models) + self._filter_unsupported_beta_headers_for_bedrock( + model=model, + beta_set=beta_set, + ) + if beta_set: anthropic_messages_request["anthropic_beta"] = list(beta_set) diff --git a/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py b/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py new file mode 100644 index 00000000000..635ace016fe --- /dev/null +++ b/tests/pass_through_unit_tests/test_bedrock_tool_use_beta_header.py @@ -0,0 +1,69 @@ +""" +Simple E2E test for Bedrock with advanced-tool-use beta header. + +Tests that LiteLLM correctly filters out the advanced-tool-use-2025-11-20 beta header +for Bedrock Invoke API, which doesn't support it and returns a 400 "invalid beta flag" error. +""" +import os +import sys +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm + + +@pytest.mark.asyncio +async def test_bedrock_sonnet_4_5_with_advanced_tool_use_beta_header(): + """ + Simple E2E test: Call Bedrock Sonnet 4.5 with advanced-tool-use beta header. + + This should work without throwing "invalid beta flag" error because LiteLLM + filters out the advanced-tool-use beta header for Bedrock Invoke API. + """ + litellm._turn_on_debug() + response = await litellm.anthropic.messages.acreate( + model="bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=[{"role": "user", "content": "What is 2+2?"}], + max_tokens=100, + provider_specific_header={ + "custom_llm_provider": "bedrock", + "extra_headers": { + "anthropic-beta": "advanced-tool-use-2025-11-20", + }, + }, + ) + + # Verify response + assert response is not None + assert "content" in response + print(f"✅ Test passed! Response: {response}") + + +@pytest.mark.asyncio +async def test_bedrock_claude_3_5_with_advanced_tool_use_beta_header_filtered(): + """ + Simple E2E test: Call Bedrock Claude 3.5 with advanced-tool-use beta header. + + This should work because the beta header is filtered out by LiteLLM before + sending the request to Bedrock Invoke API. + """ + + response = await litellm.anthropic.messages.acreate( + model="bedrock/invoke/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + messages=[{"role": "user", "content": "What is 2+2?"}], + max_tokens=100, + provider_specific_header={ + "custom_llm_provider": "bedrock", + "extra_headers": { + "anthropic-beta": "advanced-tool-use-2025-11-20", + }, + }, + ) + + # Verify response + assert response is not None + assert "content" in response + print(f"✅ Test passed! Claude 3.5 response (beta header filtered): {response}") + + From 99c4ba7adf8dcbc0d35f5b423f23fea2fbf7bf6b Mon Sep 17 00:00:00 2001 From: Harshit Jain <48647625+Harshit28j@users.noreply.github.com> Date: Mon, 19 Jan 2026 23:57:25 +0530 Subject: [PATCH 07/15] docs: fix bad examples from sdk (#19322) --- docs/my-website/docs/completion/token_usage.md | 4 ++-- docs/my-website/docs/providers/openai/text_to_speech.md | 2 +- docs/my-website/docs/text_to_speech.md | 2 +- docs/my-website/src/pages/token_usage.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/completion/token_usage.md b/docs/my-website/docs/completion/token_usage.md index 0bec6b3f902..d99564765a1 100644 --- a/docs/my-website/docs/completion/token_usage.md +++ b/docs/my-website/docs/completion/token_usage.md @@ -100,7 +100,7 @@ from litellm import cost_per_token prompt_tokens = 5 completion_tokens = 10 -prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)) +prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens) print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar) ``` @@ -162,7 +162,7 @@ print(model_cost) # {'gpt-3.5-turbo': {'max_tokens': 4000, 'input_cost_per_token **Dictionary** ```python -from litellm import register_model +import litellm litellm.register_model({ "gpt-4": { diff --git a/docs/my-website/docs/providers/openai/text_to_speech.md b/docs/my-website/docs/providers/openai/text_to_speech.md index a4aeb9e5257..f4507faa066 100644 --- a/docs/my-website/docs/providers/openai/text_to_speech.md +++ b/docs/my-website/docs/providers/openai/text_to_speech.md @@ -46,7 +46,7 @@ os.environ["OPENAI_API_KEY"] = "sk-.." async def test_async_speech(): speech_file_path = Path(__file__).parent / "speech.mp3" - response = await litellm.aspeech( + response = await aspeech( model="openai/tts-1", voice="alloy", input="the quick brown fox jumped over the lazy dogs", diff --git a/docs/my-website/docs/text_to_speech.md b/docs/my-website/docs/text_to_speech.md index 77d15ccb3a5..667ffc925c1 100644 --- a/docs/my-website/docs/text_to_speech.md +++ b/docs/my-website/docs/text_to_speech.md @@ -46,7 +46,7 @@ os.environ["OPENAI_API_KEY"] = "sk-.." async def test_async_speech(): speech_file_path = Path(__file__).parent / "speech.mp3" - response = await litellm.aspeech( + response = await aspeech( model="openai/tts-1", voice="alloy", input="the quick brown fox jumped over the lazy dogs", diff --git a/docs/my-website/src/pages/token_usage.md b/docs/my-website/src/pages/token_usage.md index 028e010a967..61deb61c94f 100644 --- a/docs/my-website/src/pages/token_usage.md +++ b/docs/my-website/src/pages/token_usage.md @@ -27,7 +27,7 @@ from litellm import cost_per_token prompt_tokens = 5 completion_tokens = 10 -prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)) +prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens) print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar) ``` From 608979c7e998aec23bbaf8e7ab2b5ce127e0161a Mon Sep 17 00:00:00 2001 From: "R.Sicart" Date: Mon, 19 Jan 2026 19:38:41 +0100 Subject: [PATCH 08/15] feat: add support for keda in helm chart (#19337) * feat: add support for keda in helm chart Signed-off-by: R.Sicart * chore: bump chart version --------- Signed-off-by: R.Sicart --- deploy/charts/litellm-helm/Chart.yaml | 2 +- .../litellm-helm/templates/deployment.yaml | 2 +- .../charts/litellm-helm/templates/keda.yaml | 37 +++++++++++++++++++ deploy/charts/litellm-helm/values.yaml | 34 +++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 deploy/charts/litellm-helm/templates/keda.yaml diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index b37597c7c82..8a08f0b4e29 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 +version: 1.1.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 682d97ae3b8..c3e0055e380 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -10,7 +10,7 @@ metadata: {{- toYaml .Values.deploymentLabels | nindent 4 }} {{- end }} spec: - {{- if not .Values.autoscaling.enabled }} + {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }} replicas: {{ .Values.replicaCount }} {{- end }} selector: diff --git a/deploy/charts/litellm-helm/templates/keda.yaml b/deploy/charts/litellm-helm/templates/keda.yaml new file mode 100644 index 00000000000..fe5190fffc6 --- /dev/null +++ b/deploy/charts/litellm-helm/templates/keda.yaml @@ -0,0 +1,37 @@ +{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "litellm.fullname" . }} + labels: + {{- include "litellm.labels" . | nindent 4 }} + {{- if .Values.keda.scaledObject.annotations }} + annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + name: {{ include "litellm.fullname" . }} + pollingInterval: {{ .Values.keda.pollingInterval }} + cooldownPeriod: {{ .Values.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.keda.minReplicas }} + maxReplicaCount: {{ .Values.keda.maxReplicas }} +{{- with .Values.keda.fallback }} + fallback: + failureThreshold: {{ .failureThreshold | default 3 }} + replicas: {{ .replicas | default $.Values.keda.maxReplicas }} +{{- end }} + triggers: +{{- with .Values.keda.triggers }} + {{- toYaml . | nindent 2 }} +{{- end }} + advanced: + restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }} +{{- if .Values.keda.behavior }} + horizontalPodAutoscalerConfig: + behavior: +{{- with .Values.keda.behavior }} +{{- toYaml . | nindent 8 }} +{{- end }} + +{{- end }} +{{- end }} diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index e9e8e75a1fb..54271756998 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -156,6 +156,40 @@ autoscaling: targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 +# Autoscaling with keda is mutually exclusive with hpa +keda: + enabled: false + minReplicas: 1 + maxReplicas: 100 + pollingInterval: 30 + cooldownPeriod: 300 + # fallback: + # failureThreshold: 3 + # replicas: 11 + restoreToOriginalReplicaCount: false + scaledObject: + annotations: {} + triggers: [] + # - type: prometheus + # metadata: + # serverAddress: http://:9090 + # metricName: http_requests_total + # threshold: '100' + # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) + behavior: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + # Additional volumes on the output Deployment definition. volumes: [] # - name: foo From 2ba7d2e82193f5fe3f27e99e2a6e0b641b532c72 Mon Sep 17 00:00:00 2001 From: Connor Luebbehusen Date: Mon, 19 Jan 2026 13:41:28 -0500 Subject: [PATCH 09/15] fix: correct Groq gpt-oss pricing and add cache pricing (#19311) --- litellm/model_prices_and_context_window_backup.json | 8 +++++--- model_prices_and_context_window.json | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 43f7bde5da3..87f3566ae81 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18826,13 +18826,14 @@ "supports_tool_choice": true }, "groq/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 32766, "max_tokens": 32766, "mode": "chat", - "output_cost_per_token": 7.5e-07, + "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -18841,13 +18842,14 @@ "supports_web_search": true }, "groq/openai/gpt-oss-20b": { - "input_cost_per_token": 1e-07, + "cache_read_input_token_cost": 3.75e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 3e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 43f7bde5da3..87f3566ae81 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18826,13 +18826,14 @@ "supports_tool_choice": true }, "groq/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 32766, "max_tokens": 32766, "mode": "chat", - "output_cost_per_token": 7.5e-07, + "output_cost_per_token": 6e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, @@ -18841,13 +18842,14 @@ "supports_web_search": true }, "groq/openai/gpt-oss-20b": { - "input_cost_per_token": 1e-07, + "cache_read_input_token_cost": 3.75e-08, + "input_cost_per_token": 7.5e-08, "litellm_provider": "groq", "max_input_tokens": 131072, "max_output_tokens": 32768, "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 3e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, From 3cdeebb5b848739baba4200523b9c4ee0d3aabff Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Mon, 19 Jan 2026 10:47:56 -0800 Subject: [PATCH 10/15] fix(gcs_bucket): prevent unbounded queue growth due to slow API calls (#19297) --- docs/my-website/docs/proxy/config_settings.md | 1 + litellm/integrations/gcs_bucket/gcs_bucket.py | 242 +++++++++++++++--- litellm/types/integrations/gcs_bucket.py | 1 + 3 files changed, 203 insertions(+), 41 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index b941f21b33e..8b4514f5688 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -603,6 +603,7 @@ router_settings: | GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file | GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds** | GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048** +| GCS_USE_BATCHED_LOGGING | Enable batched logging for GCS. When enabled (default), multiple log payloads are combined into single GCS object uploads (NDJSON format), dramatically reducing API calls. When disabled, sends each log individually as separate GCS objects (legacy behavior). **Default is true** | GCS_PUBSUB_TOPIC_ID | PubSub Topic ID to send LiteLLM SpendLogs to. | GCS_PUBSUB_PROJECT_ID | PubSub Project ID to send LiteLLM SpendLogs to. | GENERIC_AUTHORIZATION_ENDPOINT | Authorization endpoint for generic OAuth providers diff --git a/litellm/integrations/gcs_bucket/gcs_bucket.py b/litellm/integrations/gcs_bucket/gcs_bucket.py index 9190f921d50..3cb62905531 100644 --- a/litellm/integrations/gcs_bucket/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket/gcs_bucket.py @@ -1,9 +1,11 @@ import asyncio +import hashlib import json import os +import time from litellm._uuid import uuid from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from urllib.parse import quote from litellm._logging import verbose_logger @@ -26,19 +28,21 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): super().__init__(bucket_name=bucket_name) - # Init Batch logging settings - self.log_queue: List[GCSLogQueueItem] = [] self.batch_size = int(os.getenv("GCS_BATCH_SIZE", GCS_DEFAULT_BATCH_SIZE)) self.flush_interval = int( os.getenv("GCS_FLUSH_INTERVAL", GCS_DEFAULT_FLUSH_INTERVAL_SECONDS) ) - asyncio.create_task(self.periodic_flush()) + self.use_batched_logging = ( + os.getenv("GCS_USE_BATCHED_LOGGING", str(GCS_DEFAULT_USE_BATCHED_LOGGING).lower()).lower() == "true" + ) self.flush_lock = asyncio.Lock() super().__init__( flush_lock=self.flush_lock, batch_size=self.batch_size, flush_interval=self.flush_interval, ) + self.log_queue: asyncio.Queue[GCSLogQueueItem] = asyncio.Queue() # type: ignore[assignment] + asyncio.create_task(self.periodic_flush()) AdditionalLoggingUtils.__init__(self) if premium_user is not True: @@ -65,8 +69,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): ) if logging_payload is None: raise ValueError("standard_logging_object not found in kwargs") - # Add to logging queue - this will be flushed periodically - self.log_queue.append( + await self.log_queue.put( GCSLogQueueItem( payload=logging_payload, kwargs=kwargs, response_obj=response_obj ) @@ -89,7 +92,9 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): if logging_payload is None: raise ValueError("standard_logging_object not found in kwargs") # Add to logging queue - this will be flushed periodically - self.log_queue.append( + # Use asyncio.Queue.put() for thread-safe concurrent access + # If queue is full, this will block until space is available (backpressure) + await self.log_queue.put( GCSLogQueueItem( payload=logging_payload, kwargs=kwargs, response_obj=response_obj ) @@ -98,28 +103,98 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): except Exception as e: verbose_logger.exception(f"GCS Bucket logging error: {str(e)}") - async def async_send_batch(self): + def _drain_queue_batch(self) -> List[GCSLogQueueItem]: """ - Process queued logs in batch - sends logs to GCS Bucket - - - GCS Bucket does not have a Batch endpoint to batch upload logs - - Instead, we - - collect the logs to flush every `GCS_FLUSH_INTERVAL` seconds - - during async_send_batch, we make 1 POST request per log to GCS Bucket - + Drain items from the queue (non-blocking), respecting batch_size limit. + + This prevents unbounded queue growth when processing is slower than log accumulation. + + Returns: + List of items to process, up to batch_size items """ - if not self.log_queue: - return + items_to_process: List[GCSLogQueueItem] = [] + while len(items_to_process) < self.batch_size: + try: + items_to_process.append(self.log_queue.get_nowait()) + except asyncio.QueueEmpty: + break + return items_to_process - for log_item in self.log_queue: - logging_payload = log_item["payload"] - kwargs = log_item["kwargs"] - response_obj = log_item.get("response_obj", None) or {} + def _generate_batch_object_name(self, date_str: str, batch_id: str) -> str: + """ + Generate object name for a batched log file. + Format: {date}/batch-{batch_id}.ndjson + """ + return f"{date_str}/batch-{batch_id}.ndjson" + def _get_config_key(self, kwargs: Dict[str, Any]) -> str: + """ + Extract a synchronous grouping key from kwargs to group items by GCS config. + This allows us to batch items with the same bucket/credentials together. + + Returns a string key that uniquely identifies the GCS config combination. + This key may contain sensitive information (bucket names, paths) - use _sanitize_config_key() + for logging purposes. + """ + standard_callback_dynamic_params = kwargs.get("standard_callback_dynamic_params", None) or {} + + bucket_name = standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME or "default" + path_service_account = standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json or "default" + + return f"{bucket_name}|{path_service_account}" + + def _sanitize_config_key(self, config_key: str) -> str: + """ + Create a sanitized version of the config key for logging. + Uses a hash to avoid exposing sensitive bucket names or service account paths. + + Returns a short hash prefix for safe logging. + """ + hash_obj = hashlib.sha256(config_key.encode('utf-8')) + return f"config-{hash_obj.hexdigest()[:8]}" + + def _group_items_by_config(self, items: List[GCSLogQueueItem]) -> Dict[str, List[GCSLogQueueItem]]: + """ + Group items by their GCS config (bucket + credentials). + This ensures items with different configs are processed separately. + + Returns a dict mapping config_key -> list of items with that config. + """ + grouped: Dict[str, List[GCSLogQueueItem]] = {} + for item in items: + config_key = self._get_config_key(item["kwargs"]) + if config_key not in grouped: + grouped[config_key] = [] + grouped[config_key].append(item) + return grouped + + def _combine_payloads_to_ndjson(self, items: List[GCSLogQueueItem]) -> str: + """ + Combine multiple log payloads into newline-delimited JSON (NDJSON) format. + Each line is a valid JSON object representing one log entry. + """ + lines = [] + for item in items: + logging_payload = item["payload"] + json_line = json.dumps(logging_payload, default=str, ensure_ascii=False) + lines.append(json_line) + return "\n".join(lines) + + async def _send_grouped_batch(self, items: List[GCSLogQueueItem], config_key: str) -> Tuple[int, int]: + """ + Send a batch of items that share the same GCS config. + + Returns: + (success_count, error_count) + """ + if not items: + return (0, 0) + + first_kwargs = items[0]["kwargs"] + + try: gcs_logging_config: GCSLoggingConfig = await self.get_gcs_logging_config( - kwargs + first_kwargs ) headers = await self.construct_request_headers( @@ -127,24 +202,92 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): service_account_json=gcs_logging_config["path_service_account"], ) bucket_name = gcs_logging_config["bucket_name"] - object_name = self._get_object_name(kwargs, logging_payload, response_obj) + + current_date = self._get_object_date_from_datetime(datetime.now(timezone.utc)) + batch_id = f"{int(time.time() * 1000)}-{uuid.uuid4().hex[:8]}" + object_name = self._generate_batch_object_name(current_date, batch_id) + combined_payload = self._combine_payloads_to_ndjson(items) + + await self._log_json_data_on_gcs( + headers=headers, + bucket_name=bucket_name, + object_name=object_name, + logging_payload=combined_payload, + ) + + success_count = len(items) + error_count = 0 + return (success_count, error_count) + + except Exception as e: + success_count = 0 + error_count = len(items) + verbose_logger.exception( + f"GCS Bucket error logging batch payload to GCS bucket: {str(e)}" + ) + return (success_count, error_count) - try: - await self._log_json_data_on_gcs( - headers=headers, - bucket_name=bucket_name, - object_name=object_name, - logging_payload=logging_payload, - ) - except Exception as e: - # don't let one log item fail the entire batch - verbose_logger.exception( - f"GCS Bucket error logging payload to GCS bucket: {str(e)}" - ) - pass + async def _send_individual_logs(self, items: List[GCSLogQueueItem]) -> None: + """ + Send each log individually as separate GCS objects (legacy behavior). + This is used when GCS_USE_BATCHED_LOGGING is disabled. + """ + for item in items: + await self._send_single_log_item(item) - # Clear the queue after processing - self.log_queue.clear() + async def _send_single_log_item(self, item: GCSLogQueueItem) -> None: + """ + Send a single log item to GCS as an individual object. + """ + try: + gcs_logging_config: GCSLoggingConfig = await self.get_gcs_logging_config( + item["kwargs"] + ) + + headers = await self.construct_request_headers( + vertex_instance=gcs_logging_config["vertex_instance"], + service_account_json=gcs_logging_config["path_service_account"], + ) + bucket_name = gcs_logging_config["bucket_name"] + + object_name = self._get_object_name( + kwargs=item["kwargs"], + logging_payload=item["payload"], + response_obj=item["response_obj"], + ) + + await self._log_json_data_on_gcs( + headers=headers, + bucket_name=bucket_name, + object_name=object_name, + logging_payload=item["payload"], + ) + except Exception as e: + verbose_logger.exception( + f"GCS Bucket error logging individual payload to GCS bucket: {str(e)}" + ) + + async def async_send_batch(self): + """ + Process queued logs - sends logs to GCS Bucket. + + If `GCS_USE_BATCHED_LOGGING` is enabled (default), batches multiple log payloads + into single GCS object uploads (NDJSON format), dramatically reducing API calls. + + If disabled, sends each log individually as separate GCS objects (legacy behavior). + """ + items_to_process = self._drain_queue_batch() + + if not items_to_process: + return + + if self.use_batched_logging: + grouped_items = self._group_items_by_config(items_to_process) + + for config_key, group_items in grouped_items.items(): + await self._send_grouped_batch(group_items, config_key) + else: + await self._send_individual_logs(items_to_process) def _get_object_name( self, kwargs: Dict, logging_payload: StandardLoggingPayload, response_obj: Any @@ -186,7 +329,6 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): "start_time_utc is required for getting a payload from GCS Bucket" ) - # Try current day, next day, and previous day dates_to_try = [ start_time_utc, start_time_utc + timedelta(days=1), @@ -230,5 +372,23 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils): def _get_object_date_from_datetime(self, datetime_obj: datetime) -> str: return datetime_obj.strftime("%Y-%m-%d") + async def flush_queue(self): + """ + Override flush_queue to work with asyncio.Queue. + """ + await self.async_send_batch() + self.last_flush_time = time.time() + + async def periodic_flush(self): + """ + Override periodic_flush to work with asyncio.Queue. + """ + while True: + await asyncio.sleep(self.flush_interval) + verbose_logger.debug( + f"GCS Bucket periodic flush after {self.flush_interval} seconds" + ) + await self.flush_queue() + async def async_health_check(self) -> IntegrationHealthCheckStatus: raise NotImplementedError("GCS Bucket does not support health check") diff --git a/litellm/types/integrations/gcs_bucket.py b/litellm/types/integrations/gcs_bucket.py index 2be2acab2f2..b297246f4f3 100644 --- a/litellm/types/integrations/gcs_bucket.py +++ b/litellm/types/integrations/gcs_bucket.py @@ -12,6 +12,7 @@ else: GCS_DEFAULT_BATCH_SIZE = 2048 GCS_DEFAULT_FLUSH_INTERVAL_SECONDS = 20 +GCS_DEFAULT_USE_BATCHED_LOGGING = True class GCSLoggingConfig(TypedDict): From a82467d679671c98348ecd4277907fa0bc3324a2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 19 Jan 2026 14:05:47 -0800 Subject: [PATCH 11/15] [Feat] - Add self hosted Claude Code Plugin Marketplace (#19378) * init schema * init endpoints * fix: claude_code_marketplace_router * refactor * fix: claude_code_marketplace_router * claude_code_marketplace_router --- .../claude_code_endpoints/__init__.py | 11 + .../claude_code_marketplace.py | 533 ++++++++++++++++++ litellm/proxy/proxy_config.yaml | 4 +- litellm/proxy/proxy_server.py | 4 + litellm/proxy/schema.prisma | 17 + litellm/types/proxy/claude_code_endpoints.py | 116 ++++ .../test_claude_code_marketplace.py | 145 +++++ 7 files changed, 828 insertions(+), 2 deletions(-) create mode 100644 litellm/proxy/anthropic_endpoints/claude_code_endpoints/__init__.py create mode 100644 litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py create mode 100644 litellm/types/proxy/claude_code_endpoints.py create mode 100644 tests/pass_through_unit_tests/test_claude_code_marketplace.py diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/__init__.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/__init__.py new file mode 100644 index 00000000000..0d1a5a20836 --- /dev/null +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/__init__.py @@ -0,0 +1,11 @@ +""" +Claude Code Endpoints + +Provides endpoints for Claude Code plugin marketplace integration. +""" + +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( + router as claude_code_marketplace_router, +) + +__all__ = ["claude_code_marketplace_router"] diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py new file mode 100644 index 00000000000..7c212020a3d --- /dev/null +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -0,0 +1,533 @@ +""" +CLAUDE CODE MARKETPLACE + +Provides a registry/discovery layer for Claude Code plugins. +Plugins are stored as metadata + git source references in LiteLLM database. +Actual plugin files are hosted on GitHub/GitLab/Bitbucket. + +Endpoints: +/claude-code/marketplace.json - GET - List plugins for Claude Code discovery +/claude-code/plugins - POST - Register a plugin +/claude-code/plugins - GET - List plugins (admin) +/claude-code/plugins/{name} - GET - Get plugin details +/claude-code/plugins/{name}/enable - POST - Enable a plugin +/claude-code/plugins/{name}/disable - POST - Disable a plugin +/claude-code/plugins/{name} - DELETE - Delete a plugin +""" + +import json +import re +from datetime import datetime, timezone +from typing import Any, Dict + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import JSONResponse + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.types.proxy.claude_code_endpoints import ( + ListPluginsResponse, + PluginListItem, + RegisterPluginRequest, +) + +router = APIRouter() + + +async def _get_prisma_client(): + """Get the prisma client from proxy_server.""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + return prisma_client + + +@router.get( + "/claude-code/marketplace.json", + tags=["Claude Code Marketplace"], +) +async def get_marketplace(): + """ + Serve marketplace.json for Claude Code plugin discovery. + + This endpoint is accessed by Claude Code CLI when users run: + - claude plugin marketplace add + - claude plugin install @ + + Returns: + Marketplace catalog with list of available plugins and their git sources. + + Example: + ```bash + claude plugin marketplace add http://localhost:4000/claude-code/marketplace.json + claude plugin install my-plugin@litellm + ``` + """ + try: + prisma_client = await _get_prisma_client() + + plugins = await prisma_client.db.litellm_claudecodeplugintable.find_many( + where={"enabled": True} + ) + + plugin_list = [] + for plugin in plugins: + try: + manifest = json.loads(plugin.manifest_json) + except json.JSONDecodeError: + verbose_proxy_logger.warning( + f"Plugin {plugin.name} has invalid manifest JSON, skipping" + ) + continue + + # Source must be specified for URL-based marketplaces + if "source" not in manifest: + verbose_proxy_logger.warning( + f"Plugin {plugin.name} has no source field, skipping" + ) + continue + + entry: Dict[str, Any] = { + "name": plugin.name, + "source": manifest["source"], + } + + if plugin.version: + entry["version"] = plugin.version + if plugin.description: + entry["description"] = plugin.description + if "author" in manifest: + entry["author"] = manifest["author"] + if "homepage" in manifest: + entry["homepage"] = manifest["homepage"] + if "keywords" in manifest: + entry["keywords"] = manifest["keywords"] + if "category" in manifest: + entry["category"] = manifest["category"] + + plugin_list.append(entry) + + marketplace = { + "name": "litellm", + "owner": {"name": "LiteLLM", "email": "support@litellm.ai"}, + "plugins": plugin_list, + } + + return JSONResponse(content=marketplace) + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error generating marketplace: {e}") + raise HTTPException( + status_code=500, + detail={"error": f"Failed to generate marketplace: {str(e)}"}, + ) + + +@router.post( + "/claude-code/plugins", + tags=["Claude Code Marketplace"], + dependencies=[Depends(user_api_key_auth)], +) +async def register_plugin( + request: RegisterPluginRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Register a plugin in the LiteLLM marketplace. + + LiteLLM acts as a registry/discovery layer. Plugins are hosted on + GitHub/GitLab/Bitbucket. Claude Code will clone from the git source + when users install. + + Parameters: + - name: Plugin name (kebab-case) + - source: Git source reference (github or url format) + - version: Semantic version (optional) + - description: Plugin description (optional) + - author: Author information (optional) + - homepage: Plugin homepage URL (optional) + - keywords: Search keywords (optional) + - category: Plugin category (optional) + + Returns: + Registration status and plugin information. + + Example: + ```bash + curl -X POST http://localhost:4000/claude-code/plugins \\ + -H "Authorization: Bearer sk-..." \\ + -H "Content-Type: application/json" \\ + -d '{ + "name": "my-plugin", + "source": {"source": "github", "repo": "org/my-plugin"}, + "version": "1.0.0", + "description": "My awesome plugin" + }' + ``` + """ + try: + prisma_client = await _get_prisma_client() + + # Validate name format + if not re.match(r"^[a-z0-9-]+$", request.name): + raise HTTPException( + status_code=400, + detail={ + "error": "Plugin name must be kebab-case (lowercase letters, numbers, hyphens)" + }, + ) + + # Validate source format + source = request.source + source_type = source.get("source") + + if source_type == "github": + if "repo" not in source: + raise HTTPException( + status_code=400, + detail={ + "error": "GitHub source must include 'repo' field (e.g., 'org/repo')" + }, + ) + elif source_type == "url": + if "url" not in source: + raise HTTPException( + status_code=400, + detail={ + "error": "URL source must include 'url' field (e.g., 'https://github.com/org/repo.git')" + }, + ) + else: + raise HTTPException( + status_code=400, + detail={"error": "source.source must be 'github' or 'url'"}, + ) + + # Build manifest for storage + manifest: Dict[str, Any] = { + "name": request.name, + "source": request.source, + } + if request.version: + manifest["version"] = request.version + if request.description: + manifest["description"] = request.description + if request.author: + manifest["author"] = request.author.model_dump(exclude_none=True) + if request.homepage: + manifest["homepage"] = request.homepage + if request.keywords: + manifest["keywords"] = request.keywords + if request.category: + manifest["category"] = request.category + + # Check if plugin exists + existing = await prisma_client.db.litellm_claudecodeplugintable.find_unique( + where={"name": request.name} + ) + + if existing: + plugin = await prisma_client.db.litellm_claudecodeplugintable.update( + where={"name": request.name}, + data={ + "version": request.version, + "description": request.description, + "manifest_json": json.dumps(manifest), + "files_json": "{}", + "updated_at": datetime.now(timezone.utc), + }, + ) + action = "updated" + else: + plugin = await prisma_client.db.litellm_claudecodeplugintable.create( + data={ + "name": request.name, + "version": request.version, + "description": request.description, + "manifest_json": json.dumps(manifest), + "files_json": "{}", + "enabled": True, + "created_at": datetime.now(timezone.utc), + "updated_at": datetime.now(timezone.utc), + "created_by": user_api_key_dict.user_id, + } + ) + action = "created" + + verbose_proxy_logger.info(f"Plugin {request.name} {action} successfully") + + return { + "status": "success", + "action": action, + "plugin": { + "id": plugin.id, + "name": plugin.name, + "version": plugin.version, + "description": plugin.description, + "source": request.source, + "enabled": plugin.enabled, + }, + } + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error registering plugin: {e}") + raise HTTPException( + status_code=500, + detail={"error": f"Registration failed: {str(e)}"}, + ) + + +@router.get( + "/claude-code/plugins", + tags=["Claude Code Marketplace"], + dependencies=[Depends(user_api_key_auth)], + response_model=ListPluginsResponse, +) +async def list_plugins( + enabled_only: bool = False, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + List all plugins in the marketplace. + + Parameters: + - enabled_only: If true, only return enabled plugins + + Returns: + List of plugins with their metadata. + """ + try: + prisma_client = await _get_prisma_client() + + where = {"enabled": True} if enabled_only else {} + plugins = await prisma_client.db.litellm_claudecodeplugintable.find_many( + where=where, + order_by={"created_at": "desc"}, + ) + + return ListPluginsResponse( + plugins=[ + PluginListItem( + id=p.id, + name=p.name, + version=p.version, + description=p.description, + enabled=p.enabled, + created_at=p.created_at.isoformat() if p.created_at else None, + updated_at=p.updated_at.isoformat() if p.updated_at else None, + ) + for p in plugins + ], + count=len(plugins), + ) + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error listing plugins: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) + + +@router.get( + "/claude-code/plugins/{plugin_name}", + tags=["Claude Code Marketplace"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_plugin( + plugin_name: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Get details of a specific plugin. + + Parameters: + - plugin_name: The name of the plugin + + Returns: + Plugin details including source and metadata. + """ + try: + prisma_client = await _get_prisma_client() + + plugin = await prisma_client.db.litellm_claudecodeplugintable.find_unique( + where={"name": plugin_name} + ) + + if not plugin: + raise HTTPException( + status_code=404, + detail={"error": f"Plugin '{plugin_name}' not found"}, + ) + + manifest = json.loads(plugin.manifest_json) if plugin.manifest_json else {} + + return { + "id": plugin.id, + "name": plugin.name, + "version": plugin.version, + "description": plugin.description, + "source": manifest.get("source"), + "author": manifest.get("author"), + "homepage": manifest.get("homepage"), + "keywords": manifest.get("keywords"), + "category": manifest.get("category"), + "enabled": plugin.enabled, + "created_at": plugin.created_at.isoformat() if plugin.created_at else None, + "updated_at": plugin.updated_at.isoformat() if plugin.updated_at else None, + "created_by": plugin.created_by, + } + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error getting plugin: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) + + +@router.post( + "/claude-code/plugins/{plugin_name}/enable", + tags=["Claude Code Marketplace"], + dependencies=[Depends(user_api_key_auth)], +) +async def enable_plugin( + plugin_name: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Enable a disabled plugin. + + Parameters: + - plugin_name: The name of the plugin to enable + """ + try: + prisma_client = await _get_prisma_client() + + plugin = await prisma_client.db.litellm_claudecodeplugintable.find_unique( + where={"name": plugin_name} + ) + if not plugin: + raise HTTPException( + status_code=404, + detail={"error": f"Plugin '{plugin_name}' not found"}, + ) + + await prisma_client.db.litellm_claudecodeplugintable.update( + where={"name": plugin_name}, + data={"enabled": True, "updated_at": datetime.now(timezone.utc)}, + ) + + verbose_proxy_logger.info(f"Plugin {plugin_name} enabled") + return {"status": "success", "message": f"Plugin '{plugin_name}' enabled"} + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error enabling plugin: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) + + +@router.post( + "/claude-code/plugins/{plugin_name}/disable", + tags=["Claude Code Marketplace"], + dependencies=[Depends(user_api_key_auth)], +) +async def disable_plugin( + plugin_name: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Disable a plugin without deleting it. + + Parameters: + - plugin_name: The name of the plugin to disable + """ + try: + prisma_client = await _get_prisma_client() + + plugin = await prisma_client.db.litellm_claudecodeplugintable.find_unique( + where={"name": plugin_name} + ) + if not plugin: + raise HTTPException( + status_code=404, + detail={"error": f"Plugin '{plugin_name}' not found"}, + ) + + await prisma_client.db.litellm_claudecodeplugintable.update( + where={"name": plugin_name}, + data={"enabled": False, "updated_at": datetime.now(timezone.utc)}, + ) + + verbose_proxy_logger.info(f"Plugin {plugin_name} disabled") + return {"status": "success", "message": f"Plugin '{plugin_name}' disabled"} + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error disabling plugin: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) + + +@router.delete( + "/claude-code/plugins/{plugin_name}", + tags=["Claude Code Marketplace"], + dependencies=[Depends(user_api_key_auth)], +) +async def delete_plugin( + plugin_name: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Delete a plugin from the marketplace. + + Parameters: + - plugin_name: The name of the plugin to delete + """ + try: + prisma_client = await _get_prisma_client() + + plugin = await prisma_client.db.litellm_claudecodeplugintable.find_unique( + where={"name": plugin_name} + ) + if not plugin: + raise HTTPException( + status_code=404, + detail={"error": f"Plugin '{plugin_name}' not found"}, + ) + + await prisma_client.db.litellm_claudecodeplugintable.delete( + where={"name": plugin_name} + ) + + verbose_proxy_logger.info(f"Plugin {plugin_name} deleted") + return {"status": "success", "message": f"Plugin '{plugin_name}' deleted"} + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"Error deleting plugin: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index cf852805f83..646a062b720 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -2,7 +2,7 @@ model_list: - model_name: gemini/* litellm_params: model: gemini/* - - model_name: claude-sonnet-4-5-20250929 + - model_name: -claude-sonnet-4-5-20250929 litellm_params: model: bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0 model_info: @@ -40,7 +40,7 @@ model_list: model_info: litellm_provider: bedrock_converse mode: chat - - model_name: azure-claude-opus-4-5 + - model_name: claude-sonnet-4-5-20250929 litellm_params: model: azure_ai/claude-opus-4-5 api_base: https://krish-mh44t553-eastus2.services.ai.azure.com diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f4e68c481a0..c3a4de314e5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -207,6 +207,9 @@ from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_rout from litellm.proxy.anthropic_endpoints.skills_endpoints import ( router as anthropic_skills_router, ) +from litellm.proxy.anthropic_endpoints.claude_code_endpoints import ( + claude_code_marketplace_router, +) from litellm.proxy.auth.auth_checks import ( ExperimentalUIJWTToken, get_team_object, @@ -10499,6 +10502,7 @@ app.include_router(llm_passthrough_router) app.include_router(mcp_management_router) app.include_router(anthropic_router) app.include_router(anthropic_skills_router) +app.include_router(claude_code_marketplace_router) app.include_router(google_router) app.include_router(langfuse_router) app.include_router(pass_through_router) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 71b398c59a4..22888f6d3af 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -863,3 +863,20 @@ model LiteLLM_SkillsTable { updated_at DateTime @default(now()) @updatedAt updated_by String? } + +// Claude Code Marketplace - stores plugins for Claude Code integration +model LiteLLM_ClaudeCodePluginTable { + id String @id @default(uuid()) + name String @unique // Plugin name (kebab-case) + version String? // Semantic version + description String? // Plugin description + manifest_json String // Full plugin.json as JSON string + files_json String // All files as JSON: {"path": "content"} + enabled Boolean @default(true) + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + created_by String? + + @@index([name]) + @@map("litellm_claudecodeplugin") +} diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py new file mode 100644 index 00000000000..663b182b805 --- /dev/null +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -0,0 +1,116 @@ +""" +Claude Code Marketplace endpoint types for LiteLLM Proxy +""" + +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field + + +class PluginAuthor(BaseModel): + """Plugin author information.""" + + name: str = Field(..., description="Author name") + email: Optional[str] = Field(None, description="Author email") + + +class PluginOwner(BaseModel): + """Marketplace owner information.""" + + name: str = Field(..., description="Owner name") + email: Optional[str] = Field(None, description="Owner email") + + +class RegisterPluginRequest(BaseModel): + """ + Request body for registering a plugin in the marketplace. + + LiteLLM acts as a registry/discovery layer. Plugins are hosted on + GitHub/GitLab/Bitbucket and referenced by their git source. + """ + + name: str = Field( + ..., + description="Plugin name (kebab-case, e.g., 'my-plugin')", + pattern=r"^[a-z0-9-]+$", + ) + source: Dict[str, str] = Field( + ..., + description=( + "Git source reference. Supported formats:\n" + "- GitHub: {'source': 'github', 'repo': 'org/repo'}\n" + "- Git URL: {'source': 'url', 'url': 'https://github.com/org/repo.git'}" + ), + ) + version: Optional[str] = Field("1.0.0", description="Semantic version") + description: Optional[str] = Field(None, description="Plugin description") + author: Optional[PluginAuthor] = Field(None, description="Plugin author") + homepage: Optional[str] = Field(None, description="Plugin homepage URL") + keywords: Optional[List[str]] = Field(None, description="Search keywords") + category: Optional[str] = Field(None, description="Plugin category") + + +class PluginResponse(BaseModel): + """Plugin information in API responses.""" + + id: str = Field(..., description="Plugin unique ID") + name: str = Field(..., description="Plugin name") + version: Optional[str] = Field(None, description="Plugin version") + description: Optional[str] = Field(None, description="Plugin description") + source: Dict[str, str] = Field(..., description="Git source reference") + enabled: bool = Field(..., description="Whether plugin is enabled") + + +class RegisterPluginResponse(BaseModel): + """Response from plugin registration.""" + + status: str = Field(..., description="Operation status") + action: str = Field(..., description="Action taken (created/updated)") + plugin: PluginResponse = Field(..., description="Plugin information") + + +class PluginListItem(BaseModel): + """Plugin item in list responses.""" + + id: str + name: str + version: Optional[str] + description: Optional[str] + enabled: bool + created_at: Optional[str] + updated_at: Optional[str] + + +class ListPluginsResponse(BaseModel): + """Response from listing plugins.""" + + plugins: List[PluginListItem] + count: int + + +class MarketplacePluginEntry(BaseModel): + """Plugin entry in marketplace.json.""" + + name: str + source: Dict[str, str] + version: Optional[str] = None + description: Optional[str] = None + author: Optional[PluginAuthor] = None + homepage: Optional[str] = None + keywords: Optional[List[str]] = None + category: Optional[str] = None + + +class MarketplaceResponse(BaseModel): + """ + Marketplace catalog response. + + This format is consumed by Claude Code CLI. + See: https://docs.anthropic.com/en/docs/claude-code/plugins + """ + + name: str = Field(..., description="Marketplace identifier") + owner: PluginOwner = Field(..., description="Marketplace owner") + plugins: List[MarketplacePluginEntry] = Field( + default_factory=list, description="Available plugins" + ) diff --git a/tests/pass_through_unit_tests/test_claude_code_marketplace.py b/tests/pass_through_unit_tests/test_claude_code_marketplace.py new file mode 100644 index 00000000000..b4ba30e9c71 --- /dev/null +++ b/tests/pass_through_unit_tests/test_claude_code_marketplace.py @@ -0,0 +1,145 @@ +""" +Tests for Claude Code Marketplace endpoints. + +Tests: +1. Register a plugin +2. Get marketplace.json (list enabled plugins) +""" + +import os +import sys +import time + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.proxy_server import LitellmUserRoles +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.caching.caching import DualCache +from litellm.types.proxy.claude_code_endpoints import RegisterPluginRequest + +# Import the functions we're testing +from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( + register_plugin, + get_marketplace, +) + +proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) + + +@pytest.fixture +def prisma_client(): + from litellm.proxy.proxy_cli import append_query_params + + params = {"connection_limit": 100, "pool_timeout": 60} + database_url = os.getenv("DATABASE_URL") + modified_url = append_query_params(database_url, params) + os.environ["DATABASE_URL"] = modified_url + + prisma_client = PrismaClient( + database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj + ) + + litellm.proxy.proxy_server.litellm_proxy_budget_name = ( + f"litellm-proxy-budget-{time.time()}" + ) + + return prisma_client + + +@pytest.mark.asyncio +async def test_register_plugin(prisma_client): + """Test registering a plugin in the marketplace.""" + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + + await litellm.proxy.proxy_server.prisma_client.connect() + + # Create a unique plugin name for this test + plugin_name = f"test-plugin-{int(time.time())}" + + request = RegisterPluginRequest( + name=plugin_name, + source={"source": "github", "repo": "test-org/test-repo"}, + version="1.0.0", + description="Test plugin for unit tests", + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="test-user", + ) + + response = await register_plugin( + request=request, + user_api_key_dict=user_api_key_dict, + ) + + assert response["status"] == "success" + assert response["action"] == "created" + assert response["plugin"]["name"] == plugin_name + assert response["plugin"]["version"] == "1.0.0" + assert response["plugin"]["enabled"] is True + + # Cleanup - delete the plugin + await prisma_client.db.litellm_claudecodeplugintable.delete( + where={"name": plugin_name} + ) + + +@pytest.mark.asyncio +async def test_get_marketplace(prisma_client): + """Test getting marketplace.json with registered plugins.""" + setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) + setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") + + await litellm.proxy.proxy_server.prisma_client.connect() + + # First register a plugin + plugin_name = f"test-marketplace-plugin-{int(time.time())}" + + request = RegisterPluginRequest( + name=plugin_name, + source={"source": "github", "repo": "test-org/marketplace-test"}, + version="2.0.0", + description="Test plugin for marketplace test", + ) + + user_api_key_dict = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-1234", + user_id="test-user", + ) + + await register_plugin( + request=request, + user_api_key_dict=user_api_key_dict, + ) + + # Now get the marketplace + response = await get_marketplace() + + # Response is a JSONResponse, get the body + import json + body = json.loads(response.body.decode()) + + assert body["name"] == "litellm" + assert "plugins" in body + + # Find our plugin in the list + our_plugin = next( + (p for p in body["plugins"] if p["name"] == plugin_name), + None + ) + assert our_plugin is not None + assert our_plugin["source"] == {"source": "github", "repo": "test-org/marketplace-test"} + assert our_plugin["version"] == "2.0.0" + + # Cleanup + await prisma_client.db.litellm_claudecodeplugintable.delete( + where={"name": plugin_name} + ) From 13bcecb13ee06bcd5677c004e1239476d45517b8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 19 Jan 2026 14:18:59 -0800 Subject: [PATCH 12/15] docs: fix doc title --- cookbook/ai_coding_tool_guides/index.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/ai_coding_tool_guides/index.json b/cookbook/ai_coding_tool_guides/index.json index f879292aeff..3e71670d623 100644 --- a/cookbook/ai_coding_tool_guides/index.json +++ b/cookbook/ai_coding_tool_guides/index.json @@ -110,7 +110,7 @@ ] }, { - "title": "Use Web Search with Claude Code (across OpenAI/Anthropic/Gemini/etc.)", + "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)", "description": "This is a guide for using Web Search with Claude Code via LiteLLM.", "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch", "date": "2026-01-17", From 270b41b0f486adad81b7e1ddfa74c191a6d7b382 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Mon, 19 Jan 2026 17:01:38 -0800 Subject: [PATCH 13/15] Simplify file comments (#19382) --- scripts/health_check/health_check_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/health_check/health_check_client.py b/scripts/health_check/health_check_client.py index 337c754a75c..4848735bb7b 100644 --- a/scripts/health_check/health_check_client.py +++ b/scripts/health_check/health_check_client.py @@ -3,8 +3,8 @@ LiteLLM Health Check Client A sentinel health check tool that tests all configured models on a LiteLLM proxy. -Similar to HRT's health check system, this script: -- Can read models from YAML config file (like HRT) or fetch from proxy API +This script: +- Can read models from YAML config file or fetch from proxy API - Sends a simple test request to each model concurrently - Reports health status for each model - Supports both chat/completion and embedding models From 818913ee23100ca60210eaad33a6aacc6db60882 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 19 Jan 2026 18:28:55 -0800 Subject: [PATCH 14/15] [Fix] Fix Pass through routes to work with server root path (#19383) * test_build_full_path_with_root_default * fix pt feat --- .../pass_through_endpoints.py | 29 +++- .../test_pass_through_endpoints.py | 149 +++++++++++++++++- 2 files changed, 173 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 75d8253a904..996ee7412c9 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -51,6 +51,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import _read_request_body +from litellm.proxy.utils import get_server_root_path from litellm.secret_managers.main import get_secret_str from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.passthrough_endpoints.pass_through_endpoints import ( @@ -1973,10 +1974,26 @@ class InitPassThroughEndpointHelpers: _registered_pass_through_routes.clear() @staticmethod - def get_registered_pass_through_endpoints_keys() -> List[str]: + def get_all_registered_pass_through_routes() -> List[str]: """Get all registered pass-through endpoints from the registry""" return list(_registered_pass_through_routes.keys()) + @staticmethod + def _build_full_path_with_root(path: str) -> str: + """ + Build full path by prepending server root path if needed. + + Args: + path: The relative path to build + + Returns: + Full path with server root prepended (if root is not "/") + """ + root_path = get_server_root_path() + if root_path == "/": + return path + return f"{root_path}{path}" + @staticmethod def is_registered_pass_through_route(route: str) -> bool: """ @@ -2003,7 +2020,9 @@ class InitPassThroughEndpointHelpers: parts = key.split(":", 2) # Split into [endpoint_id, type, path] if len(parts) == 3: route_type = parts[1] - registered_path = parts[2] + registered_path = InitPassThroughEndpointHelpers._build_full_path_with_root( + parts[2] + ) if route_type == "exact" and route == registered_path: return True elif route_type == "subpath": @@ -2021,7 +2040,9 @@ class InitPassThroughEndpointHelpers: parts = key.split(":", 2) # Split into [endpoint_id, type, path] if len(parts) == 3: route_type = parts[1] - registered_path = parts[2] + registered_path = InitPassThroughEndpointHelpers._build_full_path_with_root( + parts[2] + ) if route_type == "exact" and route == registered_path: return _registered_pass_through_routes[key] @@ -2085,7 +2106,7 @@ async def initialize_pass_through_endpoints( # mark the ones that are visited in the list # remove the ones that are not visited from the list registered_pass_through_endpoints = ( - InitPassThroughEndpointHelpers.get_registered_pass_through_endpoints_keys() + InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() ) visited_endpoints = set() diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index c585089c7be..a39c95f7118 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1897,8 +1897,8 @@ async def test_add_litellm_data_to_request_adds_headers_to_metadata(): The fix ensures headers are available in data["metadata"]["headers"] so guardrails can validate User-Agent, API keys, and other header-based checks. """ - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request # Create mock request with headers including User-Agent mock_request = MagicMock(spec=Request) @@ -1954,3 +1954,150 @@ async def test_add_litellm_data_to_request_adds_headers_to_metadata(): # Also verify proxy_server_request has headers (original location) assert "proxy_server_request" in result assert "headers" in result["proxy_server_request"] + + +def test_build_full_path_with_root_default(): + """ + Test _build_full_path_with_root with default root path (/) + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + + with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root: + # Test with default root path + mock_get_root.return_value = "/" + + result = InitPassThroughEndpointHelpers._build_full_path_with_root("/api/v1/endpoint") + assert result == "/api/v1/endpoint" + + +def test_build_full_path_with_root_custom(): + """ + Test _build_full_path_with_root with custom root path + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + + with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root: + # Test with custom root path /proxy + mock_get_root.return_value = "/proxy" + + result = InitPassThroughEndpointHelpers._build_full_path_with_root("/api/v1/endpoint") + assert result == "/proxy/api/v1/endpoint" + + +def test_build_full_path_with_root_nested(): + """ + Test _build_full_path_with_root with nested root path + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + + with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root: + # Test with nested root path /api/v2 + mock_get_root.return_value = "/api/v2" + + result = InitPassThroughEndpointHelpers._build_full_path_with_root("/endpoint") + assert result == "/api/v2/endpoint" + + +def test_is_registered_pass_through_route_with_custom_root(): + """ + Test is_registered_pass_through_route correctly handles server root path + + When server has a custom root path like /proxy, the registered path + should be constructed by prepending the root to match incoming routes. + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, + ) + + # Clear the registry first + _registered_pass_through_routes.clear() + + # Register a pass-through route with endpoint format: {endpoint_id}:exact:{path} + endpoint_id = "test-endpoint-123" + path = "/api/endpoint" + route_key = f"{endpoint_id}:exact:{path}" + _registered_pass_through_routes[route_key] = { + "target": "http://example.com", + "headers": {}, + } + + with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root: + # Test with custom root path /proxy + mock_get_root.return_value = "/proxy" + + # Should match when request route includes the root path + assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/proxy/api/endpoint") is True + + # Should not match when request route doesn't include root path + assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/api/endpoint") is False + + # Test with default root path + mock_get_root.return_value = "/" + + # Should match with default root + assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/api/endpoint") is True + + # Should not match with root prepended when root is / + assert InitPassThroughEndpointHelpers.is_registered_pass_through_route("/proxy/api/endpoint") is False + + # Clean up + _registered_pass_through_routes.clear() + + +def test_get_registered_pass_through_route_with_custom_root(): + """ + Test get_registered_pass_through_route correctly handles server root path + + When server has a custom root path, the method should return the correct + endpoint configuration by matching the full path including the root. + """ + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + _registered_pass_through_routes, + ) + + # Clear the registry first + _registered_pass_through_routes.clear() + + # Register a pass-through route + endpoint_id = "test-endpoint-456" + path = "/chat/completions" + target_config = { + "target": "http://api.example.com/v1/chat/completions", + "headers": {"Authorization": "Bearer token123"}, + "forward_headers": True, + } + route_key = f"{endpoint_id}:exact:{path}" + _registered_pass_through_routes[route_key] = target_config + + with patch("litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_server_root_path") as mock_get_root: + # Test with custom root path /litellm + mock_get_root.return_value = "/litellm" + + # Should return config when request route includes root path + result = InitPassThroughEndpointHelpers.get_registered_pass_through_route("/litellm/chat/completions") + assert result is not None + assert result["target"] == "http://api.example.com/v1/chat/completions" + assert result["headers"]["Authorization"] == "Bearer token123" + + # Should return None when route doesn't match + result = InitPassThroughEndpointHelpers.get_registered_pass_through_route("/chat/completions") + assert result is None + + # Test with default root path + mock_get_root.return_value = "/" + + # Should return config with default root + result = InitPassThroughEndpointHelpers.get_registered_pass_through_route("/chat/completions") + assert result is not None + assert result["target"] == "http://api.example.com/v1/chat/completions" + + # Clean up + _registered_pass_through_routes.clear() From d48df6c17d2e1c669e8a1c5412ed6f5a553ee8b8 Mon Sep 17 00:00:00 2001 From: Connor Luebbehusen Date: Mon, 19 Jan 2026 22:00:24 -0500 Subject: [PATCH 15/15] fix: correct us.anthropic.claude-opus-4-5 In-region pricing (#19310) --- litellm/model_prices_and_context_window_backup.json | 8 ++++---- model_prices_and_context_window.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 87f3566ae81..4599cafe708 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26720,15 +26720,15 @@ "tool_use_system_prompt_tokens": 159 }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 87f3566ae81..4599cafe708 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26720,15 +26720,15 @@ "tool_use_system_prompt_tokens": 159 }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_read_input_token_cost": 5e-07, - "input_cost_per_token": 5e-06, + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 2.5e-05, + "output_cost_per_token": 2.75e-05, "search_context_cost_per_query": { "search_context_size_high": 0.01, "search_context_size_low": 0.01,